[PATCH] mirrorlist_client.wsgi use select() waiting for server

[Matt Domsch]

> From 7cd05b296ab426c386e99c3ff6f7143fbf6ed052 Mon Sep 17 00:00:00 2001
From: Matt Domsch <Matt_Domsch(a)dell.com&gt;
Date: Wed, 12 May 2010 08:59:16 -0500
Subject: [PATCH] mirrorlist_client: use select() waiting on the response from
mirrorlist_server

Client was spinning waiting for read() to complete, during the time
the server was doing its thinking.  Instead, use select() to sleep
until the server has data to return.  This should reduce CPU time
spent in the client considerably.
---
 mirrorlist-server/mirrorlist_client.wsgi |   15 ++++++++++-----
 1 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/mirrorlist-server/mirrorlist_client.wsgi
b/mirrorlist-server/mirrorlist_client.wsgi
index cc4416c..15b3a15 100755
--- a/mirrorlist-server/mirrorlist_client.wsgi
+++ b/mirrorlist-server/mirrorlist_client.wsgi
@@ -4,7 +4,7 @@
 #  by Matt Domsch <Matt_Domsch(a)dell.com&gt;
 # Licensed under the MIT/X11 license
 
-import socket
+import socket, select
 import cPickle as pickle
 from string import zfill, atoi, strip, replace
 from paste.wsgiwrappers import *
@@ -32,24 +32,29 @@ def get_mirrorlist(d):
     s.shutdown(socket.SHUT_WR)
     del p
 
+    # wait for other end to start writing
+    expiry = datetime.utcnow() + timedelta(seconds=request_timeout)
+    rlist, wlist, xlist = select.select([s],[],[],request_timeout)
+    if len(rlist) == 0:
+        s.shutdown(socket.SHUT_RD)
+        raise socket.timeout
+    
     readlen = 0
     resultsize = ''
     while readlen < 10:
         resultsize += s.recv(10 - readlen)
         readlen = len(resultsize)
     resultsize = atoi(resultsize)
-    
-    expiry = datetime.utcnow() + timedelta(seconds=request_timeout)
+
     readlen = 0
     p = ''
     while readlen < resultsize and datetime.utcnow() < expiry:
         p += s.recv(resultsize - readlen)
         readlen = len(p)
-        
-    s.shutdown(socket.SHUT_RD)
     results = pickle.loads(p)
     del p
 
+    s.shutdown(socket.SHUT_RD)
     return results
 
 def real_client_ip(xforwardedfor):
-- 
1.7.0.1


-- 
Matt Domsch
Technology Strategist
Dell | Office of the CTO

[Ricky Zhou]

On 2010-05-12 09:07:35 AM, Matt Domsch wrote:
>  >From 7cd05b296ab426c386e99c3ff6f7143fbf6ed052 Mon Sep 17 00:00:00
> 2001
>  From: Matt Domsch <Matt_Domsch(a)dell.com&gt;
>  Date: Wed, 12 May 2010 08:59:16 -0500
>  Subject: [PATCH] mirrorlist_client: use select() waiting on the response from
> mirrorlist_server
>  
>  Client was spinning waiting for read() to complete, during the time
>  the server was doing its thinking.  Instead, use select() to sleep
>  until the server has data to return.  This should reduce CPU time
>  spent in the client considerably.
>  ---
>   mirrorlist-server/mirrorlist_client.wsgi |   15 ++++++++++-----
>   1 files changed, 10 insertions(+), 5 deletions(-)
>  
>  diff --git a/mirrorlist-server/mirrorlist_client.wsgi
> b/mirrorlist-server/mirrorlist_client.wsgi
>  index cc4416c..15b3a15 100755
>  --- a/mirrorlist-server/mirrorlist_client.wsgi
>  +++ b/mirrorlist-server/mirrorlist_client.wsgi
>  @@ -4,7 +4,7 @@
>   #  by Matt Domsch <Matt_Domsch(a)dell.com&gt;
>   # Licensed under the MIT/X11 license
>   
>  -import socket
>  +import socket, select
>   import cPickle as pickle
>   from string import zfill, atoi, strip, replace
>   from paste.wsgiwrappers import *
>  @@ -32,24 +32,29 @@ def get_mirrorlist(d):
>       s.shutdown(socket.SHUT_WR)
>       del p
>   
>  +    # wait for other end to start writing
>  +    expiry = datetime.utcnow() + timedelta(seconds=request_timeout)
>  +    rlist, wlist, xlist = select.select([s],[],[],request_timeout)
>  +    if len(rlist) == 0:
>  +        s.shutdown(socket.SHUT_RD)
>  +        raise socket.timeout
>  +    
>       readlen = 0
>       resultsize = ''
>       while readlen < 10:
>           resultsize += s.recv(10 - readlen)
>           readlen = len(resultsize)
>       resultsize = atoi(resultsize)
>  -    
>  -    expiry = datetime.utcnow() + timedelta(seconds=request_timeout)
>  +
>       readlen = 0
>       p = ''
>       while readlen < resultsize and datetime.utcnow() < expiry:
>           p += s.recv(resultsize - readlen)
>           readlen = len(p)
>  -        
>  -    s.shutdown(socket.SHUT_RD)
>       results = pickle.loads(p)
>       del p
>   
>  +    s.shutdown(socket.SHUT_RD)
>       return results
>   
>   def real_client_ip(xforwardedfor):
>  -- 
>  1.7.0.1 
+1

Thanks,
Ricky

[Mike McGrath]

On Wed, 12 May 2010, Ricky Zhou wrote:

>  On 2010-05-12 09:07:35 AM, Matt Domsch wrote:
>  > >From 7cd05b296ab426c386e99c3ff6f7143fbf6ed052 Mon Sep 17 00:00:00 2001
>  > From: Matt Domsch <Matt_Domsch(a)dell.com&gt;
>  > Date: Wed, 12 May 2010 08:59:16 -0500
>  > Subject: [PATCH] mirrorlist_client: use select() waiting on the response from
> mirrorlist_server
>  >
>  > Client was spinning waiting for read() to complete, during the time
>  > the server was doing its thinking.  Instead, use select() to sleep
>  > until the server has data to return.  This should reduce CPU time
>  > spent in the client considerably.
>  > ---
>  >  mirrorlist-server/mirrorlist_client.wsgi |   15 ++++++++++-----
>  >  1 files changed, 10 insertions(+), 5 deletions(-)
>  >
>  > diff --git a/mirrorlist-server/mirrorlist_client.wsgi
> b/mirrorlist-server/mirrorlist_client.wsgi
>  > index cc4416c..15b3a15 100755
>  > --- a/mirrorlist-server/mirrorlist_client.wsgi
>  > +++ b/mirrorlist-server/mirrorlist_client.wsgi
>  > @@ -4,7 +4,7 @@
>  >  #  by Matt Domsch <Matt_Domsch(a)dell.com&gt;
>  >  # Licensed under the MIT/X11 license
>  >
>  > -import socket
>  > +import socket, select
>  >  import cPickle as pickle
>  >  from string import zfill, atoi, strip, replace
>  >  from paste.wsgiwrappers import *
>  > @@ -32,24 +32,29 @@ def get_mirrorlist(d):
>  >      s.shutdown(socket.SHUT_WR)
>  >      del p
>  >
>  > +    # wait for other end to start writing
>  > +    expiry = datetime.utcnow() + timedelta(seconds=request_timeout)
>  > +    rlist, wlist, xlist = select.select([s],[],[],request_timeout)
>  > +    if len(rlist) == 0:
>  > +        s.shutdown(socket.SHUT_RD)
>  > +        raise socket.timeout
>  > +
>  >      readlen = 0
>  >      resultsize = ''
>  >      while readlen < 10:
>  >          resultsize += s.recv(10 - readlen)
>  >          readlen = len(resultsize)
>  >      resultsize = atoi(resultsize)
>  > -
>  > -    expiry = datetime.utcnow() + timedelta(seconds=request_timeout)
>  > +
>  >      readlen = 0
>  >      p = ''
>  >      while readlen < resultsize and datetime.utcnow() < expiry:
>  >          p += s.recv(resultsize - readlen)
>  >          readlen = len(p)
>  > -
>  > -    s.shutdown(socket.SHUT_RD)
>  >      results = pickle.loads(p)
>  >      del p
>  >
>  > +    s.shutdown(socket.SHUT_RD)
>  >      return results
>  >
>  >  def real_client_ip(xforwardedfor):
>  > --
>  > 1.7.0.1
>  +1
>  
+1

	-Mike