Author: tmckay
Date: 2012-12-13 22:20:57 +0000 (Thu, 13 Dec 2012)
New Revision: 5591
Modified:
branches/elephant/sage/python/sage/aviary/aviaryoperations.py
Log:
More cleanup after refactor
Modified: branches/elephant/sage/python/sage/aviary/aviaryoperations.py
===================================================================
--- branches/elephant/sage/python/sage/aviary/aviaryoperations.py 2012-12-13 21:54:54 UTC
(rev 5590)
+++ branches/elephant/sage/python/sage/aviary/aviaryoperations.py 2012-12-13 22:20:57 UTC
(rev 5591)
@@ -27,7 +27,10 @@
the Suds module. It is meant as a debugging aid during development.
'''
_on = False
- sudslogs = {"suds.client": None, "suds.transport": None,
"suds.xsd.schema": None, "suds.wsdl": None}
+ sudslogs = {"suds.client": None,
+ "suds.transport": None,
+ "suds.xsd.schema": None,
+ "suds.wsdl": None}
@classmethod
def set(cls, flag, home):
@@ -69,62 +72,67 @@
#the clients that we pool so that we can call set_options on the
#transport.
-def _get_host(name, servers):
- '''
- Lookup a host in a dictionary produced by sage.util.host_list.
- The host may be just a hostname or hostname:port.
- Return the scheme and URL for the host. The scheme is separated
- out for convenience (although it could easily be parsed from the
- URL)
- '''
- scheme = ""
- host = ""
- port = None
-
- # Support hostnames with ports, too.
- name = socket.getfqdn(name)
- if ":" in name:
- u = parse_URL(name)
- name = u.host
- port = u.port
-
- if name in servers:
- url = None
- urls = servers[name]
- if port is not None:
- # find the matching port
- for u in urls:
- if u.port == port:
- url = u
- break
-
- elif len(urls) > 0:
- url = random.sample(urls, 1)[0]
-
- if url:
- scheme = url.scheme
- host = str(url)
- # A particular method name is going to be appended to path,
- # so ensure the final "/" here.
- if not host.endswith("/"):
- host += "/"
- return scheme, host
-
-# Nice, friendly strings for error messages on lookup
-_nice = {"JOB": "job service",
- "QUERY_SERVER": "query service",
- "HADOOP": "hadoop service"}
-
class BaseServerList(object):
- def __init__(self):
+ # Nice, friendly strings for error messages on lookup
+ _nice = {"JOB": "job service",
+ "QUERY_SERVER": "query service",
+ "HADOOP": "hadoop service"}
+
+ def __init__(self, subtype):
self.servers = None
+ self.should_retry = False
+ try:
+ self.nice = BaseServerList._nice[subtype]
+ except:
+ self.nice = subtype
+
def _get_host_set(self):
hosts = set()
for k,v in self.servers.iteritems():
for url in v:
hosts.add(k+":"+url.port)
return hosts
+
+ def _get_host(self, name):
+ '''
+ Lookup a host in a dictionary produced by sage.util.host_list.
+ The host may be just a hostname or hostname:port.
+ Return the scheme and URL for the host. The scheme is separated
+ out for convenience (although it could be parsed from the URL)
+ '''
+ scheme = ""
+ host = ""
+ port = None
+
+ # Support hostnames with ports, too.
+ name = socket.getfqdn(name)
+ if ":" in name:
+ u = parse_URL(name)
+ name = u.host
+ port = u.port
+
+ if name in self.servers:
+ url = None
+ urls = self.servers[name]
+ if port is not None:
+ # find the matching port
+ for u in urls:
+ if u.port == port:
+ url = u
+ break
+
+ elif len(urls) > 0:
+ url = random.sample(urls, 1)[0]
+
+ if url:
+ scheme = url.scheme
+ host = str(url)
+ # A particular method name is going to be appended to path,
+ # so ensure the final "/" here.
+ if not host.endswith("/"):
+ host += "/"
+ return scheme, host
def get_hosts(self):
return self._get_host_set()
@@ -136,22 +144,17 @@
Allow the cached list to be refreshed on demand.
'''
def __init__(self, locator, resource, subtype):
+ super(ServerList, self).__init__(subtype)
# Since we have a dynamic server list, failed operations
# may be retried
self.should_retry = True
self._lock = Lock()
- self.servers = None
self.locator = locator
self.resource = resource
self.subtype = subtype
- try:
- self.nice = _nice[subtype]
- except:
- self.nice = subtype
-
def _refresh(self):
log.debug("AviaryOperations: refresh server list for %s %s" \
% (self.resource, self.subtype))
@@ -193,7 +196,7 @@
try:
if self.servers is None or refresh:
self._refresh()
- scheme, host = _get_host(machine, self.servers)
+ scheme, host = self._get_host(machine)
finally:
self._lock.release()
return scheme, host
@@ -251,9 +254,8 @@
messages. Predefind values are "JOB" and "QUERY_SERVER".
The scheme for a URL will be "http" if not specified.
'''
- # Fixed server list, there is no point to a retry on failed ops.
- self.should_retry = False
-
+ super(FixedServerList, self).__init__(subtype)
+
# Replace any occurrence of locahost with output of gethostname()
# before parsing to match Machine fields of QMF objects later on.
host = socket.gethostname()
@@ -264,18 +266,13 @@
default_port=port,
default_path=path)
- try:
- self.nice = _nice[subtype]
- except:
- self.nice = subtype
-
def find_server(self, machine, *args):
'''
Lookup a URL in the fixed server list by hostname or hostname:port.
Return the scheme and URL for the specified host if found or raise
an exception if the host could not be found.
'''
- scheme, host = _get_host(machine, self.servers)
+ scheme, host = self._get_host(machine)
if host == "":
log.info("AviaryOperations: failed to locate %s on %s" \
% (self.nice, machine))
@@ -335,7 +332,7 @@
if resource == self.resource and \
(not subtype or self.subtype == subtype):
return self.servers.get_hosts()
- return []
+ return set()
def _set_client_info(self, client, refresh=False):
# See if the server_name is a full URL. If so, we'll use it.