From: Jiri Pirko <jiri(a)mellanox.com>
So far, each resource is checked and loaded one by one. That results
in many rpc calls. On my 20ms line, this makes ~4secs for one slave.
Bulk the checks and loads together which allows to be done in ~1sec.
Signed-off-by: Jiri Pirko <jiri(a)mellanox.com>
---
lnst/Controller/Machine.py | 68 +++++++++++++++++++++++++++++-----------------
lnst/Slave/NetTestSlave.py | 10 +++++++
2 files changed, 53 insertions(+), 25 deletions(-)
diff --git a/lnst/Controller/Machine.py b/lnst/Controller/Machine.py
index 38a3646..2290e2f 100644
--- a/lnst/Controller/Machine.py
+++ b/lnst/Controller/Machine.py
@@ -257,18 +257,15 @@ class Machine(object):
for cls_name, cls in device_classes:
classes.extend(reversed(self._get_base_classes(cls)))
+ minfos = []
for cls in classes:
if cls is object:
continue
- module_name = cls.__module__
- module = sys.modules[module_name]
- filename = module.__file__
+ minfos.append(self._get_module_info(cls.__module__))
- if filename[-3:] == "pyc":
- filename = filename[:-1]
+ self.sync_resources(minfos)
- res_hash = self.sync_resource(module_name, filename)
- self.rpc_call("load_cached_module", module_name, res_hash)
+ self.rpc_call("load_cached_modules", minfos)
for cls_name, cls in device_classes:
module_name = cls.__module__
@@ -348,15 +345,10 @@ class Machine(object):
for cls in reversed(classes):
if cls is object or cls is BaseTestModule:
continue
- m_name = cls.__module__
- m = sys.modules[m_name]
- filename = m.__file__
- if filename[-3:] == "pyc":
- filename = filename[:-1]
-
- res_hash = self.sync_resource(m_name, filename)
-
- self.rpc_call("load_cached_module", m_name, res_hash)
+ minfo = self._get_module_info(cls.__module__)
+ self.sync_resource(minfo)
+ self.rpc_call("load_cached_module",
+ minfo["name"], minfo["res_hash"])
logging.info("Host %s executing job %d: %s" %
(self._id, job.id, str(job)))
@@ -518,19 +510,45 @@ class Machine(object):
local_file.close()
self.rpc_call("finish_copy_from", remote_path)
- def sync_resource(self, res_name, file_path):
- digest = sha256sum(file_path)
+ def _get_module_info(self, name):
+ module = sys.modules[name]
+ filepath = module.__file__
+
+ if filepath[-3:] == "pyc":
+ filepath = filepath[:-1]
+
+ return {"name": name, "filepath": filepath}
+
+ def _module_info_res_hash(self, minfo):
+ if not "res_hash" in minfo:
+ minfo["res_hash"] = sha256sum(minfo["filepath"])
- if not self.rpc_call("has_resource", digest):
- msg = "Transfering %s to machine %s as '%s'" % (file_path,
+ def sync_resource(self, minfo):
+ self._module_info_res_hash(minfo)
+
+ if not self.rpc_call("has_resource", minfo["res_hash"]):
+ msg = "Transfering %s to machine %s as '%s'" %
(minfo["filepath"],
self.get_id(),
- res_name)
+ minfo["name"])
logging.debug(msg)
- remote_path = self.copy_file_to_machine(file_path)
- self.rpc_call("add_resource_to_cache",
- "file", remote_path, res_name)
- return digest
+ remote_path = self.copy_file_to_machine(minfo["filepath"])
+ self.rpc_call("add_resource_to_cache", "file",
+ remote_path, minfo["name"])
+
+ def sync_resources(self, minfos):
+ for minfo in minfos:
+ self._module_info_res_hash(minfo)
+
+ res_hashes = list(map(lambda minfo: minfo["res_hash"], minfos))
+
+ # To avoid asking per-resource, ask for all resources in one call.
+ # If at least one resource is out of sync,
+ # try to sync them individually.
+ if self.rpc_call("has_resources", res_hashes):
+ return
+ for minfo in minfos:
+ self.sync_resource(minfo)
# def enable_nm(self):
# return self._rpc_call("enable_nm")
diff --git a/lnst/Slave/NetTestSlave.py b/lnst/Slave/NetTestSlave.py
index 908e85c..5623c70 100644
--- a/lnst/Slave/NetTestSlave.py
+++ b/lnst/Slave/NetTestSlave.py
@@ -156,6 +156,10 @@ class SlaveMethods:
module = imp.load_source(module_name, module_path)
self._dynamic_modules[module_name] = module
+ def load_cached_modules(self, minfos):
+ for minfo in minfos:
+ self.load_cached_module(minfo["name"],
minfo["res_hash"])
+
def init_if_manager(self):
self._if_manager = InterfaceManager(self._server_handler)
for cls_name in dir(Devices):
@@ -407,6 +411,12 @@ class SlaveMethods:
return False
+ def has_resources(self, res_hashes):
+ for res_hash in res_hashes:
+ if not self.has_resource(res_hash):
+ return False
+ return True
+
def add_resource_to_cache(self, res_type, local_path, name):
if res_type == "file":
self._cache.add_file_entry(local_path, name)
--
2.9.5