[patch lnst-next RFC] Machine: reduce initial resource synchronization and load phase

[Jiri Pirko]

From: Jiri Pirko <jiri(a)mellanox.com&gt;

So far, each resource is checked and loaded one by one. That results
in many rpc calls. On my 20ms line, this makes ~4secs for one slave.
Bulk the checks and loads together which allows to be done in ~1sec.

Signed-off-by: Jiri Pirko <jiri(a)mellanox.com&gt;
---
 lnst/Controller/Machine.py | 68 +++++++++++++++++++++++++++++-----------------
 lnst/Slave/NetTestSlave.py | 10 +++++++
 2 files changed, 53 insertions(+), 25 deletions(-)

diff --git a/lnst/Controller/Machine.py b/lnst/Controller/Machine.py
index 38a3646..2290e2f 100644
--- a/lnst/Controller/Machine.py
+++ b/lnst/Controller/Machine.py
@@ -257,18 +257,15 @@ class Machine(object):
         for cls_name, cls in device_classes:
             classes.extend(reversed(self._get_base_classes(cls)))
 
+        minfos = []
         for cls in classes:
             if cls is object:
                 continue
-            module_name = cls.__module__
-            module = sys.modules[module_name]
-            filename = module.__file__
+            minfos.append(self._get_module_info(cls.__module__))
 
-            if filename[-3:] == "pyc":
-                filename = filename[:-1]
+        self.sync_resources(minfos)
 
-            res_hash = self.sync_resource(module_name, filename)
-            self.rpc_call("load_cached_module", module_name, res_hash)
+        self.rpc_call("load_cached_modules", minfos)
 
         for cls_name, cls in device_classes:
             module_name = cls.__module__
@@ -348,15 +345,10 @@ class Machine(object):
             for cls in reversed(classes):
                 if cls is object or cls is BaseTestModule:
                     continue
-                m_name = cls.__module__
-                m = sys.modules[m_name]
-                filename = m.__file__
-                if filename[-3:] == "pyc":
-                    filename = filename[:-1]
-
-                res_hash = self.sync_resource(m_name, filename)
-
-                self.rpc_call("load_cached_module", m_name, res_hash)
+                minfo = self._get_module_info(cls.__module__)
+                self.sync_resource(minfo)
+                self.rpc_call("load_cached_module",
+                              minfo["name"], minfo["res_hash"])
 
         logging.info("Host %s executing job %d: %s" %
                      (self._id, job.id, str(job)))
@@ -518,19 +510,45 @@ class Machine(object):
         local_file.close()
         self.rpc_call("finish_copy_from", remote_path)
 
-    def sync_resource(self, res_name, file_path):
-        digest = sha256sum(file_path)
+    def _get_module_info(self, name):
+        module = sys.modules[name]
+        filepath = module.__file__
+
+        if filepath[-3:] == "pyc":
+            filepath = filepath[:-1]
+
+        return {"name": name, "filepath": filepath}
+
+    def _module_info_res_hash(self, minfo):
+        if not "res_hash" in minfo:
+            minfo["res_hash"] = sha256sum(minfo["filepath"])
 
-        if not self.rpc_call("has_resource", digest):
-            msg = "Transfering %s to machine %s as '%s'" % (file_path,
+    def sync_resource(self, minfo):
+        self._module_info_res_hash(minfo)
+
+        if not self.rpc_call("has_resource", minfo["res_hash"]):
+            msg = "Transfering %s to machine %s as '%s'" %
(minfo["filepath"],
                                                             self.get_id(),
-                                                            res_name)
+                                                            minfo["name"])
             logging.debug(msg)
 
-            remote_path = self.copy_file_to_machine(file_path)
-            self.rpc_call("add_resource_to_cache",
-                           "file", remote_path, res_name)
-        return digest
+            remote_path = self.copy_file_to_machine(minfo["filepath"])
+            self.rpc_call("add_resource_to_cache", "file",
+                          remote_path, minfo["name"])
+
+    def sync_resources(self, minfos):
+        for minfo in minfos:
+            self._module_info_res_hash(minfo)
+
+        res_hashes = list(map(lambda minfo: minfo["res_hash"], minfos))
+
+        # To avoid asking per-resource, ask for all resources in one call.
+        # If at least one resource is out of sync,
+        # try to sync them individually.
+        if self.rpc_call("has_resources", res_hashes):
+            return
+        for minfo in minfos:
+            self.sync_resource(minfo)
 
     # def enable_nm(self):
         # return self._rpc_call("enable_nm")
diff --git a/lnst/Slave/NetTestSlave.py b/lnst/Slave/NetTestSlave.py
index 908e85c..5623c70 100644
--- a/lnst/Slave/NetTestSlave.py
+++ b/lnst/Slave/NetTestSlave.py
@@ -156,6 +156,10 @@ class SlaveMethods:
         module = imp.load_source(module_name, module_path)
         self._dynamic_modules[module_name] = module
 
+    def load_cached_modules(self, minfos):
+        for minfo in minfos:
+            self.load_cached_module(minfo["name"],
minfo["res_hash"])
+
     def init_if_manager(self):
         self._if_manager = InterfaceManager(self._server_handler)
         for cls_name in dir(Devices):
@@ -407,6 +411,12 @@ class SlaveMethods:
 
         return False
 
+    def has_resources(self, res_hashes):
+        for res_hash in res_hashes:
+            if not self.has_resource(res_hash):
+                return False
+        return True
+
     def add_resource_to_cache(self, res_type, local_path, name):
         if res_type == "file":
             self._cache.add_file_entry(local_path, name)
-- 
2.9.5