Change in vdsm[master]: sampling: introduce expensive checks - vdsm-patches - Fedora mailing-lists

29 Apr 2015

Francesco Romani has uploaded a new change for review.
Change subject: sampling: introduce expensive checks
......................................................................
sampling: introduce expensive checks
To improve troubleshooting, introduce more
expensive sanity checks, with a default-off
tunable to enable them.
These sanity checks will make no attempt to recover,
will only add logs to report possible issues.
Change-Id: I7b3bb707dd60de194eedfc2e3de1efbf05574ff7
Signed-off-by: Francesco Romani fromani@redhat.com
---
M lib/vdsm/config.py.in
M vdsm/virt/sampling.py
2 files changed, 19 insertions(+), 2 deletions(-)
git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/91/40391/1

diff --git a/lib/vdsm/config.py.in b/lib/vdsm/config.py.in
index e213839..b60e836 100644
--- a/lib/vdsm/config.py.in
+++ b/lib/vdsm/config.py.in
@@ -325,6 +325,10 @@
         ('periodic_task_per_worker', '100',
             'Max number of tasks which can be queued on workers.'
             ' This is for internal usage and may change without warning'),
+
+        ('expensive_checks', 'false',
+            'Perform additional sanity checks and does additional debug logs'
+            'which are expensive performance-wise'),
     ]),
# Section: [devel]
diff --git a/vdsm/virt/sampling.py b/vdsm/virt/sampling.py
index b0628a4..3c1510f 100644
--- a/vdsm/virt/sampling.py
+++ b/vdsm/virt/sampling.py
@@ -504,6 +504,7 @@
         self._skip_doms = ExpiringCache(timeout)
         self._sampling = Stage()
         self._log = logging.getLogger("sampling.VMBulkSampler")
+        self._extra_check = config.getboolean('sampling', 'expensive_checks')
def __call__(self):
         timestamp = self._vm_stats_cache.clock()
@@ -517,7 +518,6 @@
                 # If everything's ok, we can skip all the costly checks.
                 bulk_stats = self._conn.getAllDomainStats(
                     self._stats_flags)
-                self._vm_stats_cache.put(_translate(bulk_stats), timestamp)
             else:
                 # A previous call got stuck, or not every domain
                 # has properly recovered. Thus we must whitelist domains.
@@ -526,7 +526,12 @@
                 if doms:
                     bulk_stats = self._conn.domainListGetStats(
                         doms, self._stats_flags)
-                    self._vm_stats_cache.put(_translate(bulk_stats), timestamp)
+                else:
+                    bulk_stats = []
+
+            stats = _translate(bulk_stats)
+            self._vm_stats_cache.put(stats, timestamp)
+            self._log_missing_vms(self._get_vms(), stats, timestamp)
def _get_responsive_doms(self):
         vms = self._get_vms()
@@ -541,6 +546,14 @@
                 doms.append(vm_obj._dom._dom)
         return doms
+    def _log_missing_vms(self, expected_vms, retrieved_stats, timestamp):
+        # costly check. add another layer of check before to embark on it.
+        if self._extra_check:
+            for vm_id in expected_vms:
+                if vm_id not in retrieved_stats:
+                    self._log.debug('VM %s not updated in bulk at %f',
+                                    vm_id, timestamp)
+
class HostStatsThread(threading.Thread):
     """
-- 
To view, visit https://gerrit.ovirt.org/40391
To unsubscribe, visit https://gerrit.ovirt.org/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I7b3bb707dd60de194eedfc2e3de1efbf05574ff7
Gerrit-PatchSet: 1
Gerrit-Project: vdsm
Gerrit-Branch: master
Gerrit-Owner: Francesco Romani fromani@redhat.com

    