[virt-who] libvirt: properly handle libvirt shutdown when doing background monitoring

[Radek Novacek]

commit 45f61bf021cfee2c5d387502cf13fab06d0660a7
Author: Radek Novacek <rnovacek(a)redhat.com&gt;
Date:   Fri Aug 1 16:10:30 2014 +0200

    libvirt: properly handle libvirt shutdown when doing background monitoring

 tests/test_libvirtd.py    |    5 +++--
 virt/libvirtd/libvirtd.py |   38 +++++++++++++++++++++++++++-----------
 2 files changed, 30 insertions(+), 13 deletions(-)
---
diff --git a/tests/test_libvirtd.py b/tests/test_libvirtd.py
index bb3f428..4494524 100644
--- a/tests/test_libvirtd.py
+++ b/tests/test_libvirtd.py
@@ -25,7 +25,8 @@ import logging
 
 from config import Config
 from virt import Virt, Domain, VirtError
-from virt.libvirtd.libvirtd import LibvirtMonitor
+from virt.libvirtd.libvirtd import LibvirtMonitor, VirEventLoopThread
+import virt.libvirtd.libvirtd
 
 
 def raiseLibvirtError(*args, **kwargs):
@@ -54,7 +55,7 @@ class TestLibvirtd(unittest.TestCase):
         self.assertRaises(VirtError, libvirtd.listDomains)
 
     @patch('libvirt.openReadOnly')
-    @patch('threading.Thread')
+    @patch('virt.libvirtd.libvirtd.VirEventLoopThread')
     def test_monitoring(self, thread, virt):
         event = threading.Event()
         LibvirtMonitor().set_event(event)
diff --git a/virt/libvirtd/libvirtd.py b/virt/libvirtd/libvirtd.py
index 2b7a3b3..a46869c 100644
--- a/virt/libvirtd/libvirtd.py
+++ b/virt/libvirtd/libvirtd.py
@@ -26,12 +26,17 @@ import threading
 import virt
 
 
-eventLoopThread = None
+class VirEventLoopThread(threading.Thread):
+    def __init__(self, logger, *args, **kwargs):
+        self._terminated = threading.Event()
+        threading.Thread.__init__(self, *args, **kwargs)
 
+    def run(self):
+        while not self._terminated.is_set():
+            libvirt.virEventRunDefaultImpl()
 
-def virEventLoopNativeRun():
-    while True:
-        libvirt.virEventRunDefaultImpl()
+    def terminate(self):
+        self._terminated.set()
 
 
 class LibvirtMonitor(object):
@@ -45,7 +50,6 @@ class LibvirtMonitor(object):
             cls._instance.logger = logging.getLogger("rhsm-app")
             cls._instance.event = None
             cls._instance.eventLoopThread = None
-            cls._instance.terminate = threading.Event()
             cls._instance.domainIds = []
             cls._instance.definedDomains = []
             cls._instance.vc = None
@@ -61,15 +65,19 @@ class LibvirtMonitor(object):
         libvirt.virEventRegisterDefaultImpl()
         if self.eventLoopThread is not None and self.eventLoopThread.isAlive():
             self.eventLoopThread.terminate()
-        self.eventLoopThread = threading.Thread(target=virEventLoopNativeRun,
name="libvirtEventLoop")
+        self.eventLoopThread = VirEventLoopThread(self.logger,
name="libvirtEventLoop")
         self.eventLoopThread.setDaemon(True)
         self.eventLoopThread.start()
         self._create_connection()
 
     def _create_connection(self):
-        self.vc = libvirt.openReadOnly('')
         try:
-            self.vc.registerCloseCallback(self._close_callback, None, None)
+            self.vc = libvirt.openReadOnly('')
+        except libvirt.libvirtError, e:
+            self.logger.warn("Unable to connect to libvirt: %s" % str(e))
+            return
+        try:
+            self.vc.registerCloseCallback(self._close_callback, None)
         except AttributeError:
             self.logger.warn("Can't monitor libvirtd restarts due to bug in
libvirt-python")
         self.vc.domainEventRegister(self._callback, None)
@@ -86,9 +94,17 @@ class LibvirtMonitor(object):
     def _callback(self, *args, **kwargs):
         self.event.set()
 
-    def _close_callback(self, *args, **kwargs):
-        self.logger.info("Connection to libvirtd lost, reconnecting")
-        self._create_connection()
+    def _close_callback(self, conn, reason, opaque):
+        reasonStrings = ("Error", "End-of-file",
"Keepalive", "Client")
+        self.logger.info("Connection to libvirtd lost: %s, reconnecting" %
reasonStrings[reason])
+        # it might be just a restart, give it some time to recover
+        time.sleep(2)
+        try:
+            self.vc.close()
+        except Exception, e:
+            pass
+        self.vc = None
+        self.event.set()
 
     def set_event(self, event):
         self.event = event