[lvm2-commits] master - lvmdbusd: Exit daemon when unable to retrieve state

[Tony Asleson]

Gitweb:       
https://sourceware.org/git/?p=lvm2.git;a=commitdiff;h=ab1f1a306bf8e146c12...
Commit:        ab1f1a306bf8e146c1221786a1c1c24f8b71a377
Parent:        3320ab8334794684b4d324bb78d0b293a27287a6
Author:        Tony Asleson <tasleson(a)redhat.com&gt;
AuthorDate:    Tue Dec 18 09:49:36 2018 -0600
Committer:     Tony Asleson <tasleson(a)redhat.com&gt;
CommitterDate: Thu Dec 20 10:27:30 2018 -0600

lvmdbusd: Exit daemon when unable to retrieve state

In some cases we get stuck where we are unable to retrieve the current
state of lvm as we are encountering an error.  When the error is
persistent we will log and exit the daemon instead of consuming vast
amounts of resources.

Signed-off-by: Tony Asleson <tasleson(a)redhat.com&gt;
---
 daemons/lvmdbusd/cfg.py   |   10 ++++++++++
 daemons/lvmdbusd/fetch.py |   19 ++++++++++++++++++-
 2 files changed, 28 insertions(+), 1 deletions(-)

diff --git a/daemons/lvmdbusd/cfg.py b/daemons/lvmdbusd/cfg.py
index 771909f..be497d0 100644
--- a/daemons/lvmdbusd/cfg.py
+++ b/daemons/lvmdbusd/cfg.py
@@ -87,3 +87,13 @@ blackbox = None
 
 # RequestEntry ctor
 create_request_entry = None
+
+
+def exit_daemon():
+    """
+    Exit the daemon cleanly
+    :return:
+    """
+    if run and loop:
+        run.value = 0
+        loop.quit()
diff --git a/daemons/lvmdbusd/fetch.py b/daemons/lvmdbusd/fetch.py
index 69a4aae..e8f3521 100644
--- a/daemons/lvmdbusd/fetch.py
+++ b/daemons/lvmdbusd/fetch.py
@@ -14,6 +14,7 @@ from . import cfg
 from .utils import MThreadRunner, log_debug, log_error
 import threading
 import queue
+import time
 import traceback
 
 
@@ -82,6 +83,8 @@ class StateUpdate(object):
 
 	@staticmethod
 	def update_thread(obj):
+		exception_count = 0
+
 		queued_requests = []
 		while cfg.run.value != 0:
 			# noinspection PyBroadException
@@ -136,12 +139,26 @@ class StateUpdate(object):
 				# wake up if we get an exception
 				queued_requests = []
 
+				# We retrieved OK, clear exception count
+				exception_count = 0
+
 			except queue.Empty:
 				pass
-			except Exception:
+			except Exception as e:
 				st = traceback.format_exc()
 				log_error("update_thread exception: \n%s" % st)
 				cfg.blackbox.dump()
+				exception_count += 1
+				if exception_count >= 5:
+					for i in queued_requests:
+						i.set_result(e)
+
+					log_error("Too many errors in update_thread, exiting daemon")
+					cfg.exit_daemon()
+
+				else:
+					# Slow things down when encountering errors
+					time.sleep(1)
 
 	def __init__(self):
 		self.lock = threading.RLock()