Gitweb:
https://sourceware.org/git/?p=lvm2.git;a=commitdiff;h=32c87d56b12e9b6b8e6...
Commit: 32c87d56b12e9b6b8e6e1e7e85178abacf19811a
Parent: 60e3dbd6d50ce2c28206c96f824a5afc9bb287e6
Author: Tony Asleson <tasleson(a)redhat.com>
AuthorDate: Mon Sep 25 15:20:03 2017 -0500
Committer: Tony Asleson <tasleson(a)redhat.com>
CommitterDate: Wed Sep 27 07:45:00 2017 -0500
lvmdbusd: thread stacks dump support
If you send a SIGUSR1 (10) to the daemon it will dump all the
threads current stacks to stdout. This will be useful when the
daemon is apparently hung and not processing requests.
eg.
$ sudo kill -10 <daemon pid>
---
daemons/lvmdbusd/main.py | 29 ++++++++++++++++++++------
daemons/lvmdbusd/utils.py | 48 +++++++++++++++++++++++++++++++++++++++-----
2 files changed, 64 insertions(+), 13 deletions(-)
diff --git a/daemons/lvmdbusd/main.py b/daemons/lvmdbusd/main.py
index dc31b6f..7f0a028 100644
--- a/daemons/lvmdbusd/main.py
+++ b/daemons/lvmdbusd/main.py
@@ -63,6 +63,24 @@ def check_bb_size(value):
return v
+def install_signal_handlers():
+ # Because of the glib main loop stuff the python signal handler code is
+ # apparently not usable and we need to use the glib calls instead
+ signal_add = None
+
+ if hasattr(GLib, 'unix_signal_add'):
+ signal_add = GLib.unix_signal_add
+ elif hasattr(GLib, 'unix_signal_add_full'):
+ signal_add = GLib.unix_signal_add_full
+
+ if signal_add:
+ signal_add(GLib.PRIORITY_HIGH, signal.SIGHUP, utils.handler, signal.SIGHUP)
+ signal_add(GLib.PRIORITY_HIGH, signal.SIGINT, utils.handler, signal.SIGINT)
+ signal_add(GLib.PRIORITY_HIGH, signal.SIGUSR1, utils.handler, signal.SIGUSR1)
+ else:
+ log_error("GLib.unix_signal_[add|add_full] are NOT available!")
+
+
def main():
start = time.time()
# Add simple command line handling
@@ -112,12 +130,7 @@ def main():
# List of threads that we start up
thread_list = []
- # Install signal handlers
- for s in [signal.SIGHUP, signal.SIGINT]:
- try:
- signal.signal(s, utils.handler)
- except RuntimeError:
- pass
+ install_signal_handlers()
dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
dbus.mainloop.glib.threads_init()
@@ -177,5 +190,7 @@ def main():
for thread in thread_list:
thread.join()
except KeyboardInterrupt:
- utils.handler(signal.SIGINT, None)
+ # If we are unable to register signal handler, we will end up here when
+ # the service gets a ^C or a kill -2 <parent pid>
+ utils.handler(signal.SIGINT)
return 0
diff --git a/daemons/lvmdbusd/utils.py b/daemons/lvmdbusd/utils.py
index ce2ed22..3c006c4 100644
--- a/daemons/lvmdbusd/utils.py
+++ b/daemons/lvmdbusd/utils.py
@@ -21,6 +21,7 @@ from lvmdbusd import cfg
from gi.repository import GLib
import threading
import traceback
+import signal
STDOUT_TTY = os.isatty(sys.stdout.fileno())
@@ -281,12 +282,47 @@ def log_error(msg, *attributes):
_common_log(msg, *attributes)
+def dump_threads_stackframe():
+ ident_to_name = {}
+
+ for thread_object in threading.enumerate():
+ ident_to_name[thread_object.ident] = thread_object
+
+ stacks = []
+ for thread_ident, frame in sys._current_frames().items():
+ stack = traceback.format_list(traceback.extract_stack(frame))
+
+ # There is a possibility that a thread gets created after we have
+ # enumerated all threads, so this lookup table may be incomplete, so
+ # account for this
+ if thread_ident in ident_to_name:
+ thread_name = ident_to_name[thread_ident].name
+ else:
+ thread_name = "unknown"
+
+ stacks.append("Thread: %s" % (thread_name))
+ stacks.append("".join(stack))
+
+ log_error("Dumping thread stack frames!\n" + "\n".join(stacks))
+
+
# noinspection PyUnusedLocal
-def handler(signum, frame):
- cfg.run.value = 0
- log_debug('Signal handler called with signal %d' % signum)
- if cfg.loop is not None:
- cfg.loop.quit()
+def handler(signum):
+ try:
+ if signum == signal.SIGUSR1:
+ dump_threads_stackframe()
+ else:
+ cfg.run.value = 0
+ log_debug('Exiting daemon with signal %d' % signum)
+ if cfg.loop is not None:
+ cfg.loop.quit()
+ except:
+ st = traceback.format_exc()
+ log_error("signal handler: exception (logged, not reported!) \n %s" % st)
+
+ # It's important we report that we handled the exception for the exception
+ # handler to continue to work, especially for signal 10 (SIGUSR1)
+ return True
def pv_obj_path_generate():
@@ -535,7 +571,7 @@ def add_no_notify(cmdline):
def _async_handler(call_back, parameters):
- params_str = ", ".join([str(x) for x in parameters])
+ params_str = ", ".join(str(x) for x in parameters)
log_debug('Main thread execution, callback = %s, parameters = (%s)' %
(str(call_back), params_str))