Change in vdsm[master]: command: Store stdout and stderr to a log file

Monday, 27 June 2016

Tomas Golembiovsky has uploaded a new change for review.

Change subject: command: Store stdout and stderr to a log file
......................................................................

command: Store stdout and stderr to a log file

Method execCmd() can be used to execute arbitrary command and return
stdout and stderr either as a string, or provide wrapper streams to the
file descriptors in case of asynchronous execution.

This change adds the possibility to store the stdout and stderr to a log
file independently of the execution method used (synchronous or
asynchronous). Redirecting both stdout and stderr into a single file is
also possible for asynchronous processesing.

Change-Id: I5a8b9afaa196729c8ab98f380a39833a9a0840cd
Signed-off-by: Tomáš Golembiovský <tgolembi(a)redhat.com&gt;
---
M lib/vdsm/commands.py
M tests/commands_test.py
2 files changed, 128 insertions(+), 3 deletions(-)


  git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/33/59833/1

diff --git a/lib/vdsm/commands.py b/lib/vdsm/commands.py
index bffec83..de96097 100644
--- a/lib/vdsm/commands.py
+++ b/lib/vdsm/commands.py
@@ -43,7 +43,8 @@
 def execCmd(command, sudo=False, cwd=None, data=None, raw=False,
             printable=None, env=None, sync=True, nice=None, ioclass=None,
             ioclassdata=None, setsid=False, execCmdLogger=logging.root,
-            deathSignal=0, resetCpuAffinity=True):
+            deathSignal=0, resetCpuAffinity=True, outLog=None, errLog=None,
+            errToOutLog=False):
     """
     Executes an external command, optionally via sudo.
 
@@ -52,6 +53,14 @@
     a temporary thread, spawn a sync=False sub-process, and have the thread
     finish, the new subprocess would die immediately.
     """
+    if errToOutLog and sync:
+        # Using errToOutLog together with sync makes not much sense. Since we
+        # do not get messages as they come, all we can do is write all the
+        # stderr after all the stdout.
+        if outLog is None:
+            outLog = errLog
+        else:
+            errLog = outLog
 
     command = cmdutils.wrap_command(command, with_ioclass=ioclass,
                                     ioclassdata=ioclassdata, with_nice=nice,
@@ -70,7 +79,8 @@
     p = CPopen(command, close_fds=True, cwd=cwd, env=env,
                deathSignal=deathSignal)
     if not sync:
-        p = AsyncProc(p)
+        p = AsyncProc(p, outLog=outLog, errLog=errLog,
+                      errToOutLog=errToOutLog)
         if data is not None:
             p.stdin.write(data)
             p.stdin.flush()
@@ -85,6 +95,24 @@
         out = ""
 
     execCmdLogger.debug(cmdutils.retcode_log_line(p.returncode, err=err))
+
+    # Dump stdout
+    try:
+        if outLog is not None:
+            with open(outLog, 'w') as f:
+                f.write(out)
+    except IOError:
+        execCmdLogger.exception(
+            "Failed to dump stdout to log file '%s'" % outLog)
+
+    # Dump stderr
+    try:
+        if errLog is not None:
+            with open(errLog, 'w') as f:
+                f.write(err)
+    except IOError:
+        execCmdLogger.exception(
+            "Failed to dump stderr to log file '%s'" % errLog)
 
     if not raw:
         out = out.splitlines(False)
@@ -192,10 +220,15 @@
 
             return len(data)
 
-    def __init__(self, popenToWrap):
+    def __init__(self, popenToWrap, outLog=None, errLog=None,
+                 errToOutLog=False):
         # this is an ugly hack to let this module load on Python 3, and fail
         # later when AsyncProc is used.
         from StringIO import StringIO
+
+        if errToOutLog and outLog is None:
+            outLog = errLog
+            errLog = None
 
         self._streamLock = threading.Lock()
         self._proc = popenToWrap
@@ -227,6 +260,32 @@
         self.stdin = io.BufferedWriter(self._streamWrapper(self,
                                        self._stdin, self._fdin), BUFFSIZE)
 
+        if outLog is None:
+            self._outLog = None
+        else:
+            try:
+                self._outLog = open(outLog, 'w')
+            except IOError:
+                logging.exception(
+                    "Failed to open stdout log file '%s'" % outLog)
+                self._outLog = None
+
+        if errToOutLog:
+            self._errLog = self._outLog
+        elif errLog is not None:
+            try:
+                self._errLog = open(errLog, 'w')
+            except IOError:
+                logging.exception(
+                    "Failed to open stderr log file '%s'" % errLog)
+                self._errLog = None
+        else:
+            self._errLog = None
+
+        self._fdMapLog = {fdout: self._outLog,
+                          fderr: self._errLog,
+                          self._fdin: None}
+
         self._returncode = None
 
         self.blocking = False
@@ -249,6 +308,8 @@
 
             for fd, event in pollres:
                 stream = self._fdMap[fd]
+                log = self._fdMapLog[fd]
+
                 if event & select.EPOLLOUT and self._stdin.len > 0:
                     buff = self._stdin.read(BUFFSIZE)
                     written = os.write(fd, buff)
@@ -263,6 +324,8 @@
                     stream.pos = stream.len
                     stream.write(data)
                     stream.pos = oldpos
+                    if log is not None:
+                        log.write(data)
 
                 elif event & (select.EPOLLHUP | select.EPOLLERR):
                     self._poller.unregister(fd)
@@ -308,6 +371,19 @@
             if cond is not None and cond():
                 return False
             self._processStreams()
+
+        # Finish writing logs
+        while (self._outLog is not None and
+               self.stdout.fileno() not in self._closedfds) or \
+              (self._errLog is not None and
+               self.stderr.fileno() not in self._closedfds):
+                self._processStreams()
+
+        if self._outLog is not None:
+            self._outLog.close()
+        if self._errLog is not None:
+            self._errLog.close()
+
         return True
 
     def communicate(self, data=None):
@@ -321,6 +397,10 @@
 
     def __del__(self):
         self._poller.close()
+        if self._outLog is not None:
+            self._outLog.close()
+        if self._errLog is not None:
+            self._errLog.close()
 
 
 def grepCmd(pattern, paths):
diff --git a/tests/commands_test.py b/tests/commands_test.py
index cb7dda2..75e20ae 100644
--- a/tests/commands_test.py
+++ b/tests/commands_test.py
@@ -21,6 +21,7 @@
 import os.path
 import six
 import sys
+import tempfile
 import threading
 import time
 
@@ -72,6 +73,50 @@
         self.assertEquals(rc, 0)
         self.assertEquals(int(out[0].split()[2]), 0)
 
+    @permutations(CMD_TYPES)
+    def testLog(self, cmd):
+        fdOut, outLog = tempfile.mkstemp()
+        fdErr, errLog = tempfile.mkstemp()
+
+        command = 'echo Hello >&1; echo World >&2'
+
+        def checklog():
+            with open(outLog, 'r') as f:
+                content = f.read()
+                self.assertEquals(content, 'Hello\n')
+
+            with open(errLog, 'r') as f:
+                content = f.read()
+                self.assertEquals(content, 'World\n')
+
+            os.ftruncate(fdOut, 0)
+            os.ftruncate(fdErr, 0)
+
+        # Sync
+        rc, _, _ = commands.execCmd(cmd(('bash', '-c', command)),
+                                    outLog=outLog,
+                                    errLog=errLog)
+        self.assertEquals(rc, 0)
+        checklog()
+
+        # Async: wait
+        p = commands.execCmd(cmd(('bash', '-c', command)),
+                             sync=False,
+                             outLog=outLog,
+                             errLog=errLog)
+        p.wait()
+        self.assertEquals(p.returncode, 0)
+        checklog()
+
+        # Async: communicate
+        p = commands.execCmd(cmd(('bash', '-c', command)),
+                             sync=False,
+                             outLog=outLog,
+                             errLog=errLog)
+        p.communicate()
+        self.assertEquals(p.returncode, 0)
+        checklog()
+
 
 class ExecCmdStressTest(TestCaseBase):
 


-- 
To view, visit https://gerrit.ovirt.org/59833
To unsubscribe, visit https://gerrit.ovirt.org/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I5a8b9afaa196729c8ab98f380a39833a9a0840cd
Gerrit-PatchSet: 1
Gerrit-Project: vdsm
Gerrit-Branch: master
Gerrit-Owner: Tomas Golembiovsky <tgolembi(a)redhat.com&gt;
    

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011