[PATCH] Use tail process to ensure line buffered output to screen and log file at the same moment... (#506664)

[Martin Sivak]

---
 iutil.py |   84 ++++++++++++++++++++++++++++++++++++++++++++++++++------------
 1 files changed, 68 insertions(+), 16 deletions(-)

diff --git a/iutil.py b/iutil.py
index ba1d0b4..0a4adfa 100644
--- a/iutil.py
+++ b/iutil.py
@@ -33,6 +33,7 @@ from flags import flags
 from constants import *
 from rhpl.translate import _
 import re
+import select

import logging
 log = logging.getLogger("anaconda")
@@ -81,37 +82,88 @@ def execWithRedirect(command, argv, stdin = None, stdout = None,

runningLog = open("/tmp/program.log", "a")
     runningLog.write("Running... %s\n" % ([command] + argv,))
+   
+    #prepare os pipes for feeding tee proceses
+    pstdout, pstdin = os.pipe()
+    perrout, perrin = os.pipe()

env = os.environ.copy()
     env.update({"LC_ALL": "C"})

+    #prepare "empty" tee proceses so we can properly detect errors
+    proc_std = None
+    proc_err = None
+
     try:
-        proc = subprocess.Popen([command] + argv, stdin=stdin,
-                                stdout=subprocess.PIPE,
+        #create output pipes with logging (it needs to be different process to avoid deadlock and EOF detection..)
+        proc_std = subprocess.Popen(["tee", "-a", "/tmp/program.log"], stdin=pstdout,
+                                stdout=stdout,
                                 stderr=subprocess.PIPE,
+                                cwd=root,
+                                env=env, bufsize = 1)
+        
+        proc_err = subprocess.Popen(["tee", "-a", "/tmp/program.log"], stdin=perrout,
+                                stdout=stderr,
+                                stderr=subprocess.PIPE,
+                                cwd=root,
+                                env=env, bufsize = 1)
+
+        #run the command
+        proc = subprocess.Popen([command] + argv, stdin=stdin,
+                                stdout=pstdin,
+                                stderr=perrin,
                                 preexec_fn=chroot, cwd=root,
-                                env=env)
+                                env=env, bufsize = 1)
+        
+        proc.wait()
+
+        #close the output pipes
+        if proc_std:
+            proc_std.terminate()
+            proc_std.wait()
+            proc_std = None
+
+        if proc_err:
+            proc_err.terminate()
+            proc_err.wait()
+            proc_err = None
+
+        ret = proc.returncode

-        while True:
-            (outStr, errStr) = proc.communicate()
-            if outStr:
-                os.write(stdout, outStr)
-                runningLog.write(outStr)
-            if errStr:
-                os.write(stderr, errStr)
-                runningLog.write(errStr)

-            if proc.returncode is not None:
-                ret = proc.returncode
-                break
     except OSError as e:
         errstr = "Error running %s: %s" % (command, e.strerror)
         log.error(errstr)
+
+        #free descriptors
+        os.close(pstdout)
+        os.close(perrout)
+        os.close(pstdin)
+        os.close(perrin)
+        
+        #close the output pipes
+        if proc_std:
+            proc_std.terminate()
+            proc_std.wait()
+            proc_std = None
+
+        if proc_err:
+            proc_err.terminate()
+            proc_err.wait()
+            proc_err = None
+
         runningLog.write(errstr)
         runningLog.close()
         raise RuntimeError, errstr

runningLog.close()
+
+    #free descriptors
+    os.close(pstdout)
+    os.close(perrout)
+    os.close(pstdin)
+    os.close(perrin)
+
     return ret

## Run an external program and capture standard out.

[Martin Sivak]

It is not the prettiest approach, but it works. We definitely need another thread or process to ensure the  output is line buffered and interactive %post and %pre scripts work as expected.

Martin

----- "Martin Sivak"  wrote:
> iutil.py |   84
> ++++++++++++++++++++++++++++++++++++++++++++++++++------------
>  1 files changed, 68 insertions(+), 16 deletions(-)
>
> diff --git a/iutil.py b/iutil.py
> index ba1d0b4..0a4adfa 100644
> --- a/iutil.py
> +++ b/iutil.py
> @@ -33,6 +33,7 @@ from flags import flags
>  from constants import *
>  from rhpl.translate import _
>  import re
> +import select
>
> import logging
>  log = logging.getLogger("anaconda")
> @@ -81,37 +82,88 @@ def execWithRedirect(command, argv, stdin = None,
> stdout = None,
>  runningLog = open("/tmp/program.log", "a")
>  runningLog.write("Running... %s\n" % ([command] + argv,))
>
> #prepare os pipes for feeding tee proceses
>
> pstdout, pstdin = os.pipe()
>
> perrout, perrin = os.pipe()
>
> env = os.environ.copy()
> env.update({"LC_ALL": "C"})
>
> #prepare "empty" tee proceses so we can properly detect errors
>
> proc_std = None
>
> proc_err = None
>
> try:
>    proc = subprocess.Popen([command] + argv, stdin=stdin,
>                            stdout=subprocess.PIPE,
>    #create output pipes with logging (it needs to be different
>
> process to avoid deadlock and EOF detection..)
>    proc_std = subprocess.Popen(["tee", "-a",
>
> "/tmp/program.log"], stdin=pstdout,
>                            stdout=stdout,
>                            stderr=subprocess.PIPE,
>                            cwd=root,
>                            env=env, bufsize = 1)
>    proc_err = subprocess.Popen(["tee", "-a",
>
> "/tmp/program.log"], stdin=perrout,
>                            stdout=stderr,
>                            stderr=subprocess.PIPE,
>                            cwd=root,
>                            env=env, bufsize = 1)
>    #run the command
>    proc = subprocess.Popen([command] + argv, stdin=stdin,
>                            stdout=pstdin,
>                            stderr=perrin,
>                            preexec_fn=chroot, cwd=root,
>                            env=env)
>                            env=env, bufsize = 1)
>    proc.wait()
>    #close the output pipes
>    if proc_std:
>        proc_std.terminate()
>        proc_std.wait()
>        proc_std = None
>    if proc_err:
>        proc_err.terminate()
>        proc_err.wait()
>        proc_err = None
>    ret = proc.returncode
>
>    while True:
>        (outStr, errStr) = proc.communicate()
>        if outStr:
>            os.write(stdout, outStr)
>            runningLog.write(outStr)
>        if errStr:
>            os.write(stderr, errStr)
>            runningLog.write(errStr)
>
>        if proc.returncode is not None:
>            ret = proc.returncode
>            break
>
> except OSError as e:
>        errstr = "Error running %s: %s" % (command, e.strerror)
>        log.error(errstr)
>    #free descriptors
>    os.close(pstdout)
>    os.close(perrout)
>    os.close(pstdin)
>    os.close(perrin)
>    #close the output pipes
>    if proc_std:
>        proc_std.terminate()
>        proc_std.wait()
>        proc_std = None
>    if proc_err:
>        proc_err.terminate()
>        proc_err.wait()
>        proc_err = None
>    runningLog.write(errstr)
>    runningLog.close()
>    raise RuntimeError, errstr
>
> runningLog.close()
>
> #free descriptors
>
> os.close(pstdout)
>
> os.close(perrout)
>
> os.close(pstdin)
>
> os.close(perrin)
>
> return ret
> ## Run an external program and capture standard out.
>
> 1.5.4.3