Enh: Add launch mode with a pass-through option for stderr

sumatra/commands.py

@@ -248,7 +248,7 @@ def configure(argv):
     parser.add_argument('-c', '--on-changed', help="may be 'store-diff' or 'error': the action to take if the code in the repository or any of the dependencies has changed.", choices=['store-diff', 'error'])
     parser.add_argument('-g', '--labelgenerator', choices=['timestamp', 'uuid'], metavar='OPTION', help="specify which method Sumatra should use to generate labels (options: timestamp, uuid)")
     parser.add_argument('-t', '--timestamp_format', help="the timestamp format given to strftime")
-    parser.add_argument('-L', '--launch_mode', choices=['serial', 'distributed', 'slurm-mpi'], help="how computations should be launched.")
+    parser.add_argument('-L', '--launch_mode', choices=['serial', 'serial-tqdm', 'distributed', 'slurm-mpi'], help="how computations should be launched.")
     parser.add_argument('-o', '--launch_mode_options', help="extra options for the given launch mode, to be given in quotes with a leading space, e.g. ' --foo=3'")
     parser.add_argument('-p', '--plain', dest='plain', action='store_true', help="pass arguments to the 'run' command straight through to the program. Otherwise arguments of the form name=value can be used to overwrite default parameter values.")
     parser.add_argument('--no-plain', dest='plain', action='store_false', help="arguments to the 'run' command of the form name=value will overwrite default parameter values. This is the opposite of the --plain option.")

sumatra/launch.py

@@ -92,7 +92,7 @@ def generate_command(self, paths):
         """Return a string containing the command to be launched."""
         raise NotImplementedError("must be impemented by sub-classes")
 
-    def run(self, executable, main_file, arguments, append_label=None):
+    def run(self, executable, main_file, arguments, append_label=None, catch_stderr=False):
         """
         Run a computation in a shell, with the given executable, script and
         arguments. If `append_label` is provided, it is appended to the
@@ -107,7 +107,7 @@ def run(self, executable, main_file, arguments, append_label=None):
             dependencies in order to avoid opening of Matlab shell two times '''
             result, output = save_dependencies(cmd, main_file)
         else:
-            result, output = tee.system2(cmd, cwd=self.working_directory, stdout=True)  # cwd only relevant for local launch, not for MPI, for example
+            result, output = tee.system2(cmd, cwd=self.working_directory, stdout=True, catch_stderr=catch_stderr)  # cwd only relevant for local launch, not for MPI, for example
         self.stdout_stderr = "".join(output)
         return result
 
@@ -187,6 +187,18 @@ def generate_command(self, executable, main_file, arguments):
         return cmd
     generate_command.__doc__ = LaunchMode.generate_command.__doc__
 
+@component
+class SerialTqdmLaunchMode(SerialLaunchMode):
+    """
+    Enable running with a tqdm progress bar.
+    Effectively this launch mode just disables the capture of stderr, which tqdm uses
+    for its output.
+    """
+    name = "serial-tqdm"
+
+    def run(self, *args, **kwargs):
+        return super().run(*args, catch_stderr=False, **kwargs)
+    run.__doc__ = LaunchMode.run.__doc__
 
 @component
 class DistributedLaunchMode(LaunchMode):

sumatra/tee.py

@@ -54,7 +54,7 @@ def system3(cmd):
     tf.close()
     return result, stdout_stderr
 
-def system2(cmd, cwd=None, logger=_sentinel, stdout=_sentinel, log_command=_sentinel, timing=_sentinel):
+def system2(cmd, cwd=None, logger=_sentinel, stdout=_sentinel, log_command=_sentinel, timing=_sentinel, catch_stderr=True):
         #def tee(cmd, cwd=None, logger=tee_logger, console=tee_console):
         """ This is a simple placement for os.system() or subprocess.Popen()
         that simulates how Unix tee() works - logging stdout/stderr using logging
@@ -130,11 +130,16 @@ def nop(msg):
         if cwd is not None and not os.path.isdir(cwd):
                 os.makedirs(cwd) # this throws exception if fails
 
+        if catch_stderr:
+                stderr = subprocess.STDOUT
+        else:
+                stderr = False
+
         # samarkanov: commented 'quote_command' deliberately
         # reason: if I have 'quote_command' Sumatra does not work in Windows (it encloses the command in quotes. I did not understand why should we quote)
         # I have never catched "The input line is too long" (yet?)
         # cmd = quote_command(cmd)
-        p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=(platform.system() == 'Linux'))
+        p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=subprocess.PIPE, stderr=stderr, close_fds=(platform.system() == 'Linux'))
         if(log_command):
                 mylogger("Running: %s" % cmd)
         try: