From c3bafd129ec29601280dcab4a7bd9008ce35dd15 2010-09-12 09:50:34 From: Fernando Perez Date: 2010-09-12 09:50:34 Subject: [PATCH] Fix bugs in x=!cmd; we can't use pexpect at all. pexpect makes the subprocesses format their output for a terminal, with a mix of spaces, tabs and newlines. This makes it virtually impossible to then capture their output and do anything useful with it. Fixed a few other small bugs and inconsistencies in process handling. --- diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py index f688362..ad1fc2b 100644 --- a/IPython/core/inputsplitter.py +++ b/IPython/core/inputsplitter.py @@ -731,16 +731,12 @@ _assign_system_re = re.compile(r'(?P(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))' def transform_assign_system(line): """Handle the `files = !ls` syntax.""" - # FIXME: This transforms the line to use %sc, but we've listed that magic - # as deprecated. We should then implement this functionality in a - # standalone api that we can transform to, without going through a - # deprecated magic. m = _assign_system_re.match(line) if m is not None: cmd = m.group('cmd') lhs = m.group('lhs') - expr = make_quoted_expr("sc -l = %s" % cmd) - new_line = '%s = get_ipython().magic(%s)' % (lhs, expr) + expr = make_quoted_expr(cmd) + new_line = '%s = get_ipython().getoutput(%s)' % (lhs, expr) return new_line return line diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 6b8cc59..82eb15b 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -63,7 +63,7 @@ from IPython.utils.path import get_home_dir, get_ipython_dir, HomeDirError from IPython.utils.process import system, getoutput from IPython.utils.strdispatch import StrDispatch from IPython.utils.syspathcontext import prepended_to_syspath -from IPython.utils.text import num_ini_spaces, format_screen +from IPython.utils.text import num_ini_spaces, format_screen, LSString, SList from IPython.utils.traitlets import (Int, Str, CBool, CaselessStrEnum, Enum, List, Unicode, Instance, Type) from IPython.utils.warn import warn, error, fatal @@ -1847,7 +1847,14 @@ class InteractiveShell(Configurable, Magic): #------------------------------------------------------------------------- def system(self, cmd): - """Call the given cmd in a subprocess.""" + """Call the given cmd in a subprocess. + + Parameters + ---------- + cmd : str + Command to execute (can not end in '&', as bacground processes are + not supported. + """ # We do not support backgrounding processes because we either use # pexpect or pipes to read from. Users can always just call # os.system() if they really want a background process. @@ -1856,11 +1863,30 @@ class InteractiveShell(Configurable, Magic): return system(self.var_expand(cmd, depth=2)) - def getoutput(self, cmd): - """Get output (possibly including stderr) from a subprocess.""" + def getoutput(self, cmd, split=True): + """Get output (possibly including stderr) from a subprocess. + + Parameters + ---------- + cmd : str + Command to execute (can not end in '&', as bacground processes are + not supported. + split : bool, optional + + If True, split the output into an IPython SList. Otherwise, an + IPython LSString is returned. These are objects similar to normal + lists and strings, with a few convenience attributes for easier + manipulation of line-based output. You can use '?' on them for + details. + """ if cmd.endswith('&'): raise OSError("Background processes not supported.") - return getoutput(self.var_expand(cmd, depth=2)) + out = getoutput(self.var_expand(cmd, depth=2)) + if split: + out = SList(out.splitlines()) + else: + out = LSString(out) + return out #------------------------------------------------------------------------- # Things related to aliases diff --git a/IPython/core/magic.py b/IPython/core/magic.py index 567057c..2029e26 100644 --- a/IPython/core/magic.py +++ b/IPython/core/magic.py @@ -2919,7 +2919,7 @@ Defaulting color scheme to 'NoColor'""" # If all looks ok, proceed out = self.shell.getoutput(cmd) if opts.has_key('l'): - out = SList(out.split('\n')) + out = SList(out.splitlines()) else: out = LSString(out) if opts.has_key('v'): diff --git a/IPython/core/prefilter.py b/IPython/core/prefilter.py index d56008f..e427091 100755 --- a/IPython/core/prefilter.py +++ b/IPython/core/prefilter.py @@ -486,7 +486,7 @@ class AssignSystemTransformer(PrefilterTransformer): if m is not None: cmd = m.group('cmd') lhs = m.group('lhs') - expr = make_quoted_expr("sc -l =%s" % cmd) + expr = make_quoted_expr("sc =%s" % cmd) new_line = '%s = get_ipython().magic(%s)' % (lhs, expr) return new_line return line diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py index a9cba51..a4848be 100644 --- a/IPython/core/tests/test_inputsplitter.py +++ b/IPython/core/tests/test_inputsplitter.py @@ -395,8 +395,8 @@ def transform_checker(tests, func): syntax = \ dict(assign_system = - [('a =! ls', 'a = get_ipython().magic("sc -l = ls")'), - ('b = !ls', 'b = get_ipython().magic("sc -l = ls")'), + [('a =! ls', 'a = get_ipython().getoutput("ls")'), + ('b = !ls', 'b = get_ipython().getoutput("ls")'), ('x=1', 'x=1'), # normal input is unmodified (' ',' '), # blank lines are kept intact ], diff --git a/IPython/utils/_process_common.py b/IPython/utils/_process_common.py index 4369f34..a4a0fd8 100644 --- a/IPython/utils/_process_common.py +++ b/IPython/utils/_process_common.py @@ -99,6 +99,27 @@ def process_handler(cmd, callback, stderr=subprocess.PIPE): return out +def getoutput(cmd): + """Return standard output of executing cmd in a shell. + + Accepts the same arguments as os.system(). + + Parameters + ---------- + cmd : str + A command to be executed in the system shell. + + Returns + ------- + stdout : str + """ + + out = process_handler(cmd, lambda p: p.communicate()[0], subprocess.STDOUT) + if out is None: + out = '' + return out + + def getoutputerror(cmd): """Return (standard output, standard error) of executing cmd in a shell. diff --git a/IPython/utils/_process_posix.py b/IPython/utils/_process_posix.py index 46a40b4..00fe1a7 100644 --- a/IPython/utils/_process_posix.py +++ b/IPython/utils/_process_posix.py @@ -29,6 +29,7 @@ except ImportError: # Our own from .autoattr import auto_attr +from ._process_common import getoutput #----------------------------------------------------------------------------- # Function definitions @@ -97,6 +98,28 @@ class ProcessHandler(object): except KeyboardInterrupt: print('^C', file=sys.stderr, end='') + def getoutput_pexpect(self, cmd): + """Run a command and return its stdout/stderr as a string. + + Parameters + ---------- + cmd : str + A command to be executed in the system shell. + + Returns + ------- + output : str + A string containing the combination of stdout and stderr from the + subprocess, in whatever order the subprocess originally wrote to its + file descriptors (so the order of the information in this string is the + correct order as would be seen if running the command in a terminal). + """ + pcmd = self._make_cmd(cmd) + try: + return pexpect.run(pcmd).replace('\r\n', '\n') + except KeyboardInterrupt: + print('^C', file=sys.stderr, end='') + def system(self, cmd): """Execute a command in a subshell. @@ -161,9 +184,9 @@ class ProcessHandler(object): return '%s -c "%s"' % (self.sh, cmd) - -# Make objects with a functional interface for outside use -__ph = ProcessHandler() - -system = __ph.system -getoutput = __ph.getoutput +# Make system() with a functional interface for outside use. Note that we use +# getoutput() from the _common utils, which is built on top of popen(). Using +# pexpect to get subprocess output produces difficult to parse output, since +# programs think they are talking to a tty and produce highly formatted output +# (ls is a good example) that makes them hard. +system = ProcessHandler().system