From c3bafd129ec29601280dcab4a7bd9008ce35dd15 2010-09-12 09:50:34
From: Fernando Perez <Fernando.Perez@berkeley.edu>
Date: 2010-09-12 09:50:34
Subject: [PATCH] Fix bugs in x=!cmd; we can't use pexpect at all.
pexpect makes the subprocesses format their output for a terminal,
with a mix of spaces, tabs and newlines. This makes it virtually
impossible to then capture their output and do anything useful with it.
Fixed a few other small bugs and inconsistencies in process handling.
---
diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py
index f688362..ad1fc2b 100644
--- a/IPython/core/inputsplitter.py
+++ b/IPython/core/inputsplitter.py
@@ -731,16 +731,12 @@ _assign_system_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
def transform_assign_system(line):
"""Handle the `files = !ls` syntax."""
- # FIXME: This transforms the line to use %sc, but we've listed that magic
- # as deprecated. We should then implement this functionality in a
- # standalone api that we can transform to, without going through a
- # deprecated magic.
m = _assign_system_re.match(line)
if m is not None:
cmd = m.group('cmd')
lhs = m.group('lhs')
- expr = make_quoted_expr("sc -l = %s" % cmd)
- new_line = '%s = get_ipython().magic(%s)' % (lhs, expr)
+ expr = make_quoted_expr(cmd)
+ new_line = '%s = get_ipython().getoutput(%s)' % (lhs, expr)
return new_line
return line
diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py
index 6b8cc59..82eb15b 100644
--- a/IPython/core/interactiveshell.py
+++ b/IPython/core/interactiveshell.py
@@ -63,7 +63,7 @@ from IPython.utils.path import get_home_dir, get_ipython_dir, HomeDirError
from IPython.utils.process import system, getoutput
from IPython.utils.strdispatch import StrDispatch
from IPython.utils.syspathcontext import prepended_to_syspath
-from IPython.utils.text import num_ini_spaces, format_screen
+from IPython.utils.text import num_ini_spaces, format_screen, LSString, SList
from IPython.utils.traitlets import (Int, Str, CBool, CaselessStrEnum, Enum,
List, Unicode, Instance, Type)
from IPython.utils.warn import warn, error, fatal
@@ -1847,7 +1847,14 @@ class InteractiveShell(Configurable, Magic):
#-------------------------------------------------------------------------
def system(self, cmd):
- """Call the given cmd in a subprocess."""
+ """Call the given cmd in a subprocess.
+
+ Parameters
+ ----------
+ cmd : str
+ Command to execute (can not end in '&', as bacground processes are
+ not supported.
+ """
# We do not support backgrounding processes because we either use
# pexpect or pipes to read from. Users can always just call
# os.system() if they really want a background process.
@@ -1856,11 +1863,30 @@ class InteractiveShell(Configurable, Magic):
return system(self.var_expand(cmd, depth=2))
- def getoutput(self, cmd):
- """Get output (possibly including stderr) from a subprocess."""
+ def getoutput(self, cmd, split=True):
+ """Get output (possibly including stderr) from a subprocess.
+
+ Parameters
+ ----------
+ cmd : str
+ Command to execute (can not end in '&', as bacground processes are
+ not supported.
+ split : bool, optional
+
+ If True, split the output into an IPython SList. Otherwise, an
+ IPython LSString is returned. These are objects similar to normal
+ lists and strings, with a few convenience attributes for easier
+ manipulation of line-based output. You can use '?' on them for
+ details.
+ """
if cmd.endswith('&'):
raise OSError("Background processes not supported.")
- return getoutput(self.var_expand(cmd, depth=2))
+ out = getoutput(self.var_expand(cmd, depth=2))
+ if split:
+ out = SList(out.splitlines())
+ else:
+ out = LSString(out)
+ return out
#-------------------------------------------------------------------------
# Things related to aliases
diff --git a/IPython/core/magic.py b/IPython/core/magic.py
index 567057c..2029e26 100644
--- a/IPython/core/magic.py
+++ b/IPython/core/magic.py
@@ -2919,7 +2919,7 @@ Defaulting color scheme to 'NoColor'"""
# If all looks ok, proceed
out = self.shell.getoutput(cmd)
if opts.has_key('l'):
- out = SList(out.split('\n'))
+ out = SList(out.splitlines())
else:
out = LSString(out)
if opts.has_key('v'):
diff --git a/IPython/core/prefilter.py b/IPython/core/prefilter.py
index d56008f..e427091 100755
--- a/IPython/core/prefilter.py
+++ b/IPython/core/prefilter.py
@@ -486,7 +486,7 @@ class AssignSystemTransformer(PrefilterTransformer):
if m is not None:
cmd = m.group('cmd')
lhs = m.group('lhs')
- expr = make_quoted_expr("sc -l =%s" % cmd)
+ expr = make_quoted_expr("sc =%s" % cmd)
new_line = '%s = get_ipython().magic(%s)' % (lhs, expr)
return new_line
return line
diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py
index a9cba51..a4848be 100644
--- a/IPython/core/tests/test_inputsplitter.py
+++ b/IPython/core/tests/test_inputsplitter.py
@@ -395,8 +395,8 @@ def transform_checker(tests, func):
syntax = \
dict(assign_system =
- [('a =! ls', 'a = get_ipython().magic("sc -l = ls")'),
- ('b = !ls', 'b = get_ipython().magic("sc -l = ls")'),
+ [('a =! ls', 'a = get_ipython().getoutput("ls")'),
+ ('b = !ls', 'b = get_ipython().getoutput("ls")'),
('x=1', 'x=1'), # normal input is unmodified
(' ',' '), # blank lines are kept intact
],
diff --git a/IPython/utils/_process_common.py b/IPython/utils/_process_common.py
index 4369f34..a4a0fd8 100644
--- a/IPython/utils/_process_common.py
+++ b/IPython/utils/_process_common.py
@@ -99,6 +99,27 @@ def process_handler(cmd, callback, stderr=subprocess.PIPE):
return out
+def getoutput(cmd):
+ """Return standard output of executing cmd in a shell.
+
+ Accepts the same arguments as os.system().
+
+ Parameters
+ ----------
+ cmd : str
+ A command to be executed in the system shell.
+
+ Returns
+ -------
+ stdout : str
+ """
+
+ out = process_handler(cmd, lambda p: p.communicate()[0], subprocess.STDOUT)
+ if out is None:
+ out = ''
+ return out
+
+
def getoutputerror(cmd):
"""Return (standard output, standard error) of executing cmd in a shell.
diff --git a/IPython/utils/_process_posix.py b/IPython/utils/_process_posix.py
index 46a40b4..00fe1a7 100644
--- a/IPython/utils/_process_posix.py
+++ b/IPython/utils/_process_posix.py
@@ -29,6 +29,7 @@ except ImportError:
# Our own
from .autoattr import auto_attr
+from ._process_common import getoutput
#-----------------------------------------------------------------------------
# Function definitions
@@ -97,6 +98,28 @@ class ProcessHandler(object):
except KeyboardInterrupt:
print('^C', file=sys.stderr, end='')
+ def getoutput_pexpect(self, cmd):
+ """Run a command and return its stdout/stderr as a string.
+
+ Parameters
+ ----------
+ cmd : str
+ A command to be executed in the system shell.
+
+ Returns
+ -------
+ output : str
+ A string containing the combination of stdout and stderr from the
+ subprocess, in whatever order the subprocess originally wrote to its
+ file descriptors (so the order of the information in this string is the
+ correct order as would be seen if running the command in a terminal).
+ """
+ pcmd = self._make_cmd(cmd)
+ try:
+ return pexpect.run(pcmd).replace('\r\n', '\n')
+ except KeyboardInterrupt:
+ print('^C', file=sys.stderr, end='')
+
def system(self, cmd):
"""Execute a command in a subshell.
@@ -161,9 +184,9 @@ class ProcessHandler(object):
return '%s -c "%s"' % (self.sh, cmd)
-
-# Make objects with a functional interface for outside use
-__ph = ProcessHandler()
-
-system = __ph.system
-getoutput = __ph.getoutput
+# Make system() with a functional interface for outside use. Note that we use
+# getoutput() from the _common utils, which is built on top of popen(). Using
+# pexpect to get subprocess output produces difficult to parse output, since
+# programs think they are talking to a tty and produce highly formatted output
+# (ls is a good example) that makes them hard.
+system = ProcessHandler().system