Fix bugs in x=!cmd; we can't use pexpect at all.

pexpect makes the subprocesses format their output for a terminal,
with a mix of spaces, tabs and newlines.  This makes it virtually
impossible to then capture their output and do anything useful with it.

Fixed a few other small bugs and inconsistencies in process handling.

IPython/core/inputsplitter.py

@@ -731,16 +731,12 b" _assign_system_re = re.compile(r'(?P<lhs>(\\s*)([\\w\\.]+)((\\s*,\\s*[\\w\\.]+)*))'"
 
 def transform_assign_system(line):
     """Handle the `files = !ls` syntax."""
-    # FIXME: This transforms the line to use %sc, but we've listed that magic
-    # as deprecated.  We should then implement this functionality in a
-    # standalone api that we can transform to, without going through a
-    # deprecated magic.
     m = _assign_system_re.match(line)
     if m is not None:
         cmd = m.group('cmd')
         lhs = m.group('lhs')
-        expr = make_quoted_expr("sc -l = %s" % cmd)
-        new_line = '%s = get_ipython().magic(%s)' % (lhs, expr)
+        expr = make_quoted_expr(cmd)
+        new_line = '%s = get_ipython().getoutput(%s)' % (lhs, expr)
         return new_line
     return line
 

IPython/core/interactiveshell.py

@@ -63,7 +63,7 b' from IPython.utils.path import get_home_dir, get_ipython_dir, HomeDirError'
 from IPython.utils.process import system, getoutput
 from IPython.utils.strdispatch import StrDispatch
 from IPython.utils.syspathcontext import prepended_to_syspath
-from IPython.utils.text import num_ini_spaces, format_screen
+from IPython.utils.text import num_ini_spaces, format_screen, LSString, SList
 from IPython.utils.traitlets import (Int, Str, CBool, CaselessStrEnum, Enum,
                                      List, Unicode, Instance, Type)
 from IPython.utils.warn import warn, error, fatal
@@ -1847,7 +1847,14 b' class InteractiveShell(Configurable, Magic):'
     #-------------------------------------------------------------------------
 
     def system(self, cmd):
-        """Call the given cmd in a subprocess."""
+        """Call the given cmd in a subprocess.
+
+        Parameters
+        ----------
+        cmd : str
+          Command to execute (can not end in '&', as bacground processes are
+          not supported.
+        """
         # We do not support backgrounding processes because we either use
         # pexpect or pipes to read from.  Users can always just call
         # os.system() if they really want a background process.
@@ -1856,11 +1863,30 b' class InteractiveShell(Configurable, Magic):'
 
         return system(self.var_expand(cmd, depth=2))
 
-    def getoutput(self, cmd):
-        """Get output (possibly including stderr) from a subprocess."""
+    def getoutput(self, cmd, split=True):
+        """Get output (possibly including stderr) from a subprocess.
+
+        Parameters
+        ----------
+        cmd : str
+          Command to execute (can not end in '&', as bacground processes are
+          not supported.
+        split : bool, optional
+        
+          If True, split the output into an IPython SList.  Otherwise, an
+          IPython LSString is returned.  These are objects similar to normal
+          lists and strings, with a few convenience attributes for easier
+          manipulation of line-based output.  You can use '?' on them for
+          details.
+          """
         if cmd.endswith('&'):
             raise OSError("Background processes not supported.")
-        return getoutput(self.var_expand(cmd, depth=2))
+        out = getoutput(self.var_expand(cmd, depth=2))
+        if split:
+            out = SList(out.splitlines())
+        else:
+            out = LSString(out)
+        return out
 
     #-------------------------------------------------------------------------
     # Things related to aliases

IPython/core/magic.py

@@ -2919,7 +2919,7 b' Defaulting color scheme to \'NoColor\'"""'
         # If all looks ok, proceed
         out = self.shell.getoutput(cmd)
         if opts.has_key('l'):
-            out = SList(out.split('\n'))
+            out = SList(out.splitlines())
         else:
             out = LSString(out)
         if opts.has_key('v'):

IPython/core/prefilter.py

@@ -486,7 +486,7 b' class AssignSystemTransformer(PrefilterTransformer):'
         if m is not None:
             cmd = m.group('cmd')
             lhs = m.group('lhs')
-            expr = make_quoted_expr("sc -l =%s" % cmd)
+            expr = make_quoted_expr("sc =%s" % cmd)
             new_line = '%s = get_ipython().magic(%s)' % (lhs, expr)
             return new_line
         return line

IPython/core/tests/test_inputsplitter.py

@@ -395,8 +395,8 b' def transform_checker(tests, func):'
 
 syntax = \
   dict(assign_system =
-       [('a =! ls', 'a = get_ipython().magic("sc -l = ls")'),
-        ('b = !ls', 'b = get_ipython().magic("sc -l = ls")'),
+       [('a =! ls', 'a = get_ipython().getoutput("ls")'),
+        ('b = !ls', 'b = get_ipython().getoutput("ls")'),
         ('x=1', 'x=1'), # normal input is unmodified
         ('    ','    '),  # blank lines are kept intact
         ],

IPython/utils/_process_common.py

@@ -99,6 +99,27 b' def process_handler(cmd, callback, stderr=subprocess.PIPE):'
     return out
 
 
+def getoutput(cmd):
+    """Return standard output of executing cmd in a shell.
+
+    Accepts the same arguments as os.system().
+
+    Parameters
+    ----------
+    cmd : str
+      A command to be executed in the system shell.
+
+    Returns
+    -------
+    stdout : str
+    """
+
+    out = process_handler(cmd, lambda p: p.communicate()[0], subprocess.STDOUT)
+    if out is None:
+        out = ''
+    return out
+
+
 def getoutputerror(cmd):
     """Return (standard output, standard error) of executing cmd in a shell.
 

IPython/utils/_process_posix.py

@@ -29,6 +29,7 b' except ImportError:'
 
 # Our own
 from .autoattr import auto_attr
+from ._process_common import getoutput
 
 #-----------------------------------------------------------------------------
 # Function definitions
@@ -97,6 +98,28 b' class ProcessHandler(object):'
         except KeyboardInterrupt:
             print('^C', file=sys.stderr, end='')
 
+    def getoutput_pexpect(self, cmd):
+        """Run a command and return its stdout/stderr as a string.
+
+        Parameters
+        ----------
+        cmd : str
+          A command to be executed in the system shell.
+
+        Returns
+        -------
+        output : str
+          A string containing the combination of stdout and stderr from the
+        subprocess, in whatever order the subprocess originally wrote to its
+        file descriptors (so the order of the information in this string is the
+        correct order as would be seen if running the command in a terminal).
+        """
+        pcmd = self._make_cmd(cmd)
+        try:
+            return pexpect.run(pcmd).replace('\r\n', '\n')
+        except KeyboardInterrupt:
+            print('^C', file=sys.stderr, end='')
+
     def system(self, cmd):
         """Execute a command in a subshell.
 
@@ -161,9 +184,9 b' class ProcessHandler(object):'
         return '%s -c "%s"' % (self.sh, cmd)
 
 
-
-# Make objects with a functional interface for outside use
-__ph = ProcessHandler()
-
-system = __ph.system
-getoutput = __ph.getoutput
+# Make system() with a functional interface for outside use.  Note that we use
+# getoutput() from the _common utils, which is built on top of popen(). Using
+# pexpect to get subprocess output produces difficult to parse output, since
+# programs think they are talking to a tty and produce highly formatted output
+# (ls is a good example) that makes them hard.
+system = ProcessHandler().system