From dc04772b04b7197d3fe8c3f7e63da0b25a6733c1 2018-03-10 12:20:41
From: Thomas Kluyver <thomas@kluyver.me.uk>
Date: 2018-03-10 12:20:41
Subject: [PATCH] Add transformation for system assignments

---

diff --git a/IPython/core/inputtransformer2.py b/IPython/core/inputtransformer2.py
index e7622a0..fc38ea5 100644
--- a/IPython/core/inputtransformer2.py
+++ b/IPython/core/inputtransformer2.py
@@ -137,6 +137,57 @@ class MagicAssign:
         
         return lines_before + [new_line] + lines_after
 
+
+class SystemAssign:
+    @staticmethod
+    def find(tokens_by_line):
+        """Find the first system assignment (a = !foo) in the cell.
+
+        Returns (line, column) of the ! if found, or None.
+        """
+        for line in tokens_by_line:
+            assign_ix = _find_assign_op(line)
+            if (assign_ix is not None) \
+                    and (len(line) >= assign_ix + 2) \
+                    and (line[assign_ix + 1].type == tokenize2.ERRORTOKEN):
+                ix = assign_ix + 1
+
+                while ix < len(line) and line[ix].type == tokenize2.ERRORTOKEN:
+                    if line[ix].string == '!':
+                        return line[ix].start
+                    elif not line[ix].string.isspace():
+                        break
+                    ix += 1
+
+    @staticmethod
+    def transform(lines: List[str], start: Tuple[int, int]):
+        """Transform a system assignment found by find
+        """
+        start_line = start[0] - 1  # Shift from 1-index to 0-index
+        start_col = start[1]
+
+        print("Start at", start_line, start_col)
+        print("Line", lines[start_line])
+
+        lhs, rhs = lines[start_line][:start_col], lines[start_line][
+                                                  start_col:-1]
+        assert rhs.startswith('!'), rhs
+        cmd_parts = [rhs[1:]]
+        end_line = start_line
+        # Follow explicit (backslash) line continuations
+        while end_line < len(lines) and cmd_parts[-1].endswith('\\'):
+            end_line += 1
+            cmd_parts[-1] = cmd_parts[-1][:-1]  # Trim backslash
+            cmd_parts.append(lines[end_line][:-1])  # Trim newline
+        cmd = ' '.join(cmd_parts)
+
+        lines_before = lines[:start_line]
+        call = "get_ipython().getoutput({!r})".format(cmd)
+        new_line = lhs + call + '\n'
+        lines_after = lines[end_line + 1:]
+
+        return lines_before + [new_line] + lines_after
+
 def make_tokens_by_line(lines):
     tokens_by_line = [[]]
     for token in generate_tokens(iter(lines).__next__):
@@ -149,7 +200,8 @@ def make_tokens_by_line(lines):
 class TokenTransformers:
     def __init__(self):
         self.transformers = [
-            MagicAssign
+            MagicAssign,
+            SystemAssign,
         ]
     
     def do_one_transform(self, lines):
diff --git a/IPython/core/tests/test_inputtransformer2.py b/IPython/core/tests/test_inputtransformer2.py
index 5a44d8f..6df2688 100644
--- a/IPython/core/tests/test_inputtransformer2.py
+++ b/IPython/core/tests/test_inputtransformer2.py
@@ -1,6 +1,6 @@
 import nose.tools as nt
 
-from IPython.core import inputtransformer2
+from IPython.core import inputtransformer2 as ipt2
 from IPython.core.inputtransformer2 import make_tokens_by_line
 
 MULTILINE_MAGIC_ASSIGN = ("""\
@@ -21,17 +21,34 @@ b = !foo \\
 g()
 """.splitlines(keepends=True), """\
 a = f()
-b = get_ipython().getoutput('foo   bar')
+b = get_ipython().getoutput('foo    bar')
 g()
 """.splitlines(keepends=True))
 
 def test_find_assign_magic():
     tbl = make_tokens_by_line(MULTILINE_MAGIC_ASSIGN[0])
-    nt.assert_equal(inputtransformer2.MagicAssign.find(tbl), (2, 4))
+    nt.assert_equal(ipt2.MagicAssign.find(tbl), (2, 4))
 
-    tbl = make_tokens_by_line(MULTILINE_SYSTEM_ASSIGN[0]) # Nothing to find
-    nt.assert_equal(inputtransformer2.MagicAssign.find(tbl), None)
+    tbl = make_tokens_by_line(MULTILINE_SYSTEM_ASSIGN[0])  # Nothing to find
+    nt.assert_equal(ipt2.MagicAssign.find(tbl), None)
 
 def test_transform_assign_magic():
-    res = inputtransformer2.MagicAssign.transform(MULTILINE_MAGIC_ASSIGN[0], (2, 4))
+    res = ipt2.MagicAssign.transform(MULTILINE_MAGIC_ASSIGN[0], (2, 4))
     nt.assert_equal(res, MULTILINE_MAGIC_ASSIGN[1])
+
+def test_find_assign_system():
+    tbl = make_tokens_by_line(MULTILINE_SYSTEM_ASSIGN[0])
+    nt.assert_equal(ipt2.SystemAssign.find(tbl), (2, 4))
+
+    tbl = make_tokens_by_line(["a =  !ls\n"])
+    nt.assert_equal(ipt2.SystemAssign.find(tbl), (1, 5))
+
+    tbl = make_tokens_by_line(["a=!ls\n"])
+    nt.assert_equal(ipt2.SystemAssign.find(tbl), (1, 2))
+
+    tbl = make_tokens_by_line(MULTILINE_MAGIC_ASSIGN[0])  # Nothing to find
+    nt.assert_equal(ipt2.SystemAssign.find(tbl), None)
+
+def test_transform_assign_system():
+    res = ipt2.SystemAssign.transform(MULTILINE_SYSTEM_ASSIGN[0], (2, 4))
+    nt.assert_equal(res, MULTILINE_SYSTEM_ASSIGN[1])