Transformations for 'help?' syntax

IPython/core/inputtransformer2.py

@@ -74,12 +74,6 b' line_transforms = ['
 
 # -----
 
-def help_end(tokens_by_line):
-    pass
-
-def escaped_command(tokens_by_line):
-    pass
-
 def _find_assign_op(token_line):
     # Find the first assignment in the line ('=' not inside brackets)
     # We don't try to support multiple special assignment (a = b = %foo)
@@ -114,9 +108,23 b' def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):'
     return ' '.join([p[:-2] for p in parts[:-1]]  # Strip backslash+newline
                     + [parts[-1][:-1]])         # Strip newline from last line
 
-class MagicAssign:
-    @staticmethod
-    def find(tokens_by_line):
+class TokenTransformBase:
+    # Lower numbers -> higher priority (for matches in the same location)
+    priority = 10
+
+    def sortby(self):
+        return self.start_line, self.start_col, self.priority
+
+    def __init__(self, start):
+        self.start_line = start[0] - 1   # Shift from 1-index to 0-index
+        self.start_col = start[1]
+
+    def transform(self, lines: List[str]):
+        raise NotImplementedError
+
+class MagicAssign(TokenTransformBase):
+    @classmethod
+    def find(cls, tokens_by_line):
         """Find the first magic assignment (a = %foo) in the cell.
         
         Returns (line, column) of the % if found, or None. *line* is 1-indexed.
@@ -127,15 +135,12 b' class MagicAssign:'
                     and (len(line) >= assign_ix + 2) \
                     and (line[assign_ix+1].string == '%') \
                     and (line[assign_ix+2].type == tokenize2.NAME):
-                return line[assign_ix+1].start
+                return cls(line[assign_ix+1].start)
     
-    @staticmethod
-    def transform(lines: List[str], start: Tuple[int, int]):
+    def transform(self, lines: List[str]):
         """Transform a magic assignment found by find
         """
-        start_line = start[0] - 1   # Shift from 1-index to 0-index
-        start_col  = start[1]
-
+        start_line, start_col = self.start_line, self.start_col
         lhs = lines[start_line][:start_col]
         end_line = find_end_of_continued_line(lines, start_line)
         rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
@@ -150,9 +155,9 b' class MagicAssign:'
         return lines_before + [new_line] + lines_after
 
 
-class SystemAssign:
-    @staticmethod
-    def find(tokens_by_line):
+class SystemAssign(TokenTransformBase):
+    @classmethod
+    def find(cls, tokens_by_line):
         """Find the first system assignment (a = !foo) in the cell.
 
         Returns (line, column) of the ! if found, or None. *line* is 1-indexed.
@@ -166,17 +171,15 b' class SystemAssign:'
 
                 while ix < len(line) and line[ix].type == tokenize2.ERRORTOKEN:
                     if line[ix].string == '!':
-                        return line[ix].start
+                        return cls(line[ix].start)
                     elif not line[ix].string.isspace():
                         break
                     ix += 1
 
-    @staticmethod
-    def transform(lines: List[str], start: Tuple[int, int]):
+    def transform(self, lines: List[str]):
         """Transform a system assignment found by find
         """
-        start_line = start[0] - 1  # Shift from 1-index to 0-index
-        start_col = start[1]
+        start_line, start_col = self.start_line, self.start_col
 
         lhs = lines[start_line][:start_col]
         end_line = find_end_of_continued_line(lines, start_line)
@@ -271,9 +274,9 b" tr = { ESC_SHELL : 'get_ipython().system({!r})'.format,"
        ESC_QUOTE2 : _tr_quote2,
        ESC_PAREN  : _tr_paren }
 
-class EscapedCommand:
-    @staticmethod
-    def find(tokens_by_line):
+class EscapedCommand(TokenTransformBase):
+    @classmethod
+    def find(cls, tokens_by_line):
         """Find the first escaped command (%foo, !foo, etc.) in the cell.
 
         Returns (line, column) of the escape if found, or None. *line* is 1-indexed.
@@ -283,12 +286,10 b' class EscapedCommand:'
             while line[ix].type in {tokenize2.INDENT, tokenize2.DEDENT}:
                 ix += 1
             if line[ix].string in ESCAPE_SINGLES:
-                return line[ix].start
+                return cls(line[ix].start)
 
-    @staticmethod
-    def transform(lines, start):
-        start_line = start[0] - 1  # Shift from 1-index to 0-index
-        start_col = start[1]
+    def transform(self, lines):
+        start_line, start_col = self.start_line, self.start_col
 
         indent = lines[start_line][:start_col]
         end_line = find_end_of_continued_line(lines, start_line)
@@ -306,6 +307,57 b' class EscapedCommand:'
 
         return lines_before + [new_line] + lines_after
 
+_help_end_re = re.compile(r"""(%{0,2}
+                              [a-zA-Z_*][\w*]*        # Variable name
+                              (\.[a-zA-Z_*][\w*]*)*   # .etc.etc
+                              )
+                              (\?\??)$                # ? or ??
+                              """,
+                              re.VERBOSE)
+
+class HelpEnd(TokenTransformBase):
+    # This needs to be higher priority (lower number) than EscapedCommand so
+    # that inspecting magics (%foo?) works.
+    priority = 5
+
+    def __init__(self, start, q_locn):
+        super().__init__(start)
+        self.q_line = q_locn[0] - 1  # Shift from 1-indexed to 0-indexed
+        self.q_col = q_locn[1]
+
+    @classmethod
+    def find(cls, tokens_by_line):
+        for line in tokens_by_line:
+            # Last token is NEWLINE; look at last but one
+            if len(line) > 2 and line[-2].string == '?':
+                # Find the first token that's not INDENT/DEDENT
+                ix = 0
+                while line[ix].type in {tokenize2.INDENT, tokenize2.DEDENT}:
+                    ix += 1
+                return cls(line[ix].start, line[-2].start)
+
+    def transform(self, lines):
+        piece = ''.join(lines[self.start_line:self.q_line+1])
+        indent, content = piece[:self.start_col], piece[self.start_col:]
+        lines_before = lines[:self.start_line]
+        lines_after = lines[self.q_line + 1:]
+
+        m = _help_end_re.search(content)
+        assert m is not None, content
+        target = m.group(1)
+        esc = m.group(3)
+
+        # If we're mid-command, put it back on the next prompt for the user.
+        next_input = None
+        if (not lines_before) and (not lines_after) \
+                and content.strip() != m.group(0):
+            next_input = content.rstrip('?\n')
+
+        call = _make_help_call(target, esc, next_input=next_input)
+        new_line = indent + call + '\n'
+
+        return lines_before + [new_line] + lines_after
+
 def make_tokens_by_line(lines):
     tokens_by_line = [[]]
     for token in generate_tokens(iter(lines).__next__):
@@ -330,6 +382,8 b' class TokenTransformers:'
         self.transformers = [
             MagicAssign,
             SystemAssign,
+            EscapedCommand,
+            HelpEnd,
         ]
     
     def do_one_transform(self, lines):
@@ -348,17 +402,17 b' class TokenTransformers:'
         """
         tokens_by_line = make_tokens_by_line(lines)
         candidates = []
-        for transformer in self.transformers:
-            locn = transformer.find(tokens_by_line)
-            if locn:
-                candidates.append((locn, transformer))
+        for transformer_cls in self.transformers:
+            transformer = transformer_cls.find(tokens_by_line)
+            if transformer:
+                candidates.append(transformer)
         
         if not candidates:
             # Nothing to transform
             return False, lines
         
-        first_locn, transformer = min(candidates)
-        return True, transformer.transform(lines, first_locn)
+        transformer = min(candidates, key=TokenTransformBase.sortby)
+        return True, transformer.transform(lines)
 
     def __call__(self, lines):
         while True:
@@ -366,9 +420,6 b' class TokenTransformers:'
             if not changed:
                 return lines
 
-def assign_from_system(tokens_by_line, lines):
-    pass
-
 
 def transform_cell(cell):
     if not cell.endswith('\n'):

IPython/core/tests/test_inputtransformer2.py

@@ -8,7 +8,7 b' a = f()'
 %foo \\
 bar
 g()
-""".splitlines(keepends=True), """\
+""".splitlines(keepends=True), (2, 0), """\
 a = f()
 get_ipython().run_line_magic('foo', ' bar')
 g()
@@ -17,7 +17,7 b' g()'
 INDENTED_MAGIC = ("""\
 for a in range(5):
     %ls
-""".splitlines(keepends=True), """\
+""".splitlines(keepends=True), (2, 4), """\
 for a in range(5):
     get_ipython().run_line_magic('ls', '')
 """.splitlines(keepends=True))
@@ -27,7 +27,7 b' a = f()'
 b = %foo \\
   bar
 g()
-""".splitlines(keepends=True), """\
+""".splitlines(keepends=True), (2, 4), """\
 a = f()
 b = get_ipython().run_line_magic('foo', '   bar')
 g()
@@ -38,27 +38,78 b' a = f()'
 b = !foo \\
   bar
 g()
-""".splitlines(keepends=True), """\
+""".splitlines(keepends=True), (2, 4), """\
 a = f()
 b = get_ipython().getoutput('foo    bar')
 g()
 """.splitlines(keepends=True))
 
 AUTOCALL_QUOTE = (
-    [",f 1 2 3\n"],
+    [",f 1 2 3\n"], (1, 0),
     ['f("1", "2", "3")\n']
 )
 
 AUTOCALL_QUOTE2 = (
-    [";f 1 2 3\n"],
+    [";f 1 2 3\n"], (1, 0),
     ['f("1 2 3")\n']
 )
 
 AUTOCALL_PAREN = (
-    ["/f 1 2 3\n"],
+    ["/f 1 2 3\n"], (1, 0),
     ['f(1, 2, 3)\n']
 )
 
+SIMPLE_HELP = (
+    ["foo?\n"], (1, 0),
+    ["get_ipython().run_line_magic('pinfo', 'foo')\n"]
+)
+
+DETAILED_HELP = (
+    ["foo??\n"], (1, 0),
+    ["get_ipython().run_line_magic('pinfo2', 'foo')\n"]
+)
+
+MAGIC_HELP = (
+    ["%foo?\n"], (1, 0),
+    ["get_ipython().run_line_magic('pinfo', '%foo')\n"]
+)
+
+HELP_IN_EXPR = (
+    ["a = b + c?\n"], (1, 0),
+    ["get_ipython().set_next_input('a = b + c');"
+     "get_ipython().run_line_magic('pinfo', 'c')\n"]
+)
+
+HELP_CONTINUED_LINE = ("""\
+a = \\
+zip?
+""".splitlines(keepends=True), (1, 0),
+[r"get_ipython().set_next_input('a = \\\nzip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
+)
+
+HELP_MULTILINE = ("""\
+(a,
+b) = zip?
+""".splitlines(keepends=True), (1, 0),
+[r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
+)
+
+def check_find(transformer, case, match=True):
+    sample, expected_start, _  = case
+    tbl = make_tokens_by_line(sample)
+    res = transformer.find(tbl)
+    if match:
+        # start_line is stored 0-indexed, expected values are 1-indexed
+        nt.assert_equal((res.start_line+1, res.start_col), expected_start)
+        return res
+    else:
+        nt.assert_is(res, None)
+
+def check_transform(transformer_cls, case):
+    lines, start, expected = case
+    transformer = transformer_cls(start)
+    nt.assert_equal(transformer.transform(lines), expected)
+
 def test_continued_line():
     lines = MULTILINE_MAGIC_ASSIGN[0]
     nt.assert_equal(ipt2.find_end_of_continued_line(lines, 1), 2)
@@ -66,58 +117,63 b' def test_continued_line():'
     nt.assert_equal(ipt2.assemble_continued_line(lines, (1, 5), 2), "foo    bar")
 
 def test_find_assign_magic():
-    tbl = make_tokens_by_line(MULTILINE_MAGIC_ASSIGN[0])
-    nt.assert_equal(ipt2.MagicAssign.find(tbl), (2, 4))
-
-    tbl = make_tokens_by_line(MULTILINE_SYSTEM_ASSIGN[0])  # Nothing to find
-    nt.assert_equal(ipt2.MagicAssign.find(tbl), None)
+    check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
+    check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False)
 
 def test_transform_assign_magic():
-    res = ipt2.MagicAssign.transform(MULTILINE_MAGIC_ASSIGN[0], (2, 4))
-    nt.assert_equal(res, MULTILINE_MAGIC_ASSIGN[1])
+    check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
 
 def test_find_assign_system():
-    tbl = make_tokens_by_line(MULTILINE_SYSTEM_ASSIGN[0])
-    nt.assert_equal(ipt2.SystemAssign.find(tbl), (2, 4))
+    check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
+    check_find(ipt2.SystemAssign, (["a =  !ls\n"], (1, 5), None))
+    check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None))
+    check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False)
 
-    tbl = make_tokens_by_line(["a =  !ls\n"])
-    nt.assert_equal(ipt2.SystemAssign.find(tbl), (1, 5))
+def test_transform_assign_system():
+    check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
 
-    tbl = make_tokens_by_line(["a=!ls\n"])
-    nt.assert_equal(ipt2.SystemAssign.find(tbl), (1, 2))
+def test_find_magic_escape():
+    check_find(ipt2.EscapedCommand, MULTILINE_MAGIC)
+    check_find(ipt2.EscapedCommand, INDENTED_MAGIC)
+    check_find(ipt2.EscapedCommand, MULTILINE_MAGIC_ASSIGN, match=False)
 
-    tbl = make_tokens_by_line(MULTILINE_MAGIC_ASSIGN[0])  # Nothing to find
-    nt.assert_equal(ipt2.SystemAssign.find(tbl), None)
+def test_transform_magic_escape():
+    check_transform(ipt2.EscapedCommand, MULTILINE_MAGIC)
+    check_transform(ipt2.EscapedCommand, INDENTED_MAGIC)
 
-def test_transform_assign_system():
-    res = ipt2.SystemAssign.transform(MULTILINE_SYSTEM_ASSIGN[0], (2, 4))
-    nt.assert_equal(res, MULTILINE_SYSTEM_ASSIGN[1])
+def test_find_autocalls():
+    for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
+        print("Testing %r" % case[0])
+        check_find(ipt2.EscapedCommand, case)
 
-def test_find_magic_escape():
-    tbl = make_tokens_by_line(MULTILINE_MAGIC[0])
-    nt.assert_equal(ipt2.EscapedCommand.find(tbl), (2, 0))
+def test_transform_autocall():
+    for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
+        print("Testing %r" % case[0])
+        check_transform(ipt2.EscapedCommand, case)
 
-    tbl = make_tokens_by_line(INDENTED_MAGIC[0])
-    nt.assert_equal(ipt2.EscapedCommand.find(tbl), (2, 4))
+def test_find_help():
+    for case in [SIMPLE_HELP, DETAILED_HELP, MAGIC_HELP, HELP_IN_EXPR]:
+        check_find(ipt2.HelpEnd, case)
 
-    tbl = make_tokens_by_line(MULTILINE_MAGIC_ASSIGN[0])  # Shouldn't find a = %foo
-    nt.assert_equal(ipt2.EscapedCommand.find(tbl), None)
+    tf = check_find(ipt2.HelpEnd, HELP_CONTINUED_LINE)
+    nt.assert_equal(tf.q_line, 1)
+    nt.assert_equal(tf.q_col, 3)
 
-def test_transform_magic_escape():
-    res = ipt2.EscapedCommand.transform(MULTILINE_MAGIC[0], (2, 0))
-    nt.assert_equal(res, MULTILINE_MAGIC[1])
+    tf = check_find(ipt2.HelpEnd, HELP_MULTILINE)
+    nt.assert_equal(tf.q_line, 1)
+    nt.assert_equal(tf.q_col, 8)
 
-    res = ipt2.EscapedCommand.transform(INDENTED_MAGIC[0], (2, 4))
-    nt.assert_equal(res, INDENTED_MAGIC[1])
+    # ? in a comment does not trigger help
+    check_find(ipt2.HelpEnd, (["foo # bar?\n"], None, None), match=False)
+    # Nor in a string
+    check_find(ipt2.HelpEnd, (["foo = '''bar?\n"], None, None), match=False)
 
-def test_find_autocalls():
-    for sample, _ in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
-        print("Testing %r" % sample)
-        tbl = make_tokens_by_line(sample)
-        nt.assert_equal(ipt2.EscapedCommand.find(tbl), (1, 0))
+def test_transform_help():
+    tf = ipt2.HelpEnd((1, 0), (1, 9))
+    nt.assert_equal(tf.transform(HELP_IN_EXPR[0]), HELP_IN_EXPR[2])
 
-def test_transform_autocall():
-    for sample, expected in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
-        print("Testing %r" % sample)
-        res = ipt2.EscapedCommand.transform(sample, (1, 0))
-        nt.assert_equal(res, expected)
+    tf = ipt2.HelpEnd((1, 0), (2, 3))
+    nt.assert_equal(tf.transform(HELP_CONTINUED_LINE[0]), HELP_CONTINUED_LINE[2])
+
+    tf = ipt2.HelpEnd((1, 0), (2, 8))
+    nt.assert_equal(tf.transform(HELP_MULTILINE[0]), HELP_MULTILINE[2])