Transformations for 'help?' syntax

IPython/core/inputtransformer2.py

@@ -1,383 +1,434
 import re
 from typing import List, Tuple
 from IPython.utils import tokenize2
 from IPython.utils.tokenutil import generate_tokens
 
 def leading_indent(lines):
     """Remove leading indentation.
     
     If the first line starts with a spaces or tabs, the same whitespace will be
     removed from each following line.
     """
     m = re.match(r'^[ \t]+', lines[0])
     if not m:
         return lines
     space = m.group(0)
     n = len(space)
     return [l[n:] if l.startswith(space) else l
             for l in lines]
 
 class PromptStripper:
     """Remove matching input prompts from a block of input.
     
     Parameters
     ----------
     prompt_re : regular expression
         A regular expression matching any input prompt (including continuation)
     initial_re : regular expression, optional
         A regular expression matching only the initial prompt, but not continuation.
         If no initial expression is given, prompt_re will be used everywhere.
         Used mainly for plain Python prompts, where the continuation prompt
         ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
     
     If initial_re and prompt_re differ,
     only initial_re will be tested against the first line.
     If any prompt is found on the first two lines,
     prompts will be stripped from the rest of the block.
     """
     def __init__(self, prompt_re, initial_re=None):
         self.prompt_re = prompt_re
         self.initial_re = initial_re or prompt_re
 
     def _strip(self, lines):
         return [self.prompt_re.sub('', l, count=1) for l in lines]
 
     def __call__(self, lines):
         if self.initial_re.match(lines[0]) or \
                 (len(lines) > 1 and self.prompt_re.match(lines[1])):
             return self._strip(lines)
         return lines
 
 classic_prompt = PromptStripper(
     prompt_re=re.compile(r'^(>>>|\.\.\.)( |$)'),
     initial_re=re.compile(r'^>>>( |$)')
 )
 
 ipython_prompt = PromptStripper(re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)'))
 
 def cell_magic(lines):
     if not lines[0].startswith('%%'):
         return lines
     if re.match('%%\w+\?', lines[0]):
         # This case will be handled by help_end
         return lines
     magic_name, first_line = lines[0][2:].partition(' ')
     body = '\n'.join(lines[1:])
     return ['get_ipython().run_cell_magic(%r, %r, %r)' % (magic_name, first_line, body)]
 
 line_transforms = [
     leading_indent,
     classic_prompt,
     ipython_prompt,
     cell_magic,
 ]
 
 # -----
 
-def help_end(tokens_by_line):
-    pass
-
-def escaped_command(tokens_by_line):
-    pass
-
 def _find_assign_op(token_line):
     # Find the first assignment in the line ('=' not inside brackets)
     # We don't try to support multiple special assignment (a = b = %foo)
     paren_level = 0
     for i, ti in enumerate(token_line):
         s = ti.string
         if s == '=' and paren_level == 0:
             return i 
         if s in '([{':
             paren_level += 1
         elif s in ')]}':
             paren_level -= 1
 
 def find_end_of_continued_line(lines, start_line: int):
     """Find the last line of a line explicitly extended using backslashes.
 
     Uses 0-indexed line numbers.
     """
     end_line = start_line
     while lines[end_line].endswith('\\\n'):
         end_line += 1
         if end_line >= len(lines):
             break
     return end_line
 
 def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):
     """Assemble pieces of a continued line into a single line.
 
     Uses 0-indexed line numbers. *start* is (lineno, colno).
     """
     parts = [lines[start[0]][start[1]:]] + lines[start[0]+1:end_line+1]
     return ' '.join([p[:-2] for p in parts[:-1]]  # Strip backslash+newline
                     + [parts[-1][:-1]])         # Strip newline from last line
 
-class MagicAssign:
-    @staticmethod
-    def find(tokens_by_line):
+class TokenTransformBase:
+    # Lower numbers -> higher priority (for matches in the same location)
+    priority = 10
+
+    def sortby(self):
+        return self.start_line, self.start_col, self.priority
+
+    def __init__(self, start):
+        self.start_line = start[0] - 1   # Shift from 1-index to 0-index
+        self.start_col = start[1]
+
+    def transform(self, lines: List[str]):
+        raise NotImplementedError
+
+class MagicAssign(TokenTransformBase):
+    @classmethod
+    def find(cls, tokens_by_line):
         """Find the first magic assignment (a = %foo) in the cell.
         
         Returns (line, column) of the % if found, or None. *line* is 1-indexed.
         """
         for line in tokens_by_line:
             assign_ix = _find_assign_op(line)
             if (assign_ix is not None) \
                     and (len(line) >= assign_ix + 2) \
                     and (line[assign_ix+1].string == '%') \
                     and (line[assign_ix+2].type == tokenize2.NAME):
-                return line[assign_ix+1].start
+                return cls(line[assign_ix+1].start)
     
-    @staticmethod
-    def transform(lines: List[str], start: Tuple[int, int]):
+    def transform(self, lines: List[str]):
         """Transform a magic assignment found by find
         """
-        start_line = start[0] - 1   # Shift from 1-index to 0-index
-        start_col  = start[1]
-
+        start_line, start_col = self.start_line, self.start_col
         lhs = lines[start_line][:start_col]
         end_line = find_end_of_continued_line(lines, start_line)
         rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
         assert rhs.startswith('%'), rhs
         magic_name, _, args = rhs[1:].partition(' ')
         
         lines_before = lines[:start_line]
         call = "get_ipython().run_line_magic({!r}, {!r})".format(magic_name, args)
         new_line = lhs + call + '\n'
         lines_after = lines[end_line+1:]
         
         return lines_before + [new_line] + lines_after
 
 
-class SystemAssign:
-    @staticmethod
-    def find(tokens_by_line):
+class SystemAssign(TokenTransformBase):
+    @classmethod
+    def find(cls, tokens_by_line):
         """Find the first system assignment (a = !foo) in the cell.
 
         Returns (line, column) of the ! if found, or None. *line* is 1-indexed.
         """
         for line in tokens_by_line:
             assign_ix = _find_assign_op(line)
             if (assign_ix is not None) \
                     and (len(line) >= assign_ix + 2) \
                     and (line[assign_ix + 1].type == tokenize2.ERRORTOKEN):
                 ix = assign_ix + 1
 
                 while ix < len(line) and line[ix].type == tokenize2.ERRORTOKEN:
                     if line[ix].string == '!':
-                        return line[ix].start
+                        return cls(line[ix].start)
                     elif not line[ix].string.isspace():
                         break
                     ix += 1
 
-    @staticmethod
-    def transform(lines: List[str], start: Tuple[int, int]):
+    def transform(self, lines: List[str]):
         """Transform a system assignment found by find
         """
-        start_line = start[0] - 1  # Shift from 1-index to 0-index
-        start_col = start[1]
+        start_line, start_col = self.start_line, self.start_col
 
         lhs = lines[start_line][:start_col]
         end_line = find_end_of_continued_line(lines, start_line)
         rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
         assert rhs.startswith('!'), rhs
         cmd = rhs[1:]
 
         lines_before = lines[:start_line]
         call = "get_ipython().getoutput({!r})".format(cmd)
         new_line = lhs + call + '\n'
         lines_after = lines[end_line + 1:]
 
         return lines_before + [new_line] + lines_after
 
 # The escape sequences that define the syntax transformations IPython will
 # apply to user input.  These can NOT be just changed here: many regular
 # expressions and other parts of the code may use their hardcoded values, and
 # for all intents and purposes they constitute the 'IPython syntax', so they
 # should be considered fixed.
 
 ESC_SHELL  = '!'     # Send line to underlying system shell
 ESC_SH_CAP = '!!'    # Send line to system shell and capture output
 ESC_HELP   = '?'     # Find information about object
 ESC_HELP2  = '??'    # Find extra-detailed information about object
 ESC_MAGIC  = '%'     # Call magic function
 ESC_MAGIC2 = '%%'    # Call cell-magic function
 ESC_QUOTE  = ','     # Split args on whitespace, quote each as string and call
 ESC_QUOTE2 = ';'     # Quote all args as a single string, call
 ESC_PAREN  = '/'     # Call first argument with rest of line as arguments
 
 ESCAPE_SINGLES = {'!', '?', '%', ',', ';', '/'}
 ESCAPE_DOUBLES = {'!!', '??'}  # %% (cell magic) is handled separately
 
 def _make_help_call(target, esc, next_input=None):
     """Prepares a pinfo(2)/psearch call from a target name and the escape
     (i.e. ? or ??)"""
     method  = 'pinfo2' if esc == '??' \
                 else 'psearch' if '*' in target \
                 else 'pinfo'
     arg = " ".join([method, target])
     #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
     t_magic_name, _, t_magic_arg_s = arg.partition(' ')
     t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
     if next_input is None:
         return 'get_ipython().run_line_magic(%r, %r)' % (t_magic_name, t_magic_arg_s)
     else:
         return 'get_ipython().set_next_input(%r);get_ipython().run_line_magic(%r, %r)' % \
            (next_input, t_magic_name, t_magic_arg_s)
 
 def _tr_help(content):
     "Translate lines escaped with: ?"
     # A naked help line should just fire the intro help screen
     if not content:
         return 'get_ipython().show_usage()'
 
     return _make_help_call(content, '?')
 
 def _tr_help2(content):
     "Translate lines escaped with: ??"
     # A naked help line should just fire the intro help screen
     if not content:
         return 'get_ipython().show_usage()'
 
     return _make_help_call(content, '??')
 
 def _tr_magic(content):
     "Translate lines escaped with: %"
     name, _, args = content.partition(' ')
     return 'get_ipython().run_line_magic(%r, %r)' % (name, args)
 
 def _tr_quote(content):
     "Translate lines escaped with: ,"
     name, _, args = content.partition(' ')
     return '%s("%s")' % (name, '", "'.join(args.split()) )
 
 def _tr_quote2(content):
     "Translate lines escaped with: ;"
     name, _, args = content.partition(' ')
     return '%s("%s")' % (name, args)
 
 def _tr_paren(content):
     "Translate lines escaped with: /"
     name, _, args = content.partition(' ')
     return '%s(%s)' % (name, ", ".join(args.split()))
 
 tr = { ESC_SHELL  : 'get_ipython().system({!r})'.format,
        ESC_SH_CAP : 'get_ipython().getoutput({!r})'.format,
        ESC_HELP   : _tr_help,
        ESC_HELP2  : _tr_help2,
        ESC_MAGIC  : _tr_magic,
        ESC_QUOTE  : _tr_quote,
        ESC_QUOTE2 : _tr_quote2,
        ESC_PAREN  : _tr_paren }
 
-class EscapedCommand:
-    @staticmethod
-    def find(tokens_by_line):
+class EscapedCommand(TokenTransformBase):
+    @classmethod
+    def find(cls, tokens_by_line):
         """Find the first escaped command (%foo, !foo, etc.) in the cell.
 
         Returns (line, column) of the escape if found, or None. *line* is 1-indexed.
         """
         for line in tokens_by_line:
             ix = 0
             while line[ix].type in {tokenize2.INDENT, tokenize2.DEDENT}:
                 ix += 1
             if line[ix].string in ESCAPE_SINGLES:
-                return line[ix].start
+                return cls(line[ix].start)
 
-    @staticmethod
-    def transform(lines, start):
-        start_line = start[0] - 1  # Shift from 1-index to 0-index
-        start_col = start[1]
+    def transform(self, lines):
+        start_line, start_col = self.start_line, self.start_col
 
         indent = lines[start_line][:start_col]
         end_line = find_end_of_continued_line(lines, start_line)
         line = assemble_continued_line(lines, (start_line, start_col), end_line)
 
         if line[:2] in ESCAPE_DOUBLES:
             escape, content = line[:2], line[2:]
         else:
             escape, content = line[:1], line[1:]
         call = tr[escape](content)
 
         lines_before = lines[:start_line]
         new_line = indent + call + '\n'
         lines_after = lines[end_line + 1:]
 
         return lines_before + [new_line] + lines_after
 
+_help_end_re = re.compile(r"""(%{0,2}
+                              [a-zA-Z_*][\w*]*        # Variable name
+                              (\.[a-zA-Z_*][\w*]*)*   # .etc.etc
+                              )
+                              (\?\??)$                # ? or ??
+                              """,
+                              re.VERBOSE)
+
+class HelpEnd(TokenTransformBase):
+    # This needs to be higher priority (lower number) than EscapedCommand so
+    # that inspecting magics (%foo?) works.
+    priority = 5
+
+    def __init__(self, start, q_locn):
+        super().__init__(start)
+        self.q_line = q_locn[0] - 1  # Shift from 1-indexed to 0-indexed
+        self.q_col = q_locn[1]
+
+    @classmethod
+    def find(cls, tokens_by_line):
+        for line in tokens_by_line:
+            # Last token is NEWLINE; look at last but one
+            if len(line) > 2 and line[-2].string == '?':
+                # Find the first token that's not INDENT/DEDENT
+                ix = 0
+                while line[ix].type in {tokenize2.INDENT, tokenize2.DEDENT}:
+                    ix += 1
+                return cls(line[ix].start, line[-2].start)
+
+    def transform(self, lines):
+        piece = ''.join(lines[self.start_line:self.q_line+1])
+        indent, content = piece[:self.start_col], piece[self.start_col:]
+        lines_before = lines[:self.start_line]
+        lines_after = lines[self.q_line + 1:]
+
+        m = _help_end_re.search(content)
+        assert m is not None, content
+        target = m.group(1)
+        esc = m.group(3)
+
+        # If we're mid-command, put it back on the next prompt for the user.
+        next_input = None
+        if (not lines_before) and (not lines_after) \
+                and content.strip() != m.group(0):
+            next_input = content.rstrip('?\n')
+
+        call = _make_help_call(target, esc, next_input=next_input)
+        new_line = indent + call + '\n'
+
+        return lines_before + [new_line] + lines_after
+
 def make_tokens_by_line(lines):
     tokens_by_line = [[]]
     for token in generate_tokens(iter(lines).__next__):
         tokens_by_line[-1].append(token)
         if token.type == tokenize2.NEWLINE:
             tokens_by_line.append([])
     
     return tokens_by_line
 
 def show_linewise_tokens(s: str):
     """For investigation"""
     if not s.endswith('\n'):
         s += '\n'
     lines = s.splitlines(keepends=True)
     for line in make_tokens_by_line(lines):
         print("Line -------")
         for tokinfo in line:
             print(" ", tokinfo)
 
 class TokenTransformers:
     def __init__(self):
         self.transformers = [
             MagicAssign,
             SystemAssign,
+            EscapedCommand,
+            HelpEnd,
         ]
     
     def do_one_transform(self, lines):
         """Find and run the transform earliest in the code.
         
         Returns (changed, lines).
         
         This method is called repeatedly until changed is False, indicating
         that all available transformations are complete.
 
         The tokens following IPython special syntax might not be valid, so
         the transformed code is retokenised every time to identify the next
         piece of special syntax. Hopefully long code cells are mostly valid
         Python, not using lots of IPython special syntax, so this shouldn't be
         a performance issue. 
         """
         tokens_by_line = make_tokens_by_line(lines)
         candidates = []
-        for transformer in self.transformers:
-            locn = transformer.find(tokens_by_line)
-            if locn:
-                candidates.append((locn, transformer))
-        
+        for transformer_cls in self.transformers:
+            transformer = transformer_cls.find(tokens_by_line)
+            if transformer:
+                candidates.append(transformer)
+
         if not candidates:
             # Nothing to transform
             return False, lines
-        
-        first_locn, transformer = min(candidates)
-        return True, transformer.transform(lines, first_locn)
+
+        transformer = min(candidates, key=TokenTransformBase.sortby)
+        return True, transformer.transform(lines)
 
     def __call__(self, lines):
         while True:
             changed, lines = self.do_one_transform(lines)
             if not changed:
                 return lines
 
-def assign_from_system(tokens_by_line, lines):
-    pass
-
 
 def transform_cell(cell):
     if not cell.endswith('\n'):
         cell += '\n'  # Ensure every line has a newline
     lines = cell.splitlines(keepends=True)
     for transform in line_transforms:
         #print(transform, lines)
         lines = transform(lines)
     
     lines = TokenTransformers()(lines)
     for line in lines:
         print('~~', line)

IPython/core/tests/test_inputtransformer2.py

@@ -1,123 +1,179
 import nose.tools as nt
 
 from IPython.core import inputtransformer2 as ipt2
 from IPython.core.inputtransformer2 import make_tokens_by_line
 
 MULTILINE_MAGIC = ("""\
 a = f()
 %foo \\
 bar
 g()
-""".splitlines(keepends=True), """\
+""".splitlines(keepends=True), (2, 0), """\
 a = f()
 get_ipython().run_line_magic('foo', ' bar')
 g()
 """.splitlines(keepends=True))
 
 INDENTED_MAGIC = ("""\
 for a in range(5):
     %ls
-""".splitlines(keepends=True), """\
+""".splitlines(keepends=True), (2, 4), """\
 for a in range(5):
     get_ipython().run_line_magic('ls', '')
 """.splitlines(keepends=True))
 
 MULTILINE_MAGIC_ASSIGN = ("""\
 a = f()
 b = %foo \\
   bar
 g()
-""".splitlines(keepends=True), """\
+""".splitlines(keepends=True), (2, 4), """\
 a = f()
 b = get_ipython().run_line_magic('foo', '   bar')
 g()
 """.splitlines(keepends=True))
 
 MULTILINE_SYSTEM_ASSIGN = ("""\
 a = f()
 b = !foo \\
   bar
 g()
-""".splitlines(keepends=True), """\
+""".splitlines(keepends=True), (2, 4), """\
 a = f()
 b = get_ipython().getoutput('foo    bar')
 g()
 """.splitlines(keepends=True))
 
 AUTOCALL_QUOTE = (
-    [",f 1 2 3\n"],
+    [",f 1 2 3\n"], (1, 0),
     ['f("1", "2", "3")\n']
 )
 
 AUTOCALL_QUOTE2 = (
-    [";f 1 2 3\n"],
+    [";f 1 2 3\n"], (1, 0),
     ['f("1 2 3")\n']
 )
 
 AUTOCALL_PAREN = (
-    ["/f 1 2 3\n"],
+    ["/f 1 2 3\n"], (1, 0),
     ['f(1, 2, 3)\n']
 )
 
+SIMPLE_HELP = (
+    ["foo?\n"], (1, 0),
+    ["get_ipython().run_line_magic('pinfo', 'foo')\n"]
+)
+
+DETAILED_HELP = (
+    ["foo??\n"], (1, 0),
+    ["get_ipython().run_line_magic('pinfo2', 'foo')\n"]
+)
+
+MAGIC_HELP = (
+    ["%foo?\n"], (1, 0),
+    ["get_ipython().run_line_magic('pinfo', '%foo')\n"]
+)
+
+HELP_IN_EXPR = (
+    ["a = b + c?\n"], (1, 0),
+    ["get_ipython().set_next_input('a = b + c');"
+     "get_ipython().run_line_magic('pinfo', 'c')\n"]
+)
+
+HELP_CONTINUED_LINE = ("""\
+a = \\
+zip?
+""".splitlines(keepends=True), (1, 0),
+[r"get_ipython().set_next_input('a = \\\nzip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
+)
+
+HELP_MULTILINE = ("""\
+(a,
+b) = zip?
+""".splitlines(keepends=True), (1, 0),
+[r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
+)
+
+def check_find(transformer, case, match=True):
+    sample, expected_start, _  = case
+    tbl = make_tokens_by_line(sample)
+    res = transformer.find(tbl)
+    if match:
+        # start_line is stored 0-indexed, expected values are 1-indexed
+        nt.assert_equal((res.start_line+1, res.start_col), expected_start)
+        return res
+    else:
+        nt.assert_is(res, None)
+
+def check_transform(transformer_cls, case):
+    lines, start, expected = case
+    transformer = transformer_cls(start)
+    nt.assert_equal(transformer.transform(lines), expected)
+
 def test_continued_line():
     lines = MULTILINE_MAGIC_ASSIGN[0]
     nt.assert_equal(ipt2.find_end_of_continued_line(lines, 1), 2)
 
     nt.assert_equal(ipt2.assemble_continued_line(lines, (1, 5), 2), "foo    bar")
 
 def test_find_assign_magic():
-    tbl = make_tokens_by_line(MULTILINE_MAGIC_ASSIGN[0])
-    nt.assert_equal(ipt2.MagicAssign.find(tbl), (2, 4))
-
-    tbl = make_tokens_by_line(MULTILINE_SYSTEM_ASSIGN[0])  # Nothing to find
-    nt.assert_equal(ipt2.MagicAssign.find(tbl), None)
+    check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
+    check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False)
 
 def test_transform_assign_magic():
-    res = ipt2.MagicAssign.transform(MULTILINE_MAGIC_ASSIGN[0], (2, 4))
-    nt.assert_equal(res, MULTILINE_MAGIC_ASSIGN[1])
+    check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
 
 def test_find_assign_system():
-    tbl = make_tokens_by_line(MULTILINE_SYSTEM_ASSIGN[0])
-    nt.assert_equal(ipt2.SystemAssign.find(tbl), (2, 4))
+    check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
+    check_find(ipt2.SystemAssign, (["a =  !ls\n"], (1, 5), None))
+    check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None))
+    check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False)
 
-    tbl = make_tokens_by_line(["a =  !ls\n"])
-    nt.assert_equal(ipt2.SystemAssign.find(tbl), (1, 5))
+def test_transform_assign_system():
+    check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
 
-    tbl = make_tokens_by_line(["a=!ls\n"])
-    nt.assert_equal(ipt2.SystemAssign.find(tbl), (1, 2))
+def test_find_magic_escape():
+    check_find(ipt2.EscapedCommand, MULTILINE_MAGIC)
+    check_find(ipt2.EscapedCommand, INDENTED_MAGIC)
+    check_find(ipt2.EscapedCommand, MULTILINE_MAGIC_ASSIGN, match=False)
 
-    tbl = make_tokens_by_line(MULTILINE_MAGIC_ASSIGN[0])  # Nothing to find
-    nt.assert_equal(ipt2.SystemAssign.find(tbl), None)
+def test_transform_magic_escape():
+    check_transform(ipt2.EscapedCommand, MULTILINE_MAGIC)
+    check_transform(ipt2.EscapedCommand, INDENTED_MAGIC)
 
-def test_transform_assign_system():
-    res = ipt2.SystemAssign.transform(MULTILINE_SYSTEM_ASSIGN[0], (2, 4))
-    nt.assert_equal(res, MULTILINE_SYSTEM_ASSIGN[1])
+def test_find_autocalls():
+    for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
+        print("Testing %r" % case[0])
+        check_find(ipt2.EscapedCommand, case)
 
-def test_find_magic_escape():
-    tbl = make_tokens_by_line(MULTILINE_MAGIC[0])
-    nt.assert_equal(ipt2.EscapedCommand.find(tbl), (2, 0))
+def test_transform_autocall():
+    for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
+        print("Testing %r" % case[0])
+        check_transform(ipt2.EscapedCommand, case)
 
-    tbl = make_tokens_by_line(INDENTED_MAGIC[0])
-    nt.assert_equal(ipt2.EscapedCommand.find(tbl), (2, 4))
+def test_find_help():
+    for case in [SIMPLE_HELP, DETAILED_HELP, MAGIC_HELP, HELP_IN_EXPR]:
+        check_find(ipt2.HelpEnd, case)
 
-    tbl = make_tokens_by_line(MULTILINE_MAGIC_ASSIGN[0])  # Shouldn't find a = %foo
-    nt.assert_equal(ipt2.EscapedCommand.find(tbl), None)
+    tf = check_find(ipt2.HelpEnd, HELP_CONTINUED_LINE)
+    nt.assert_equal(tf.q_line, 1)
+    nt.assert_equal(tf.q_col, 3)
 
-def test_transform_magic_escape():
-    res = ipt2.EscapedCommand.transform(MULTILINE_MAGIC[0], (2, 0))
-    nt.assert_equal(res, MULTILINE_MAGIC[1])
+    tf = check_find(ipt2.HelpEnd, HELP_MULTILINE)
+    nt.assert_equal(tf.q_line, 1)
+    nt.assert_equal(tf.q_col, 8)
 
-    res = ipt2.EscapedCommand.transform(INDENTED_MAGIC[0], (2, 4))
-    nt.assert_equal(res, INDENTED_MAGIC[1])
+    # ? in a comment does not trigger help
+    check_find(ipt2.HelpEnd, (["foo # bar?\n"], None, None), match=False)
+    # Nor in a string
+    check_find(ipt2.HelpEnd, (["foo = '''bar?\n"], None, None), match=False)
 
-def test_find_autocalls():
-    for sample, _ in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
-        print("Testing %r" % sample)
-        tbl = make_tokens_by_line(sample)
-        nt.assert_equal(ipt2.EscapedCommand.find(tbl), (1, 0))
+def test_transform_help():
+    tf = ipt2.HelpEnd((1, 0), (1, 9))
+    nt.assert_equal(tf.transform(HELP_IN_EXPR[0]), HELP_IN_EXPR[2])
 
-def test_transform_autocall():
-    for sample, expected in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
-        print("Testing %r" % sample)
-        res = ipt2.EscapedCommand.transform(sample, (1, 0))
-        nt.assert_equal(res, expected)
+    tf = ipt2.HelpEnd((1, 0), (2, 3))
+    nt.assert_equal(tf.transform(HELP_CONTINUED_LINE[0]), HELP_CONTINUED_LINE[2])
+
+    tf = ipt2.HelpEnd((1, 0), (2, 8))
+    nt.assert_equal(tf.transform(HELP_MULTILINE[0]), HELP_MULTILINE[2])