From 016de51a986e6f2b1e7b50b3d3b9bca55115d657 2018-03-10 16:19:42 From: Thomas Kluyver Date: 2018-03-10 16:19:42 Subject: [PATCH] Transformations for 'help?' syntax --- diff --git a/IPython/core/inputtransformer2.py b/IPython/core/inputtransformer2.py index 36700c0..29044d6 100644 --- a/IPython/core/inputtransformer2.py +++ b/IPython/core/inputtransformer2.py @@ -74,12 +74,6 @@ line_transforms = [ # ----- -def help_end(tokens_by_line): - pass - -def escaped_command(tokens_by_line): - pass - def _find_assign_op(token_line): # Find the first assignment in the line ('=' not inside brackets) # We don't try to support multiple special assignment (a = b = %foo) @@ -114,9 +108,23 @@ def assemble_continued_line(lines, start: Tuple[int, int], end_line: int): return ' '.join([p[:-2] for p in parts[:-1]] # Strip backslash+newline + [parts[-1][:-1]]) # Strip newline from last line -class MagicAssign: - @staticmethod - def find(tokens_by_line): +class TokenTransformBase: + # Lower numbers -> higher priority (for matches in the same location) + priority = 10 + + def sortby(self): + return self.start_line, self.start_col, self.priority + + def __init__(self, start): + self.start_line = start[0] - 1 # Shift from 1-index to 0-index + self.start_col = start[1] + + def transform(self, lines: List[str]): + raise NotImplementedError + +class MagicAssign(TokenTransformBase): + @classmethod + def find(cls, tokens_by_line): """Find the first magic assignment (a = %foo) in the cell. Returns (line, column) of the % if found, or None. *line* is 1-indexed. @@ -127,15 +135,12 @@ class MagicAssign: and (len(line) >= assign_ix + 2) \ and (line[assign_ix+1].string == '%') \ and (line[assign_ix+2].type == tokenize2.NAME): - return line[assign_ix+1].start + return cls(line[assign_ix+1].start) - @staticmethod - def transform(lines: List[str], start: Tuple[int, int]): + def transform(self, lines: List[str]): """Transform a magic assignment found by find """ - start_line = start[0] - 1 # Shift from 1-index to 0-index - start_col = start[1] - + start_line, start_col = self.start_line, self.start_col lhs = lines[start_line][:start_col] end_line = find_end_of_continued_line(lines, start_line) rhs = assemble_continued_line(lines, (start_line, start_col), end_line) @@ -150,9 +155,9 @@ class MagicAssign: return lines_before + [new_line] + lines_after -class SystemAssign: - @staticmethod - def find(tokens_by_line): +class SystemAssign(TokenTransformBase): + @classmethod + def find(cls, tokens_by_line): """Find the first system assignment (a = !foo) in the cell. Returns (line, column) of the ! if found, or None. *line* is 1-indexed. @@ -166,17 +171,15 @@ class SystemAssign: while ix < len(line) and line[ix].type == tokenize2.ERRORTOKEN: if line[ix].string == '!': - return line[ix].start + return cls(line[ix].start) elif not line[ix].string.isspace(): break ix += 1 - @staticmethod - def transform(lines: List[str], start: Tuple[int, int]): + def transform(self, lines: List[str]): """Transform a system assignment found by find """ - start_line = start[0] - 1 # Shift from 1-index to 0-index - start_col = start[1] + start_line, start_col = self.start_line, self.start_col lhs = lines[start_line][:start_col] end_line = find_end_of_continued_line(lines, start_line) @@ -271,9 +274,9 @@ tr = { ESC_SHELL : 'get_ipython().system({!r})'.format, ESC_QUOTE2 : _tr_quote2, ESC_PAREN : _tr_paren } -class EscapedCommand: - @staticmethod - def find(tokens_by_line): +class EscapedCommand(TokenTransformBase): + @classmethod + def find(cls, tokens_by_line): """Find the first escaped command (%foo, !foo, etc.) in the cell. Returns (line, column) of the escape if found, or None. *line* is 1-indexed. @@ -283,12 +286,10 @@ class EscapedCommand: while line[ix].type in {tokenize2.INDENT, tokenize2.DEDENT}: ix += 1 if line[ix].string in ESCAPE_SINGLES: - return line[ix].start + return cls(line[ix].start) - @staticmethod - def transform(lines, start): - start_line = start[0] - 1 # Shift from 1-index to 0-index - start_col = start[1] + def transform(self, lines): + start_line, start_col = self.start_line, self.start_col indent = lines[start_line][:start_col] end_line = find_end_of_continued_line(lines, start_line) @@ -306,6 +307,57 @@ class EscapedCommand: return lines_before + [new_line] + lines_after +_help_end_re = re.compile(r"""(%{0,2} + [a-zA-Z_*][\w*]* # Variable name + (\.[a-zA-Z_*][\w*]*)* # .etc.etc + ) + (\?\??)$ # ? or ?? + """, + re.VERBOSE) + +class HelpEnd(TokenTransformBase): + # This needs to be higher priority (lower number) than EscapedCommand so + # that inspecting magics (%foo?) works. + priority = 5 + + def __init__(self, start, q_locn): + super().__init__(start) + self.q_line = q_locn[0] - 1 # Shift from 1-indexed to 0-indexed + self.q_col = q_locn[1] + + @classmethod + def find(cls, tokens_by_line): + for line in tokens_by_line: + # Last token is NEWLINE; look at last but one + if len(line) > 2 and line[-2].string == '?': + # Find the first token that's not INDENT/DEDENT + ix = 0 + while line[ix].type in {tokenize2.INDENT, tokenize2.DEDENT}: + ix += 1 + return cls(line[ix].start, line[-2].start) + + def transform(self, lines): + piece = ''.join(lines[self.start_line:self.q_line+1]) + indent, content = piece[:self.start_col], piece[self.start_col:] + lines_before = lines[:self.start_line] + lines_after = lines[self.q_line + 1:] + + m = _help_end_re.search(content) + assert m is not None, content + target = m.group(1) + esc = m.group(3) + + # If we're mid-command, put it back on the next prompt for the user. + next_input = None + if (not lines_before) and (not lines_after) \ + and content.strip() != m.group(0): + next_input = content.rstrip('?\n') + + call = _make_help_call(target, esc, next_input=next_input) + new_line = indent + call + '\n' + + return lines_before + [new_line] + lines_after + def make_tokens_by_line(lines): tokens_by_line = [[]] for token in generate_tokens(iter(lines).__next__): @@ -330,6 +382,8 @@ class TokenTransformers: self.transformers = [ MagicAssign, SystemAssign, + EscapedCommand, + HelpEnd, ] def do_one_transform(self, lines): @@ -348,17 +402,17 @@ class TokenTransformers: """ tokens_by_line = make_tokens_by_line(lines) candidates = [] - for transformer in self.transformers: - locn = transformer.find(tokens_by_line) - if locn: - candidates.append((locn, transformer)) - + for transformer_cls in self.transformers: + transformer = transformer_cls.find(tokens_by_line) + if transformer: + candidates.append(transformer) + if not candidates: # Nothing to transform return False, lines - - first_locn, transformer = min(candidates) - return True, transformer.transform(lines, first_locn) + + transformer = min(candidates, key=TokenTransformBase.sortby) + return True, transformer.transform(lines) def __call__(self, lines): while True: @@ -366,9 +420,6 @@ class TokenTransformers: if not changed: return lines -def assign_from_system(tokens_by_line, lines): - pass - def transform_cell(cell): if not cell.endswith('\n'): diff --git a/IPython/core/tests/test_inputtransformer2.py b/IPython/core/tests/test_inputtransformer2.py index 6067747..cf59da2 100644 --- a/IPython/core/tests/test_inputtransformer2.py +++ b/IPython/core/tests/test_inputtransformer2.py @@ -8,7 +8,7 @@ a = f() %foo \\ bar g() -""".splitlines(keepends=True), """\ +""".splitlines(keepends=True), (2, 0), """\ a = f() get_ipython().run_line_magic('foo', ' bar') g() @@ -17,7 +17,7 @@ g() INDENTED_MAGIC = ("""\ for a in range(5): %ls -""".splitlines(keepends=True), """\ +""".splitlines(keepends=True), (2, 4), """\ for a in range(5): get_ipython().run_line_magic('ls', '') """.splitlines(keepends=True)) @@ -27,7 +27,7 @@ a = f() b = %foo \\ bar g() -""".splitlines(keepends=True), """\ +""".splitlines(keepends=True), (2, 4), """\ a = f() b = get_ipython().run_line_magic('foo', ' bar') g() @@ -38,27 +38,78 @@ a = f() b = !foo \\ bar g() -""".splitlines(keepends=True), """\ +""".splitlines(keepends=True), (2, 4), """\ a = f() b = get_ipython().getoutput('foo bar') g() """.splitlines(keepends=True)) AUTOCALL_QUOTE = ( - [",f 1 2 3\n"], + [",f 1 2 3\n"], (1, 0), ['f("1", "2", "3")\n'] ) AUTOCALL_QUOTE2 = ( - [";f 1 2 3\n"], + [";f 1 2 3\n"], (1, 0), ['f("1 2 3")\n'] ) AUTOCALL_PAREN = ( - ["/f 1 2 3\n"], + ["/f 1 2 3\n"], (1, 0), ['f(1, 2, 3)\n'] ) +SIMPLE_HELP = ( + ["foo?\n"], (1, 0), + ["get_ipython().run_line_magic('pinfo', 'foo')\n"] +) + +DETAILED_HELP = ( + ["foo??\n"], (1, 0), + ["get_ipython().run_line_magic('pinfo2', 'foo')\n"] +) + +MAGIC_HELP = ( + ["%foo?\n"], (1, 0), + ["get_ipython().run_line_magic('pinfo', '%foo')\n"] +) + +HELP_IN_EXPR = ( + ["a = b + c?\n"], (1, 0), + ["get_ipython().set_next_input('a = b + c');" + "get_ipython().run_line_magic('pinfo', 'c')\n"] +) + +HELP_CONTINUED_LINE = ("""\ +a = \\ +zip? +""".splitlines(keepends=True), (1, 0), +[r"get_ipython().set_next_input('a = \\\nzip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"] +) + +HELP_MULTILINE = ("""\ +(a, +b) = zip? +""".splitlines(keepends=True), (1, 0), +[r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"] +) + +def check_find(transformer, case, match=True): + sample, expected_start, _ = case + tbl = make_tokens_by_line(sample) + res = transformer.find(tbl) + if match: + # start_line is stored 0-indexed, expected values are 1-indexed + nt.assert_equal((res.start_line+1, res.start_col), expected_start) + return res + else: + nt.assert_is(res, None) + +def check_transform(transformer_cls, case): + lines, start, expected = case + transformer = transformer_cls(start) + nt.assert_equal(transformer.transform(lines), expected) + def test_continued_line(): lines = MULTILINE_MAGIC_ASSIGN[0] nt.assert_equal(ipt2.find_end_of_continued_line(lines, 1), 2) @@ -66,58 +117,63 @@ def test_continued_line(): nt.assert_equal(ipt2.assemble_continued_line(lines, (1, 5), 2), "foo bar") def test_find_assign_magic(): - tbl = make_tokens_by_line(MULTILINE_MAGIC_ASSIGN[0]) - nt.assert_equal(ipt2.MagicAssign.find(tbl), (2, 4)) - - tbl = make_tokens_by_line(MULTILINE_SYSTEM_ASSIGN[0]) # Nothing to find - nt.assert_equal(ipt2.MagicAssign.find(tbl), None) + check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN) + check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False) def test_transform_assign_magic(): - res = ipt2.MagicAssign.transform(MULTILINE_MAGIC_ASSIGN[0], (2, 4)) - nt.assert_equal(res, MULTILINE_MAGIC_ASSIGN[1]) + check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN) def test_find_assign_system(): - tbl = make_tokens_by_line(MULTILINE_SYSTEM_ASSIGN[0]) - nt.assert_equal(ipt2.SystemAssign.find(tbl), (2, 4)) + check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN) + check_find(ipt2.SystemAssign, (["a = !ls\n"], (1, 5), None)) + check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None)) + check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False) - tbl = make_tokens_by_line(["a = !ls\n"]) - nt.assert_equal(ipt2.SystemAssign.find(tbl), (1, 5)) +def test_transform_assign_system(): + check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN) - tbl = make_tokens_by_line(["a=!ls\n"]) - nt.assert_equal(ipt2.SystemAssign.find(tbl), (1, 2)) +def test_find_magic_escape(): + check_find(ipt2.EscapedCommand, MULTILINE_MAGIC) + check_find(ipt2.EscapedCommand, INDENTED_MAGIC) + check_find(ipt2.EscapedCommand, MULTILINE_MAGIC_ASSIGN, match=False) - tbl = make_tokens_by_line(MULTILINE_MAGIC_ASSIGN[0]) # Nothing to find - nt.assert_equal(ipt2.SystemAssign.find(tbl), None) +def test_transform_magic_escape(): + check_transform(ipt2.EscapedCommand, MULTILINE_MAGIC) + check_transform(ipt2.EscapedCommand, INDENTED_MAGIC) -def test_transform_assign_system(): - res = ipt2.SystemAssign.transform(MULTILINE_SYSTEM_ASSIGN[0], (2, 4)) - nt.assert_equal(res, MULTILINE_SYSTEM_ASSIGN[1]) +def test_find_autocalls(): + for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]: + print("Testing %r" % case[0]) + check_find(ipt2.EscapedCommand, case) -def test_find_magic_escape(): - tbl = make_tokens_by_line(MULTILINE_MAGIC[0]) - nt.assert_equal(ipt2.EscapedCommand.find(tbl), (2, 0)) +def test_transform_autocall(): + for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]: + print("Testing %r" % case[0]) + check_transform(ipt2.EscapedCommand, case) - tbl = make_tokens_by_line(INDENTED_MAGIC[0]) - nt.assert_equal(ipt2.EscapedCommand.find(tbl), (2, 4)) +def test_find_help(): + for case in [SIMPLE_HELP, DETAILED_HELP, MAGIC_HELP, HELP_IN_EXPR]: + check_find(ipt2.HelpEnd, case) - tbl = make_tokens_by_line(MULTILINE_MAGIC_ASSIGN[0]) # Shouldn't find a = %foo - nt.assert_equal(ipt2.EscapedCommand.find(tbl), None) + tf = check_find(ipt2.HelpEnd, HELP_CONTINUED_LINE) + nt.assert_equal(tf.q_line, 1) + nt.assert_equal(tf.q_col, 3) -def test_transform_magic_escape(): - res = ipt2.EscapedCommand.transform(MULTILINE_MAGIC[0], (2, 0)) - nt.assert_equal(res, MULTILINE_MAGIC[1]) + tf = check_find(ipt2.HelpEnd, HELP_MULTILINE) + nt.assert_equal(tf.q_line, 1) + nt.assert_equal(tf.q_col, 8) - res = ipt2.EscapedCommand.transform(INDENTED_MAGIC[0], (2, 4)) - nt.assert_equal(res, INDENTED_MAGIC[1]) + # ? in a comment does not trigger help + check_find(ipt2.HelpEnd, (["foo # bar?\n"], None, None), match=False) + # Nor in a string + check_find(ipt2.HelpEnd, (["foo = '''bar?\n"], None, None), match=False) -def test_find_autocalls(): - for sample, _ in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]: - print("Testing %r" % sample) - tbl = make_tokens_by_line(sample) - nt.assert_equal(ipt2.EscapedCommand.find(tbl), (1, 0)) +def test_transform_help(): + tf = ipt2.HelpEnd((1, 0), (1, 9)) + nt.assert_equal(tf.transform(HELP_IN_EXPR[0]), HELP_IN_EXPR[2]) -def test_transform_autocall(): - for sample, expected in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]: - print("Testing %r" % sample) - res = ipt2.EscapedCommand.transform(sample, (1, 0)) - nt.assert_equal(res, expected) + tf = ipt2.HelpEnd((1, 0), (2, 3)) + nt.assert_equal(tf.transform(HELP_CONTINUED_LINE[0]), HELP_CONTINUED_LINE[2]) + + tf = ipt2.HelpEnd((1, 0), (2, 8)) + nt.assert_equal(tf.transform(HELP_MULTILINE[0]), HELP_MULTILINE[2])