diff --git a/IPython/core/inputtransformer2.py b/IPython/core/inputtransformer2.py
index 36700c0..29044d6 100644
--- a/IPython/core/inputtransformer2.py
+++ b/IPython/core/inputtransformer2.py
@@ -74,12 +74,6 @@ line_transforms = [
# -----
-def help_end(tokens_by_line):
- pass
-
-def escaped_command(tokens_by_line):
- pass
-
def _find_assign_op(token_line):
# Find the first assignment in the line ('=' not inside brackets)
# We don't try to support multiple special assignment (a = b = %foo)
@@ -114,9 +108,23 @@ def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):
return ' '.join([p[:-2] for p in parts[:-1]] # Strip backslash+newline
+ [parts[-1][:-1]]) # Strip newline from last line
-class MagicAssign:
- @staticmethod
- def find(tokens_by_line):
+class TokenTransformBase:
+ # Lower numbers -> higher priority (for matches in the same location)
+ priority = 10
+
+ def sortby(self):
+ return self.start_line, self.start_col, self.priority
+
+ def __init__(self, start):
+ self.start_line = start[0] - 1 # Shift from 1-index to 0-index
+ self.start_col = start[1]
+
+ def transform(self, lines: List[str]):
+ raise NotImplementedError
+
+class MagicAssign(TokenTransformBase):
+ @classmethod
+ def find(cls, tokens_by_line):
"""Find the first magic assignment (a = %foo) in the cell.
Returns (line, column) of the % if found, or None. *line* is 1-indexed.
@@ -127,15 +135,12 @@ class MagicAssign:
and (len(line) >= assign_ix + 2) \
and (line[assign_ix+1].string == '%') \
and (line[assign_ix+2].type == tokenize2.NAME):
- return line[assign_ix+1].start
+ return cls(line[assign_ix+1].start)
- @staticmethod
- def transform(lines: List[str], start: Tuple[int, int]):
+ def transform(self, lines: List[str]):
"""Transform a magic assignment found by find
"""
- start_line = start[0] - 1 # Shift from 1-index to 0-index
- start_col = start[1]
-
+ start_line, start_col = self.start_line, self.start_col
lhs = lines[start_line][:start_col]
end_line = find_end_of_continued_line(lines, start_line)
rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
@@ -150,9 +155,9 @@ class MagicAssign:
return lines_before + [new_line] + lines_after
-class SystemAssign:
- @staticmethod
- def find(tokens_by_line):
+class SystemAssign(TokenTransformBase):
+ @classmethod
+ def find(cls, tokens_by_line):
"""Find the first system assignment (a = !foo) in the cell.
Returns (line, column) of the ! if found, or None. *line* is 1-indexed.
@@ -166,17 +171,15 @@ class SystemAssign:
while ix < len(line) and line[ix].type == tokenize2.ERRORTOKEN:
if line[ix].string == '!':
- return line[ix].start
+ return cls(line[ix].start)
elif not line[ix].string.isspace():
break
ix += 1
- @staticmethod
- def transform(lines: List[str], start: Tuple[int, int]):
+ def transform(self, lines: List[str]):
"""Transform a system assignment found by find
"""
- start_line = start[0] - 1 # Shift from 1-index to 0-index
- start_col = start[1]
+ start_line, start_col = self.start_line, self.start_col
lhs = lines[start_line][:start_col]
end_line = find_end_of_continued_line(lines, start_line)
@@ -271,9 +274,9 @@ tr = { ESC_SHELL : 'get_ipython().system({!r})'.format,
ESC_QUOTE2 : _tr_quote2,
ESC_PAREN : _tr_paren }
-class EscapedCommand:
- @staticmethod
- def find(tokens_by_line):
+class EscapedCommand(TokenTransformBase):
+ @classmethod
+ def find(cls, tokens_by_line):
"""Find the first escaped command (%foo, !foo, etc.) in the cell.
Returns (line, column) of the escape if found, or None. *line* is 1-indexed.
@@ -283,12 +286,10 @@ class EscapedCommand:
while line[ix].type in {tokenize2.INDENT, tokenize2.DEDENT}:
ix += 1
if line[ix].string in ESCAPE_SINGLES:
- return line[ix].start
+ return cls(line[ix].start)
- @staticmethod
- def transform(lines, start):
- start_line = start[0] - 1 # Shift from 1-index to 0-index
- start_col = start[1]
+ def transform(self, lines):
+ start_line, start_col = self.start_line, self.start_col
indent = lines[start_line][:start_col]
end_line = find_end_of_continued_line(lines, start_line)
@@ -306,6 +307,57 @@ class EscapedCommand:
return lines_before + [new_line] + lines_after
+_help_end_re = re.compile(r"""(%{0,2}
+ [a-zA-Z_*][\w*]* # Variable name
+ (\.[a-zA-Z_*][\w*]*)* # .etc.etc
+ )
+ (\?\??)$ # ? or ??
+ """,
+ re.VERBOSE)
+
+class HelpEnd(TokenTransformBase):
+ # This needs to be higher priority (lower number) than EscapedCommand so
+ # that inspecting magics (%foo?) works.
+ priority = 5
+
+ def __init__(self, start, q_locn):
+ super().__init__(start)
+ self.q_line = q_locn[0] - 1 # Shift from 1-indexed to 0-indexed
+ self.q_col = q_locn[1]
+
+ @classmethod
+ def find(cls, tokens_by_line):
+ for line in tokens_by_line:
+ # Last token is NEWLINE; look at last but one
+ if len(line) > 2 and line[-2].string == '?':
+ # Find the first token that's not INDENT/DEDENT
+ ix = 0
+ while line[ix].type in {tokenize2.INDENT, tokenize2.DEDENT}:
+ ix += 1
+ return cls(line[ix].start, line[-2].start)
+
+ def transform(self, lines):
+ piece = ''.join(lines[self.start_line:self.q_line+1])
+ indent, content = piece[:self.start_col], piece[self.start_col:]
+ lines_before = lines[:self.start_line]
+ lines_after = lines[self.q_line + 1:]
+
+ m = _help_end_re.search(content)
+ assert m is not None, content
+ target = m.group(1)
+ esc = m.group(3)
+
+ # If we're mid-command, put it back on the next prompt for the user.
+ next_input = None
+ if (not lines_before) and (not lines_after) \
+ and content.strip() != m.group(0):
+ next_input = content.rstrip('?\n')
+
+ call = _make_help_call(target, esc, next_input=next_input)
+ new_line = indent + call + '\n'
+
+ return lines_before + [new_line] + lines_after
+
def make_tokens_by_line(lines):
tokens_by_line = [[]]
for token in generate_tokens(iter(lines).__next__):
@@ -330,6 +382,8 @@ class TokenTransformers:
self.transformers = [
MagicAssign,
SystemAssign,
+ EscapedCommand,
+ HelpEnd,
]
def do_one_transform(self, lines):
@@ -348,17 +402,17 @@ class TokenTransformers:
"""
tokens_by_line = make_tokens_by_line(lines)
candidates = []
- for transformer in self.transformers:
- locn = transformer.find(tokens_by_line)
- if locn:
- candidates.append((locn, transformer))
-
+ for transformer_cls in self.transformers:
+ transformer = transformer_cls.find(tokens_by_line)
+ if transformer:
+ candidates.append(transformer)
+
if not candidates:
# Nothing to transform
return False, lines
-
- first_locn, transformer = min(candidates)
- return True, transformer.transform(lines, first_locn)
+
+ transformer = min(candidates, key=TokenTransformBase.sortby)
+ return True, transformer.transform(lines)
def __call__(self, lines):
while True:
@@ -366,9 +420,6 @@ class TokenTransformers:
if not changed:
return lines
-def assign_from_system(tokens_by_line, lines):
- pass
-
def transform_cell(cell):
if not cell.endswith('\n'):
diff --git a/IPython/core/tests/test_inputtransformer2.py b/IPython/core/tests/test_inputtransformer2.py
index 6067747..cf59da2 100644
--- a/IPython/core/tests/test_inputtransformer2.py
+++ b/IPython/core/tests/test_inputtransformer2.py
@@ -8,7 +8,7 @@ a = f()
%foo \\
bar
g()
-""".splitlines(keepends=True), """\
+""".splitlines(keepends=True), (2, 0), """\
a = f()
get_ipython().run_line_magic('foo', ' bar')
g()
@@ -17,7 +17,7 @@ g()
INDENTED_MAGIC = ("""\
for a in range(5):
%ls
-""".splitlines(keepends=True), """\
+""".splitlines(keepends=True), (2, 4), """\
for a in range(5):
get_ipython().run_line_magic('ls', '')
""".splitlines(keepends=True))
@@ -27,7 +27,7 @@ a = f()
b = %foo \\
bar
g()
-""".splitlines(keepends=True), """\
+""".splitlines(keepends=True), (2, 4), """\
a = f()
b = get_ipython().run_line_magic('foo', ' bar')
g()
@@ -38,27 +38,78 @@ a = f()
b = !foo \\
bar
g()
-""".splitlines(keepends=True), """\
+""".splitlines(keepends=True), (2, 4), """\
a = f()
b = get_ipython().getoutput('foo bar')
g()
""".splitlines(keepends=True))
AUTOCALL_QUOTE = (
- [",f 1 2 3\n"],
+ [",f 1 2 3\n"], (1, 0),
['f("1", "2", "3")\n']
)
AUTOCALL_QUOTE2 = (
- [";f 1 2 3\n"],
+ [";f 1 2 3\n"], (1, 0),
['f("1 2 3")\n']
)
AUTOCALL_PAREN = (
- ["/f 1 2 3\n"],
+ ["/f 1 2 3\n"], (1, 0),
['f(1, 2, 3)\n']
)
+SIMPLE_HELP = (
+ ["foo?\n"], (1, 0),
+ ["get_ipython().run_line_magic('pinfo', 'foo')\n"]
+)
+
+DETAILED_HELP = (
+ ["foo??\n"], (1, 0),
+ ["get_ipython().run_line_magic('pinfo2', 'foo')\n"]
+)
+
+MAGIC_HELP = (
+ ["%foo?\n"], (1, 0),
+ ["get_ipython().run_line_magic('pinfo', '%foo')\n"]
+)
+
+HELP_IN_EXPR = (
+ ["a = b + c?\n"], (1, 0),
+ ["get_ipython().set_next_input('a = b + c');"
+ "get_ipython().run_line_magic('pinfo', 'c')\n"]
+)
+
+HELP_CONTINUED_LINE = ("""\
+a = \\
+zip?
+""".splitlines(keepends=True), (1, 0),
+[r"get_ipython().set_next_input('a = \\\nzip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
+)
+
+HELP_MULTILINE = ("""\
+(a,
+b) = zip?
+""".splitlines(keepends=True), (1, 0),
+[r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
+)
+
+def check_find(transformer, case, match=True):
+ sample, expected_start, _ = case
+ tbl = make_tokens_by_line(sample)
+ res = transformer.find(tbl)
+ if match:
+ # start_line is stored 0-indexed, expected values are 1-indexed
+ nt.assert_equal((res.start_line+1, res.start_col), expected_start)
+ return res
+ else:
+ nt.assert_is(res, None)
+
+def check_transform(transformer_cls, case):
+ lines, start, expected = case
+ transformer = transformer_cls(start)
+ nt.assert_equal(transformer.transform(lines), expected)
+
def test_continued_line():
lines = MULTILINE_MAGIC_ASSIGN[0]
nt.assert_equal(ipt2.find_end_of_continued_line(lines, 1), 2)
@@ -66,58 +117,63 @@ def test_continued_line():
nt.assert_equal(ipt2.assemble_continued_line(lines, (1, 5), 2), "foo bar")
def test_find_assign_magic():
- tbl = make_tokens_by_line(MULTILINE_MAGIC_ASSIGN[0])
- nt.assert_equal(ipt2.MagicAssign.find(tbl), (2, 4))
-
- tbl = make_tokens_by_line(MULTILINE_SYSTEM_ASSIGN[0]) # Nothing to find
- nt.assert_equal(ipt2.MagicAssign.find(tbl), None)
+ check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
+ check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False)
def test_transform_assign_magic():
- res = ipt2.MagicAssign.transform(MULTILINE_MAGIC_ASSIGN[0], (2, 4))
- nt.assert_equal(res, MULTILINE_MAGIC_ASSIGN[1])
+ check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
def test_find_assign_system():
- tbl = make_tokens_by_line(MULTILINE_SYSTEM_ASSIGN[0])
- nt.assert_equal(ipt2.SystemAssign.find(tbl), (2, 4))
+ check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
+ check_find(ipt2.SystemAssign, (["a = !ls\n"], (1, 5), None))
+ check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None))
+ check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False)
- tbl = make_tokens_by_line(["a = !ls\n"])
- nt.assert_equal(ipt2.SystemAssign.find(tbl), (1, 5))
+def test_transform_assign_system():
+ check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
- tbl = make_tokens_by_line(["a=!ls\n"])
- nt.assert_equal(ipt2.SystemAssign.find(tbl), (1, 2))
+def test_find_magic_escape():
+ check_find(ipt2.EscapedCommand, MULTILINE_MAGIC)
+ check_find(ipt2.EscapedCommand, INDENTED_MAGIC)
+ check_find(ipt2.EscapedCommand, MULTILINE_MAGIC_ASSIGN, match=False)
- tbl = make_tokens_by_line(MULTILINE_MAGIC_ASSIGN[0]) # Nothing to find
- nt.assert_equal(ipt2.SystemAssign.find(tbl), None)
+def test_transform_magic_escape():
+ check_transform(ipt2.EscapedCommand, MULTILINE_MAGIC)
+ check_transform(ipt2.EscapedCommand, INDENTED_MAGIC)
-def test_transform_assign_system():
- res = ipt2.SystemAssign.transform(MULTILINE_SYSTEM_ASSIGN[0], (2, 4))
- nt.assert_equal(res, MULTILINE_SYSTEM_ASSIGN[1])
+def test_find_autocalls():
+ for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
+ print("Testing %r" % case[0])
+ check_find(ipt2.EscapedCommand, case)
-def test_find_magic_escape():
- tbl = make_tokens_by_line(MULTILINE_MAGIC[0])
- nt.assert_equal(ipt2.EscapedCommand.find(tbl), (2, 0))
+def test_transform_autocall():
+ for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
+ print("Testing %r" % case[0])
+ check_transform(ipt2.EscapedCommand, case)
- tbl = make_tokens_by_line(INDENTED_MAGIC[0])
- nt.assert_equal(ipt2.EscapedCommand.find(tbl), (2, 4))
+def test_find_help():
+ for case in [SIMPLE_HELP, DETAILED_HELP, MAGIC_HELP, HELP_IN_EXPR]:
+ check_find(ipt2.HelpEnd, case)
- tbl = make_tokens_by_line(MULTILINE_MAGIC_ASSIGN[0]) # Shouldn't find a = %foo
- nt.assert_equal(ipt2.EscapedCommand.find(tbl), None)
+ tf = check_find(ipt2.HelpEnd, HELP_CONTINUED_LINE)
+ nt.assert_equal(tf.q_line, 1)
+ nt.assert_equal(tf.q_col, 3)
-def test_transform_magic_escape():
- res = ipt2.EscapedCommand.transform(MULTILINE_MAGIC[0], (2, 0))
- nt.assert_equal(res, MULTILINE_MAGIC[1])
+ tf = check_find(ipt2.HelpEnd, HELP_MULTILINE)
+ nt.assert_equal(tf.q_line, 1)
+ nt.assert_equal(tf.q_col, 8)
- res = ipt2.EscapedCommand.transform(INDENTED_MAGIC[0], (2, 4))
- nt.assert_equal(res, INDENTED_MAGIC[1])
+ # ? in a comment does not trigger help
+ check_find(ipt2.HelpEnd, (["foo # bar?\n"], None, None), match=False)
+ # Nor in a string
+ check_find(ipt2.HelpEnd, (["foo = '''bar?\n"], None, None), match=False)
-def test_find_autocalls():
- for sample, _ in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
- print("Testing %r" % sample)
- tbl = make_tokens_by_line(sample)
- nt.assert_equal(ipt2.EscapedCommand.find(tbl), (1, 0))
+def test_transform_help():
+ tf = ipt2.HelpEnd((1, 0), (1, 9))
+ nt.assert_equal(tf.transform(HELP_IN_EXPR[0]), HELP_IN_EXPR[2])
-def test_transform_autocall():
- for sample, expected in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
- print("Testing %r" % sample)
- res = ipt2.EscapedCommand.transform(sample, (1, 0))
- nt.assert_equal(res, expected)
+ tf = ipt2.HelpEnd((1, 0), (2, 3))
+ nt.assert_equal(tf.transform(HELP_CONTINUED_LINE[0]), HELP_CONTINUED_LINE[2])
+
+ tf = ipt2.HelpEnd((1, 0), (2, 8))
+ nt.assert_equal(tf.transform(HELP_MULTILINE[0]), HELP_MULTILINE[2])