Merge pull request #11041 from takluyver/inputtransformer2

New input transformation machinery

IPython/core/inputtransformer2.py

@@ -0,0 +1,638 b''
+"""Input transformer machinery to support IPython special syntax.
+
+This includes the machinery to recognise and transform ``%magic`` commands,
+``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
+
+Added: IPython 7.0. Replaces inputsplitter and inputtransformer which were
+deprecated in 7.0.
+"""
+
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+from codeop import compile_command
+import re
+import tokenize
+from typing import List, Tuple
+import warnings
+
+_indent_re = re.compile(r'^[ \t]+')
+
+def leading_indent(lines):
+    """Remove leading indentation.
+    
+    If the first line starts with a spaces or tabs, the same whitespace will be
+    removed from each following line in the cell.
+    """
+    m = _indent_re.match(lines[0])
+    if not m:
+        return lines
+    space = m.group(0)
+    n = len(space)
+    return [l[n:] if l.startswith(space) else l
+            for l in lines]
+
+class PromptStripper:
+    """Remove matching input prompts from a block of input.
+    
+    Parameters
+    ----------
+    prompt_re : regular expression
+        A regular expression matching any input prompt (including continuation,
+        e.g. ``...``)
+    initial_re : regular expression, optional
+        A regular expression matching only the initial prompt, but not continuation.
+        If no initial expression is given, prompt_re will be used everywhere.
+        Used mainly for plain Python prompts (``>>>``), where the continuation prompt
+        ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
+    
+    If initial_re and prompt_re differ,
+    only initial_re will be tested against the first line.
+    If any prompt is found on the first two lines,
+    prompts will be stripped from the rest of the block.
+    """
+    def __init__(self, prompt_re, initial_re=None):
+        self.prompt_re = prompt_re
+        self.initial_re = initial_re or prompt_re
+
+    def _strip(self, lines):
+        return [self.prompt_re.sub('', l, count=1) for l in lines]
+
+    def __call__(self, lines):
+        if self.initial_re.match(lines[0]) or \
+                (len(lines) > 1 and self.prompt_re.match(lines[1])):
+            return self._strip(lines)
+        return lines
+
+classic_prompt = PromptStripper(
+    prompt_re=re.compile(r'^(>>>|\.\.\.)( |$)'),
+    initial_re=re.compile(r'^>>>( |$)')
+)
+
+ipython_prompt = PromptStripper(re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)'))
+
+def cell_magic(lines):
+    if not lines[0].startswith('%%'):
+        return lines
+    if re.match('%%\w+\?', lines[0]):
+        # This case will be handled by help_end
+        return lines
+    magic_name, _, first_line = lines[0][2:-1].partition(' ')
+    body = ''.join(lines[1:])
+    return ['get_ipython().run_cell_magic(%r, %r, %r)\n'
+            % (magic_name, first_line, body)]
+
+
+def _find_assign_op(token_line):
+    """Get the index of the first assignment in the line ('=' not inside brackets)
+
+    Note: We don't try to support multiple special assignment (a = b = %foo)
+    """
+    paren_level = 0
+    for i, ti in enumerate(token_line):
+        s = ti.string
+        if s == '=' and paren_level == 0:
+            return i 
+        if s in '([{':
+            paren_level += 1
+        elif s in ')]}':
+            if paren_level > 0:
+                paren_level -= 1
+
+def find_end_of_continued_line(lines, start_line: int):
+    """Find the last line of a line explicitly extended using backslashes.
+
+    Uses 0-indexed line numbers.
+    """
+    end_line = start_line
+    while lines[end_line].endswith('\\\n'):
+        end_line += 1
+        if end_line >= len(lines):
+            break
+    return end_line
+
+def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):
+    """Assemble a single line from multiple continued line pieces 
+
+    Continued lines are lines ending in ``\``, and the line following the last
+    ``\`` in the block.
+
+    For example, this code continues over multiple lines::
+
+        if (assign_ix is not None) \
+             and (len(line) >= assign_ix + 2) \
+             and (line[assign_ix+1].string == '%') \
+             and (line[assign_ix+2].type == tokenize.NAME):
+
+    This statement contains four continued line pieces.
+    Assembling these pieces into a single line would give::
+
+        if (assign_ix is not None) and (len(line) >= assign_ix + 2) and (line[...
+
+    This uses 0-indexed line numbers. *start* is (lineno, colno).
+
+    Used to allow ``%magic`` and ``!system`` commands to be continued over
+    multiple lines.
+    """
+    parts = [lines[start[0]][start[1]:]] + lines[start[0]+1:end_line+1]
+    return ' '.join([p[:-2] for p in parts[:-1]]  # Strip backslash+newline
+                    + [parts[-1][:-1]])         # Strip newline from last line
+
+class TokenTransformBase:
+    """Base class for transformations which examine tokens.
+
+    Special syntax should not be transformed when it occurs inside strings or
+    comments. This is hard to reliably avoid with regexes. The solution is to
+    tokenise the code as Python, and recognise the special syntax in the tokens.
+
+    IPython's special syntax is not valid Python syntax, so tokenising may go
+    wrong after the special syntax starts. These classes therefore find and
+    transform *one* instance of special syntax at a time into regular Python
+    syntax. After each transformation, tokens are regenerated to find the next
+    piece of special syntax.
+
+    Subclasses need to implement one class method (find)
+    and one regular method (transform).
+
+    The priority attribute can select which transformation to apply if multiple
+    transformers match in the same place. Lower numbers have higher priority.
+    This allows "%magic?" to be turned into a help call rather than a magic call.
+    """
+    # Lower numbers -> higher priority (for matches in the same location)
+    priority = 10
+
+    def sortby(self):
+        return self.start_line, self.start_col, self.priority
+
+    def __init__(self, start):
+        self.start_line = start[0] - 1   # Shift from 1-index to 0-index
+        self.start_col = start[1]
+
+    @classmethod
+    def find(cls, tokens_by_line):
+        """Find one instance of special syntax in the provided tokens.
+
+        Tokens are grouped into logical lines for convenience,
+        so it is easy to e.g. look at the first token of each line.
+        *tokens_by_line* is a list of lists of tokenize.TokenInfo objects.
+
+        This should return an instance of its class, pointing to the start
+        position it has found, or None if it found no match.
+        """
+        raise NotImplementedError
+
+    def transform(self, lines: List[str]):
+        """Transform one instance of special syntax found by ``find()``
+
+        Takes a list of strings representing physical lines,
+        returns a similar list of transformed lines.
+        """
+        raise NotImplementedError
+
+class MagicAssign(TokenTransformBase):
+    """Transformer for assignments from magics (a = %foo)"""
+    @classmethod
+    def find(cls, tokens_by_line):
+        """Find the first magic assignment (a = %foo) in the cell.
+        """
+        for line in tokens_by_line:
+            assign_ix = _find_assign_op(line)
+            if (assign_ix is not None) \
+                    and (len(line) >= assign_ix + 2) \
+                    and (line[assign_ix+1].string == '%') \
+                    and (line[assign_ix+2].type == tokenize.NAME):
+                return cls(line[assign_ix+1].start)
+    
+    def transform(self, lines: List[str]):
+        """Transform a magic assignment found by the ``find()`` classmethod.
+        """
+        start_line, start_col = self.start_line, self.start_col
+        lhs = lines[start_line][:start_col]
+        end_line = find_end_of_continued_line(lines, start_line)
+        rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
+        assert rhs.startswith('%'), rhs
+        magic_name, _, args = rhs[1:].partition(' ')
+        
+        lines_before = lines[:start_line]
+        call = "get_ipython().run_line_magic({!r}, {!r})".format(magic_name, args)
+        new_line = lhs + call + '\n'
+        lines_after = lines[end_line+1:]
+        
+        return lines_before + [new_line] + lines_after
+
+
+class SystemAssign(TokenTransformBase):
+    """Transformer for assignments from system commands (a = !foo)"""
+    @classmethod
+    def find(cls, tokens_by_line):
+        """Find the first system assignment (a = !foo) in the cell.
+        """
+        for line in tokens_by_line:
+            assign_ix = _find_assign_op(line)
+            if (assign_ix is not None) \
+                    and (len(line) >= assign_ix + 2) \
+                    and (line[assign_ix + 1].type == tokenize.ERRORTOKEN):
+                ix = assign_ix + 1
+
+                while ix < len(line) and line[ix].type == tokenize.ERRORTOKEN:
+                    if line[ix].string == '!':
+                        return cls(line[ix].start)
+                    elif not line[ix].string.isspace():
+                        break
+                    ix += 1
+
+    def transform(self, lines: List[str]):
+        """Transform a system assignment found by the ``find()`` classmethod.
+        """
+        start_line, start_col = self.start_line, self.start_col
+
+        lhs = lines[start_line][:start_col]
+        end_line = find_end_of_continued_line(lines, start_line)
+        rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
+        assert rhs.startswith('!'), rhs
+        cmd = rhs[1:]
+
+        lines_before = lines[:start_line]
+        call = "get_ipython().getoutput({!r})".format(cmd)
+        new_line = lhs + call + '\n'
+        lines_after = lines[end_line + 1:]
+
+        return lines_before + [new_line] + lines_after
+
+# The escape sequences that define the syntax transformations IPython will
+# apply to user input.  These can NOT be just changed here: many regular
+# expressions and other parts of the code may use their hardcoded values, and
+# for all intents and purposes they constitute the 'IPython syntax', so they
+# should be considered fixed.
+
+ESC_SHELL  = '!'     # Send line to underlying system shell
+ESC_SH_CAP = '!!'    # Send line to system shell and capture output
+ESC_HELP   = '?'     # Find information about object
+ESC_HELP2  = '??'    # Find extra-detailed information about object
+ESC_MAGIC  = '%'     # Call magic function
+ESC_MAGIC2 = '%%'    # Call cell-magic function
+ESC_QUOTE  = ','     # Split args on whitespace, quote each as string and call
+ESC_QUOTE2 = ';'     # Quote all args as a single string, call
+ESC_PAREN  = '/'     # Call first argument with rest of line as arguments
+
+ESCAPE_SINGLES = {'!', '?', '%', ',', ';', '/'}
+ESCAPE_DOUBLES = {'!!', '??'}  # %% (cell magic) is handled separately
+
+def _make_help_call(target, esc, next_input=None):
+    """Prepares a pinfo(2)/psearch call from a target name and the escape
+    (i.e. ? or ??)"""
+    method  = 'pinfo2' if esc == '??' \
+                else 'psearch' if '*' in target \
+                else 'pinfo'
+    arg = " ".join([method, target])
+    #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
+    t_magic_name, _, t_magic_arg_s = arg.partition(' ')
+    t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
+    if next_input is None:
+        return 'get_ipython().run_line_magic(%r, %r)' % (t_magic_name, t_magic_arg_s)
+    else:
+        return 'get_ipython().set_next_input(%r);get_ipython().run_line_magic(%r, %r)' % \
+           (next_input, t_magic_name, t_magic_arg_s)
+
+def _tr_help(content):
+    """Translate lines escaped with: ?
+
+    A naked help line should fire the intro help screen (shell.show_usage())
+    """
+    if not content:
+        return 'get_ipython().show_usage()'
+
+    return _make_help_call(content, '?')
+
+def _tr_help2(content):
+    """Translate lines escaped with: ??
+
+    A naked help line should fire the intro help screen (shell.show_usage())
+    """
+    if not content:
+        return 'get_ipython().show_usage()'
+
+    return _make_help_call(content, '??')
+
+def _tr_magic(content):
+    "Translate lines escaped with a percent sign: %"
+    name, _, args = content.partition(' ')
+    return 'get_ipython().run_line_magic(%r, %r)' % (name, args)
+
+def _tr_quote(content):
+    "Translate lines escaped with a comma: ,"
+    name, _, args = content.partition(' ')
+    return '%s("%s")' % (name, '", "'.join(args.split()) )
+
+def _tr_quote2(content):
+    "Translate lines escaped with a semicolon: ;"
+    name, _, args = content.partition(' ')
+    return '%s("%s")' % (name, args)
+
+def _tr_paren(content):
+    "Translate lines escaped with a slash: /"
+    name, _, args = content.partition(' ')
+    return '%s(%s)' % (name, ", ".join(args.split()))
+
+tr = { ESC_SHELL  : 'get_ipython().system({!r})'.format,
+       ESC_SH_CAP : 'get_ipython().getoutput({!r})'.format,
+       ESC_HELP   : _tr_help,
+       ESC_HELP2  : _tr_help2,
+       ESC_MAGIC  : _tr_magic,
+       ESC_QUOTE  : _tr_quote,
+       ESC_QUOTE2 : _tr_quote2,
+       ESC_PAREN  : _tr_paren }
+
+class EscapedCommand(TokenTransformBase):
+    """Transformer for escaped commands like %foo, !foo, or /foo"""
+    @classmethod
+    def find(cls, tokens_by_line):
+        """Find the first escaped command (%foo, !foo, etc.) in the cell.
+        """
+        for line in tokens_by_line:
+            ix = 0
+            while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
+                ix += 1
+            if line[ix].string in ESCAPE_SINGLES:
+                return cls(line[ix].start)
+
+    def transform(self, lines):
+        """Transform an escaped line found by the ``find()`` classmethod.
+        """
+        start_line, start_col = self.start_line, self.start_col
+
+        indent = lines[start_line][:start_col]
+        end_line = find_end_of_continued_line(lines, start_line)
+        line = assemble_continued_line(lines, (start_line, start_col), end_line)
+
+        if line[:2] in ESCAPE_DOUBLES:
+            escape, content = line[:2], line[2:]
+        else:
+            escape, content = line[:1], line[1:]
+        call = tr[escape](content)
+
+        lines_before = lines[:start_line]
+        new_line = indent + call + '\n'
+        lines_after = lines[end_line + 1:]
+
+        return lines_before + [new_line] + lines_after
+
+_help_end_re = re.compile(r"""(%{0,2}
+                              [a-zA-Z_*][\w*]*        # Variable name
+                              (\.[a-zA-Z_*][\w*]*)*   # .etc.etc
+                              )
+                              (\?\??)$                # ? or ??
+                              """,
+                              re.VERBOSE)
+
+class HelpEnd(TokenTransformBase):
+    """Transformer for help syntax: obj? and obj??"""
+    # This needs to be higher priority (lower number) than EscapedCommand so
+    # that inspecting magics (%foo?) works.
+    priority = 5
+
+    def __init__(self, start, q_locn):
+        super().__init__(start)
+        self.q_line = q_locn[0] - 1  # Shift from 1-indexed to 0-indexed
+        self.q_col = q_locn[1]
+
+    @classmethod
+    def find(cls, tokens_by_line):
+        """Find the first help command (foo?) in the cell.
+        """
+        for line in tokens_by_line:
+            # Last token is NEWLINE; look at last but one
+            if len(line) > 2 and line[-2].string == '?':
+                # Find the first token that's not INDENT/DEDENT
+                ix = 0
+                while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
+                    ix += 1
+                return cls(line[ix].start, line[-2].start)
+
+    def transform(self, lines):
+        """Transform a help command found by the ``find()`` classmethod.
+        """
+        piece = ''.join(lines[self.start_line:self.q_line+1])
+        indent, content = piece[:self.start_col], piece[self.start_col:]
+        lines_before = lines[:self.start_line]
+        lines_after = lines[self.q_line + 1:]
+
+        m = _help_end_re.search(content)
+        assert m is not None, content
+        target = m.group(1)
+        esc = m.group(3)
+
+        # If we're mid-command, put it back on the next prompt for the user.
+        next_input = None
+        if (not lines_before) and (not lines_after) \
+                and content.strip() != m.group(0):
+            next_input = content.rstrip('?\n')
+
+        call = _make_help_call(target, esc, next_input=next_input)
+        new_line = indent + call + '\n'
+
+        return lines_before + [new_line] + lines_after
+
+def make_tokens_by_line(lines):
+    """Tokenize a series of lines and group tokens by line.
+
+    The tokens for a multiline Python string or expression are
+    grouped as one line.
+    """
+    # NL tokens are used inside multiline expressions, but also after blank
+    # lines or comments. This is intentional - see https://bugs.python.org/issue17061
+    # We want to group the former case together but split the latter, so we
+    # track parentheses level, similar to the internals of tokenize.
+    NEWLINE, NL = tokenize.NEWLINE, tokenize.NL
+    tokens_by_line = [[]]
+    parenlev = 0
+    try:
+        for token in tokenize.generate_tokens(iter(lines).__next__):
+            tokens_by_line[-1].append(token)
+            if (token.type == NEWLINE) \
+                    or ((token.type == NL) and (parenlev <= 0)):
+                tokens_by_line.append([])
+            elif token.string in {'(', '[', '{'}:
+                parenlev += 1
+            elif token.string in {')', ']', '}'}:
+                if parenlev > 0:
+                    parenlev -= 1
+    except tokenize.TokenError:
+        # Input ended in a multiline string or expression. That's OK for us.
+        pass
+    
+    return tokens_by_line
+
+def show_linewise_tokens(s: str):
+    """For investigation and debugging"""
+    if not s.endswith('\n'):
+        s += '\n'
+    lines = s.splitlines(keepends=True)
+    for line in make_tokens_by_line(lines):
+        print("Line -------")
+        for tokinfo in line:
+            print(" ", tokinfo)
+
+# Arbitrary limit to prevent getting stuck in infinite loops
+TRANSFORM_LOOP_LIMIT = 500
+
+class TransformerManager:
+    """Applies various transformations to a cell or code block.
+
+    The key methods for external use are ``transform_cell()``
+    and ``check_complete()``.
+    """
+    def __init__(self):
+        self.cleanup_transforms = [
+            leading_indent,
+            classic_prompt,
+            ipython_prompt,
+        ]
+        self.line_transforms = [
+            cell_magic,
+        ]
+        self.token_transformers = [
+            MagicAssign,
+            SystemAssign,
+            EscapedCommand,
+            HelpEnd,
+        ]
+    
+    def do_one_token_transform(self, lines):
+        """Find and run the transform earliest in the code.
+        
+        Returns (changed, lines).
+        
+        This method is called repeatedly until changed is False, indicating
+        that all available transformations are complete.
+
+        The tokens following IPython special syntax might not be valid, so
+        the transformed code is retokenised every time to identify the next
+        piece of special syntax. Hopefully long code cells are mostly valid
+        Python, not using lots of IPython special syntax, so this shouldn't be
+        a performance issue.
+        """
+        tokens_by_line = make_tokens_by_line(lines)
+        candidates = []
+        for transformer_cls in self.token_transformers:
+            transformer = transformer_cls.find(tokens_by_line)
+            if transformer:
+                candidates.append(transformer)
+
+        if not candidates:
+            # Nothing to transform
+            return False, lines
+
+        transformer = min(candidates, key=TokenTransformBase.sortby)
+        return True, transformer.transform(lines)
+
+    def do_token_transforms(self, lines):
+        for _ in range(TRANSFORM_LOOP_LIMIT):
+            changed, lines = self.do_one_token_transform(lines)
+            if not changed:
+                return lines
+
+        raise RuntimeError("Input transformation still changing after "
+                           "%d iterations. Aborting." % TRANSFORM_LOOP_LIMIT)
+
+    def transform_cell(self, cell: str) -> str:
+        """Transforms a cell of input code"""
+        if not cell.endswith('\n'):
+            cell += '\n'  # Ensure the cell has a trailing newline
+        lines = cell.splitlines(keepends=True)
+        for transform in self.cleanup_transforms + self.line_transforms:
+            #print(transform, lines)
+            lines = transform(lines)
+
+        lines = self.do_token_transforms(lines)
+        return ''.join(lines)
+
+    def check_complete(self, cell: str):
+        """Return whether a block of code is ready to execute, or should be continued
+
+        Parameters
+        ----------
+        source : string
+          Python input code, which can be multiline.
+
+        Returns
+        -------
+        status : str
+          One of 'complete', 'incomplete', or 'invalid' if source is not a
+          prefix of valid code.
+        indent_spaces : int or None
+          The number of spaces by which to indent the next line of code. If
+          status is not 'incomplete', this is None.
+        """
+        if not cell.endswith('\n'):
+            cell += '\n'  # Ensure the cell has a trailing newline
+        lines = cell.splitlines(keepends=True)
+        if lines[-1][:-1].endswith('\\'):
+            # Explicit backslash continuation
+            return 'incomplete', find_last_indent(lines)
+
+        try:
+            for transform in self.cleanup_transforms:
+                lines = transform(lines)
+        except SyntaxError:
+            return 'invalid', None
+
+        if lines[0].startswith('%%'):
+            # Special case for cell magics - completion marked by blank line
+            if lines[-1].strip():
+                return 'incomplete', find_last_indent(lines)
+            else:
+                return 'complete', None
+
+        try:
+            for transform in self.line_transforms:
+                lines = transform(lines)
+            lines = self.do_token_transforms(lines)
+        except SyntaxError:
+            return 'invalid', None
+
+        tokens_by_line = make_tokens_by_line(lines)
+        if tokens_by_line[-1][-1].type != tokenize.ENDMARKER:
+            # We're in a multiline string or expression
+            return 'incomplete', find_last_indent(lines)
+
+        # Find the last token on the previous line that's not NEWLINE or COMMENT
+        toks_last_line = tokens_by_line[-2]
+        ix = len(toks_last_line) - 1
+        while ix >= 0 and toks_last_line[ix].type in {tokenize.NEWLINE,
+                                                      tokenize.COMMENT}:
+            ix -= 1
+
+        if toks_last_line[ix].string == ':':
+            # The last line starts a block (e.g. 'if foo:')
+            ix = 0
+            while toks_last_line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
+                ix += 1
+            indent = toks_last_line[ix].start[1]
+            return 'incomplete', indent + 4
+
+        # If there's a blank line at the end, assume we're ready to execute.
+        if not lines[-1].strip():
+            return 'complete', None
+
+        # At this point, our checks think the code is complete (or invalid).
+        # We'll use codeop.compile_command to check this with the real parser.
+
+        try:
+            with warnings.catch_warnings():
+                warnings.simplefilter('error', SyntaxWarning)
+                res = compile_command(''.join(lines), symbol='exec')
+        except (SyntaxError, OverflowError, ValueError, TypeError,
+                MemoryError, SyntaxWarning):
+            return 'invalid', None
+        else:
+            if res is None:
+                return 'incomplete', find_last_indent(lines)
+        return 'complete', None
+
+
+def find_last_indent(lines):
+    m = _indent_re.match(lines[-1])
+    if not m:
+        return 0
+    return len(m.group(0).replace('\t', ' '*4))

IPython/core/tests/test_inputtransformer2.py

@@ -0,0 +1,195 b''
+"""Tests for the token-based transformers in IPython.core.inputtransformer2
+
+Line-based transformers are the simpler ones; token-based transformers are
+more complex. See test_inputtransformer2_line for tests for line-based
+transformations.
+"""
+import nose.tools as nt
+
+from IPython.core import inputtransformer2 as ipt2
+from IPython.core.inputtransformer2 import make_tokens_by_line
+
+MULTILINE_MAGIC = ("""\
+a = f()
+%foo \\
+bar
+g()
+""".splitlines(keepends=True), (2, 0), """\
+a = f()
+get_ipython().run_line_magic('foo', ' bar')
+g()
+""".splitlines(keepends=True))
+
+INDENTED_MAGIC = ("""\
+for a in range(5):
+    %ls
+""".splitlines(keepends=True), (2, 4), """\
+for a in range(5):
+    get_ipython().run_line_magic('ls', '')
+""".splitlines(keepends=True))
+
+MULTILINE_MAGIC_ASSIGN = ("""\
+a = f()
+b = %foo \\
+  bar
+g()
+""".splitlines(keepends=True), (2, 4), """\
+a = f()
+b = get_ipython().run_line_magic('foo', '   bar')
+g()
+""".splitlines(keepends=True))
+
+MULTILINE_SYSTEM_ASSIGN = ("""\
+a = f()
+b = !foo \\
+  bar
+g()
+""".splitlines(keepends=True), (2, 4), """\
+a = f()
+b = get_ipython().getoutput('foo    bar')
+g()
+""".splitlines(keepends=True))
+
+AUTOCALL_QUOTE = (
+    [",f 1 2 3\n"], (1, 0),
+    ['f("1", "2", "3")\n']
+)
+
+AUTOCALL_QUOTE2 = (
+    [";f 1 2 3\n"], (1, 0),
+    ['f("1 2 3")\n']
+)
+
+AUTOCALL_PAREN = (
+    ["/f 1 2 3\n"], (1, 0),
+    ['f(1, 2, 3)\n']
+)
+
+SIMPLE_HELP = (
+    ["foo?\n"], (1, 0),
+    ["get_ipython().run_line_magic('pinfo', 'foo')\n"]
+)
+
+DETAILED_HELP = (
+    ["foo??\n"], (1, 0),
+    ["get_ipython().run_line_magic('pinfo2', 'foo')\n"]
+)
+
+MAGIC_HELP = (
+    ["%foo?\n"], (1, 0),
+    ["get_ipython().run_line_magic('pinfo', '%foo')\n"]
+)
+
+HELP_IN_EXPR = (
+    ["a = b + c?\n"], (1, 0),
+    ["get_ipython().set_next_input('a = b + c');"
+     "get_ipython().run_line_magic('pinfo', 'c')\n"]
+)
+
+HELP_CONTINUED_LINE = ("""\
+a = \\
+zip?
+""".splitlines(keepends=True), (1, 0),
+[r"get_ipython().set_next_input('a = \\\nzip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
+)
+
+HELP_MULTILINE = ("""\
+(a,
+b) = zip?
+""".splitlines(keepends=True), (1, 0),
+[r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
+)
+
+def check_find(transformer, case, match=True):
+    sample, expected_start, _  = case
+    tbl = make_tokens_by_line(sample)
+    res = transformer.find(tbl)
+    if match:
+        # start_line is stored 0-indexed, expected values are 1-indexed
+        nt.assert_equal((res.start_line+1, res.start_col), expected_start)
+        return res
+    else:
+        nt.assert_is(res, None)
+
+def check_transform(transformer_cls, case):
+    lines, start, expected = case
+    transformer = transformer_cls(start)
+    nt.assert_equal(transformer.transform(lines), expected)
+
+def test_continued_line():
+    lines = MULTILINE_MAGIC_ASSIGN[0]
+    nt.assert_equal(ipt2.find_end_of_continued_line(lines, 1), 2)
+
+    nt.assert_equal(ipt2.assemble_continued_line(lines, (1, 5), 2), "foo    bar")
+
+def test_find_assign_magic():
+    check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
+    check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False)
+
+def test_transform_assign_magic():
+    check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
+
+def test_find_assign_system():
+    check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
+    check_find(ipt2.SystemAssign, (["a =  !ls\n"], (1, 5), None))
+    check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None))
+    check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False)
+
+def test_transform_assign_system():
+    check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
+
+def test_find_magic_escape():
+    check_find(ipt2.EscapedCommand, MULTILINE_MAGIC)
+    check_find(ipt2.EscapedCommand, INDENTED_MAGIC)
+    check_find(ipt2.EscapedCommand, MULTILINE_MAGIC_ASSIGN, match=False)
+
+def test_transform_magic_escape():
+    check_transform(ipt2.EscapedCommand, MULTILINE_MAGIC)
+    check_transform(ipt2.EscapedCommand, INDENTED_MAGIC)
+
+def test_find_autocalls():
+    for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
+        print("Testing %r" % case[0])
+        check_find(ipt2.EscapedCommand, case)
+
+def test_transform_autocall():
+    for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
+        print("Testing %r" % case[0])
+        check_transform(ipt2.EscapedCommand, case)
+
+def test_find_help():
+    for case in [SIMPLE_HELP, DETAILED_HELP, MAGIC_HELP, HELP_IN_EXPR]:
+        check_find(ipt2.HelpEnd, case)
+
+    tf = check_find(ipt2.HelpEnd, HELP_CONTINUED_LINE)
+    nt.assert_equal(tf.q_line, 1)
+    nt.assert_equal(tf.q_col, 3)
+
+    tf = check_find(ipt2.HelpEnd, HELP_MULTILINE)
+    nt.assert_equal(tf.q_line, 1)
+    nt.assert_equal(tf.q_col, 8)
+
+    # ? in a comment does not trigger help
+    check_find(ipt2.HelpEnd, (["foo # bar?\n"], None, None), match=False)
+    # Nor in a string
+    check_find(ipt2.HelpEnd, (["foo = '''bar?\n"], None, None), match=False)
+
+def test_transform_help():
+    tf = ipt2.HelpEnd((1, 0), (1, 9))
+    nt.assert_equal(tf.transform(HELP_IN_EXPR[0]), HELP_IN_EXPR[2])
+
+    tf = ipt2.HelpEnd((1, 0), (2, 3))
+    nt.assert_equal(tf.transform(HELP_CONTINUED_LINE[0]), HELP_CONTINUED_LINE[2])
+
+    tf = ipt2.HelpEnd((1, 0), (2, 8))
+    nt.assert_equal(tf.transform(HELP_MULTILINE[0]), HELP_MULTILINE[2])
+
+def test_check_complete():
+    cc = ipt2.TransformerManager().check_complete
+    nt.assert_equal(cc("a = 1"), ('complete', None))
+    nt.assert_equal(cc("for a in range(5):"), ('incomplete', 4))
+    nt.assert_equal(cc("raise = 2"), ('invalid', None))
+    nt.assert_equal(cc("a = [1,\n2,"), ('incomplete', 0))
+    nt.assert_equal(cc("a = '''\n   hi"), ('incomplete', 3))
+    nt.assert_equal(cc("def a():\n x=1\n global x"), ('invalid', None))
+    nt.assert_equal(cc("a \\ "), ('invalid', None))  # Nothing allowed after backslash

IPython/core/tests/test_inputtransformer2_line.py

@@ -0,0 +1,88 b''
+"""Tests for the line-based transformers in IPython.core.inputtransformer2
+
+Line-based transformers are the simpler ones; token-based transformers are
+more complex. See test_inputtransformer2 for tests for token-based transformers.
+"""
+import nose.tools as nt
+
+from IPython.core import inputtransformer2 as ipt2
+
+CELL_MAGIC = ("""\
+%%foo arg
+body 1
+body 2
+""", """\
+get_ipython().run_cell_magic('foo', 'arg', 'body 1\\nbody 2\\n')
+""")
+
+def test_cell_magic():
+    for sample, expected in [CELL_MAGIC]:
+        nt.assert_equal(ipt2.cell_magic(sample.splitlines(keepends=True)),
+                        expected.splitlines(keepends=True))
+
+CLASSIC_PROMPT = ("""\
+>>> for a in range(5):
+...     print(a)
+""", """\
+for a in range(5):
+    print(a)
+""")
+
+CLASSIC_PROMPT_L2 = ("""\
+for a in range(5):
+...     print(a)
+...     print(a ** 2)
+""", """\
+for a in range(5):
+    print(a)
+    print(a ** 2)
+""")
+
+def test_classic_prompt():
+    for sample, expected in [CLASSIC_PROMPT, CLASSIC_PROMPT_L2]:
+        nt.assert_equal(ipt2.classic_prompt(sample.splitlines(keepends=True)),
+                        expected.splitlines(keepends=True))
+
+IPYTHON_PROMPT = ("""\
+In [1]: for a in range(5):
+   ...:     print(a)
+""", """\
+for a in range(5):
+    print(a)
+""")
+
+IPYTHON_PROMPT_L2 = ("""\
+for a in range(5):
+   ...:     print(a)
+   ...:     print(a ** 2)
+""", """\
+for a in range(5):
+    print(a)
+    print(a ** 2)
+""")
+
+def test_ipython_prompt():
+    for sample, expected in [IPYTHON_PROMPT, IPYTHON_PROMPT_L2]:
+        nt.assert_equal(ipt2.ipython_prompt(sample.splitlines(keepends=True)),
+                        expected.splitlines(keepends=True))
+
+INDENT_SPACES = ("""\
+     if True:
+        a = 3
+""", """\
+if True:
+   a = 3
+""")
+
+INDENT_TABS = ("""\
+\tif True:
+\t\tb = 4
+""", """\
+if True:
+\tb = 4
+""")
+
+def test_leading_indent():
+    for sample, expected in [INDENT_SPACES, INDENT_TABS]:
+        nt.assert_equal(ipt2.leading_indent(sample.splitlines(keepends=True)),
+                        expected.splitlines(keepends=True))

docs/source/whatsnew/pr/incompat-inputtransformer2.rst

@@ -0,0 +1,3 b''
+* The API for transforming input before it is parsed as Python code has been
+  completely redesigned, and any custom input transformations will need to be
+  rewritten. See :doc:`/config/inputtransforms` for details of the new API.

IPython/core/completer.py

@@ -131,7 +131,7 b' from types import SimpleNamespace'
 
 from traitlets.config.configurable import Configurable
 from IPython.core.error import TryNext
-from IPython.core.inputsplitter import ESC_MAGIC
+from IPython.core.inputtransformer2 import ESC_MAGIC
 from IPython.core.latex_symbols import latex_symbols, reverse_latex_symbol
 from IPython.core.oinspect import InspectColors
 from IPython.utils import generics

IPython/core/inputsplitter.py

@@ -1,4 +1,6 b''
-"""Input handling and transformation machinery.
+"""DEPRECATED: Input handling and transformation machinery.
+
+This module was deprecated in IPython 7.0, in favour of inputtransformer2.
 
 The first class in this module, :class:`InputSplitter`, is designed to tell when
 input from a line-oriented frontend is complete and should be executed, and when
@@ -14,6 +16,11 b' and stores the results.'
 For more details, see the class docstrings below.
 """
 
+from warnings import warn
+
+warn('IPython.core.inputsplitter is deprecated since IPython 7 in favor of `IPython.core.inputtransformer2`',
+     DeprecationWarning)
+
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 import ast

IPython/core/inputtransformer.py

@@ -1,4 +1,6 b''
-"""Input transformer classes to support IPython special syntax.
+"""DEPRECATED: Input transformer classes to support IPython special syntax.
+
+This module was deprecated in IPython 7.0, in favour of inputtransformer2.
 
 This includes the machinery to recognise and transform ``%magic`` commands,
 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
@@ -6,11 +8,11 b' This includes the machinery to recognise and transform ``%magic`` commands,'
 import abc
 import functools
 import re
+import tokenize
+from tokenize import generate_tokens, untokenize, TokenError
 from io import StringIO
 
 from IPython.core.splitinput import LineInfo
-from IPython.utils import tokenize2
-from IPython.utils.tokenize2 import generate_tokens, untokenize, TokenError
 
 #-----------------------------------------------------------------------------
 # Globals
@@ -138,10 +140,10 b' class TokenInputTransformer(InputTransformer):'
             for intok in self.tokenizer:
                 tokens.append(intok)
                 t = intok[0]
-                if t == tokenize2.NEWLINE or (stop_at_NL and t == tokenize2.NL):
+                if t == tokenize.NEWLINE or (stop_at_NL and t == tokenize.NL):
                     # Stop before we try to pull a line we don't have yet
                     break
-                elif t == tokenize2.ERRORTOKEN:
+                elif t == tokenize.ERRORTOKEN:
                     stop_at_NL = True
         except TokenError:
             # Multi-line statement - stop and try again with the next line
@@ -317,7 +319,7 b' def has_comment(src):'
     comment : bool
         True if source has a comment.
     """
-    return (tokenize2.COMMENT in _line_tokens(src))
+    return (tokenize.COMMENT in _line_tokens(src))
 
 def ends_in_comment_or_string(src):
     """Indicates whether or not an input line ends in a comment or within
@@ -334,7 +336,7 b' def ends_in_comment_or_string(src):'
         True if source ends in a comment or multiline string.
     """
     toktypes = _line_tokens(src)
-    return (tokenize2.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes)
+    return (tokenize.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes)
         
 
 @StatelessInputTransformer.wrap

IPython/core/interactiveshell.py

@@ -48,7 +48,7 b' from IPython.core.error import InputRejected, UsageError'
 from IPython.core.extensions import ExtensionManager
 from IPython.core.formatters import DisplayFormatter
 from IPython.core.history import HistoryManager
-from IPython.core.inputsplitter import ESC_MAGIC, ESC_MAGIC2
+from IPython.core.inputtransformer2 import ESC_MAGIC, ESC_MAGIC2
 from IPython.core.logger import Logger
 from IPython.core.macro import Macro
 from IPython.core.payload import PayloadManager
@@ -333,15 +333,31 b' class InteractiveShell(SingletonConfigurable):'
     filename = Unicode("<ipython console>")
     ipython_dir= Unicode('').tag(config=True) # Set to get_ipython_dir() in __init__
 
-    # Input splitter, to transform input line by line and detect when a block
-    # is ready to be executed.
-    input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
-                              (), {'line_input_checker': True})
+    # Used to transform cells before running them, and check whether code is complete
+    input_transformer_manager = Instance('IPython.core.inputtransformer2.TransformerManager',
+                                         ())
 
-    # This InputSplitter instance is used to transform completed cells before
-    # running them. It allows cell magics to contain blank lines.
-    input_transformer_manager = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
-                                         (), {'line_input_checker': False})
+    @property
+    def input_transformers_cleanup(self):
+        return self.input_transformer_manager.cleanup_transforms
+
+    input_transformers_post = List([],
+        help="A list of string input transformers, to be applied after IPython's "
+             "own input transformations."
+    )
+
+    @property
+    def input_splitter(self):
+        """Make this available for backward compatibility (pre-7.0 release) with existing code.
+
+        For example, ipykernel ipykernel currently uses
+        `shell.input_splitter.check_complete`
+        """
+        from warnings import warn
+        warn("`input_splitter` is deprecated since IPython 7.0, prefer `input_transformer_manager`.",
+             DeprecationWarning, stacklevel=2
+        )
+        return self.input_transformer_manager
 
     logstart = Bool(False, help=
         """
@@ -2656,6 +2672,7 b' class InteractiveShell(SingletonConfigurable):'
         -------
         result : :class:`ExecutionResult`
         """
+        result = None
         try:
             result = self._run_cell(
                 raw_cell, store_history, silent, shell_futures)
@@ -2713,21 +2730,10 b' class InteractiveShell(SingletonConfigurable):'
         preprocessing_exc_tuple = None
         try:
             # Static input transformations
-            cell = self.input_transformer_manager.transform_cell(raw_cell)
-        except SyntaxError:
+            cell = self.transform_cell(raw_cell)
+        except Exception:
             preprocessing_exc_tuple = sys.exc_info()
             cell = raw_cell  # cell has to exist so it can be stored/logged
-        else:
-            if len(cell.splitlines()) == 1:
-                # Dynamic transformations - only applied for single line commands
-                with self.builtin_trap:
-                    try:
-                        # use prefilter_lines to handle trailing newlines
-                        # restore trailing newline for ast.parse
-                        cell = self.prefilter_manager.prefilter_lines(cell) + '\n'
-                    except Exception:
-                        # don't allow prefilter errors to crash IPython
-                        preprocessing_exc_tuple = sys.exc_info()
 
         # Store raw and processed history
         if store_history:
@@ -2798,6 +2804,36 b' class InteractiveShell(SingletonConfigurable):'
             self.execution_count += 1
 
         return result
+
+    def transform_cell(self, raw_cell):
+        """Transform an input cell before parsing it.
+
+        Static transformations, implemented in IPython.core.inputtransformer2,
+        deal with things like ``%magic`` and ``!system`` commands.
+        These run on all input.
+        Dynamic transformations, for things like unescaped magics and the exit
+        autocall, depend on the state of the interpreter.
+        These only apply to single line inputs.
+
+        These string-based transformations are followed by AST transformations;
+        see :meth:`transform_ast`.
+        """
+        # Static input transformations
+        cell = self.input_transformer_manager.transform_cell(raw_cell)
+
+        if len(cell.splitlines()) == 1:
+            # Dynamic transformations - only applied for single line commands
+            with self.builtin_trap:
+                # use prefilter_lines to handle trailing newlines
+                # restore trailing newline for ast.parse
+                cell = self.prefilter_manager.prefilter_lines(cell) + '\n'
+
+        lines = cell.splitlines(keepends=True)
+        for transform in self.input_transformers_post:
+            lines = transform(lines)
+        cell = ''.join(lines)
+
+        return cell
     
     def transform_ast(self, node):
         """Apply the AST transformations from self.ast_transformers
@@ -2999,7 +3035,7 b' class InteractiveShell(SingletonConfigurable):'
           When status is 'incomplete', this is some whitespace to insert on
           the next line of the prompt.
         """
-        status, nspaces = self.input_splitter.check_complete(code)
+        status, nspaces = self.input_transformer_manager.check_complete(code)
         return status, ' ' * (nspaces or 0)
 
     #-------------------------------------------------------------------------

IPython/core/magic.py

@@ -19,7 +19,7 b' from getopt import getopt, GetoptError'
 from traitlets.config.configurable import Configurable
 from IPython.core import oinspect
 from IPython.core.error import UsageError
-from IPython.core.inputsplitter import ESC_MAGIC, ESC_MAGIC2
+from IPython.core.inputtransformer2 import ESC_MAGIC, ESC_MAGIC2
 from decorator import decorator
 from IPython.utils.ipstruct import Struct
 from IPython.utils.process import arg_split

IPython/core/magics/execution.py

@@ -297,7 +297,7 b' python-profiler package from non-free.""")'
                                            list_all=True, posix=False)
         if cell is not None:
             arg_str += '\n' + cell
-        arg_str = self.shell.input_splitter.transform_cell(arg_str)
+        arg_str = self.shell.transform_cell(arg_str)
         return self._run_with_profiler(arg_str, opts, self.shell.user_ns)
 
     def _run_with_profiler(self, code, opts, namespace):
@@ -1033,7 +1033,7 b' python-profiler package from non-free.""")'
         # this code has tight coupling to the inner workings of timeit.Timer,
         # but is there a better way to achieve that the code stmt has access
         # to the shell namespace?
-        transform  = self.shell.input_splitter.transform_cell
+        transform  = self.shell.transform_cell
 
         if cell is None:
             # called as line magic
@@ -1191,9 +1191,9 b' python-profiler package from non-free.""")'
             raise UsageError("Can't use statement directly after '%%time'!")
         
         if cell:
-            expr = self.shell.input_transformer_manager.transform_cell(cell)
+            expr = self.shell.transform_cell(cell)
         else:
-            expr = self.shell.input_transformer_manager.transform_cell(line)
+            expr = self.shell.transform_cell(line)
 
         # Minimum time above which parse time will be reported
         tp_min = 0.1

IPython/core/prefilter.py

@@ -14,7 +14,7 b' import re'
 
 from IPython.core.autocall import IPyAutocall
 from traitlets.config.configurable import Configurable
-from IPython.core.inputsplitter import (
+from IPython.core.inputtransformer2 import (
     ESC_MAGIC,
     ESC_QUOTE,
     ESC_QUOTE2,

IPython/core/tests/test_interactiveshell.py

@@ -833,28 +833,22 b' def test_user_expression():'
 class TestSyntaxErrorTransformer(unittest.TestCase):
     """Check that SyntaxError raised by an input transformer is handled by run_cell()"""
 
-    class SyntaxErrorTransformer(InputTransformer):
-
-        def push(self, line):
+    @staticmethod
+    def transformer(lines):
+        for line in lines:
             pos = line.find('syntaxerror')
             if pos >= 0:
                 e = SyntaxError('input contains "syntaxerror"')
                 e.text = line
                 e.offset = pos + 1
                 raise e
-            return line
-
-        def reset(self):
-            pass
+        return lines
 
     def setUp(self):
-        self.transformer = TestSyntaxErrorTransformer.SyntaxErrorTransformer()
-        ip.input_splitter.python_line_transforms.append(self.transformer)
-        ip.input_transformer_manager.python_line_transforms.append(self.transformer)
+        ip.input_transformers_post.append(self.transformer)
 
     def tearDown(self):
-        ip.input_splitter.python_line_transforms.remove(self.transformer)
-        ip.input_transformer_manager.python_line_transforms.remove(self.transformer)
+        ip.input_transformers_post.remove(self.transformer)
 
     def test_syntaxerror_input_transformer(self):
         with tt.AssertPrints('1234'):

IPython/core/tests/test_magic.py

@@ -691,8 +691,8 b' class CellMagicTestCase(TestCase):'
         out = _ip.run_cell_magic(magic, 'a', 'b')
         nt.assert_equal(out, ('a','b'))
         # Via run_cell, it goes into the user's namespace via displayhook
-        _ip.run_cell('%%' + magic +' c\nd')
-        nt.assert_equal(_ip.user_ns['_'], ('c','d'))
+        _ip.run_cell('%%' + magic +' c\nd\n')
+        nt.assert_equal(_ip.user_ns['_'], ('c','d\n'))
 
     def test_cell_magic_func_deco(self):
         "Cell magic using simple decorator"

IPython/core/tests/test_shellapp.py

@@ -52,5 +52,9 b' class TestFileToRun(tt.TempFileMixin, unittest.TestCase):'
         self.mktmp(src)
 
         out, err = tt.ipexec(self.fname, options=['-i'],
-                           commands=['"__file__" in globals()', 'exit()'])
-        self.assertIn("False", out)
+                           commands=['"__file__" in globals()', 'print(123)', 'exit()'])
+        if 'False' not in out:
+            print("Subprocess stderr:")
+            print(err)
+            print('-----')
+            raise AssertionError("'False' not found in %r" % out)

IPython/sphinxext/ipython_directive.py

@@ -292,6 +292,7 b' class EmbeddedSphinxShell(object):'
         self.user_ns = self.IP.user_ns
         self.user_global_ns = self.IP.user_global_ns
 
+        self.lines_waiting = []
         self.input = ''
         self.output = ''
         self.tmp_profile_dir = tmp_profile_dir
@@ -326,13 +327,12 b' class EmbeddedSphinxShell(object):'
         """process the input, capturing stdout"""
 
         stdout = sys.stdout
-        splitter = self.IP.input_splitter
         try:
             sys.stdout = self.cout
-            splitter.push(line)
-            more = splitter.push_accepts_more()
-            if not more:
-                source_raw = splitter.raw_reset()
+            self.lines_waiting.append(line)
+            if self.IP.check_complete()[0] != 'incomplete':
+                source_raw = ''.join(self.lines_waiting)
+                self.lines_waiting = []
                 self.IP.run_cell(source_raw, store_history=store_history)
         finally:
             sys.stdout = stdout

IPython/terminal/interactiveshell.py

@@ -226,16 +226,14 b' class TerminalInteractiveShell(InteractiveShell):'
     def init_prompt_toolkit_cli(self):
         if self.simple_prompt:
             # Fall back to plain non-interactive output for tests.
-            # This is very limited, and only accepts a single line.
+            # This is very limited.
             def prompt():
-                isp = self.input_splitter
                 prompt_text = "".join(x[1] for x in self.prompts.in_prompt_tokens())
+                lines = [input(prompt_text)]
                 prompt_continuation = "".join(x[1] for x in self.prompts.continuation_prompt_tokens())
-                while isp.push_accepts_more():
-                    line = input(prompt_text)
-                    isp.push(line)
-                    prompt_text = prompt_continuation
-                return isp.source_reset()
+                while self.check_complete('\n'.join(lines))[0] == 'incomplete':
+                    lines.append( input(prompt_continuation) )
+                return '\n'.join(lines)
             self.prompt_for_code = prompt
             return
 

IPython/terminal/magics.py

@@ -9,7 +9,6 b' import os'
 import sys
 
 from IPython.core.error import TryNext, UsageError
-from IPython.core.inputsplitter import IPythonInputSplitter
 from IPython.core.magic import Magics, magics_class, line_magic
 from IPython.lib.clipboard import ClipboardEmpty
 from IPython.utils.text import SList, strip_email_quotes
@@ -40,7 +39,6 b' def get_pasted_lines(sentinel, l_input=py3compat.input, quiet=False):'
 class TerminalMagics(Magics):
     def __init__(self, shell):
         super(TerminalMagics, self).__init__(shell)
-        self.input_splitter = IPythonInputSplitter()
 
     def store_or_execute(self, block, name):
         """ Execute a block, or store it in a variable, per the user's request.

IPython/terminal/shortcuts.py

@@ -68,9 +68,8 b' def create_ipython_shortcuts(shell):'
                           & insert_mode
                           & cursor_in_leading_ws
                         ))(indent_buffer)
-
-    kb.add('c-o', filter=(has_focus(DEFAULT_BUFFER)
-                  & emacs_insert_mode))(newline_autoindent_outer(shell.input_splitter))
+    kb.add('c-o', filter=(has_focus(DEFAULT_BUFFER) & emacs_insert_mode)
+           )(newline_autoindent_outer(shell.input_transformer_manager))
 
     kb.add('f2', filter=has_focus(DEFAULT_BUFFER))(open_input_in_editor)
 
@@ -107,13 +106,13 b' def newline_or_execute_outer(shell):'
             check_text = d.text
         else:
             check_text = d.text[:d.cursor_position]
-        status, indent = shell.input_splitter.check_complete(check_text + '\n')
+        status, indent = shell.check_complete(check_text)
 
         if not (d.on_last_line or
                 d.cursor_position_row >= d.line_count - d.empty_line_count_at_the_end()
                 ):
             if shell.autoindent:
-                b.insert_text('\n' + (' ' * (indent or 0)))
+                b.insert_text('\n' + indent)
             else:
                 b.insert_text('\n')
             return
@@ -122,7 +121,7 b' def newline_or_execute_outer(shell):'
             b.validate_and_handle()
         else:
             if shell.autoindent:
-                b.insert_text('\n' + (' ' * (indent or 0)))
+                b.insert_text('\n' + indent)
             else:
                 b.insert_text('\n')
     return newline_or_execute

IPython/terminal/tests/test_interactivshell.py

@@ -96,9 +96,7 b' class InteractiveShellTestCase(unittest.TestCase):'
     @mock_input
     def test_inputtransformer_syntaxerror(self):
         ip = get_ipython()
-        transformer = SyntaxErrorTransformer()
-        ip.input_splitter.python_line_transforms.append(transformer)
-        ip.input_transformer_manager.python_line_transforms.append(transformer)
+        ip.input_transformers_post.append(syntax_error_transformer)
 
         try:
             #raise Exception
@@ -112,8 +110,7 b' class InteractiveShellTestCase(unittest.TestCase):'
                 yield u'print(4*4)'
 
         finally:
-            ip.input_splitter.python_line_transforms.remove(transformer)
-            ip.input_transformer_manager.python_line_transforms.remove(transformer)
+            ip.input_transformers_post.remove(syntax_error_transformer)
 
     def test_plain_text_only(self):
         ip = get_ipython()
@@ -146,20 +143,17 b' class InteractiveShellTestCase(unittest.TestCase):'
         self.assertEqual(data, {'text/plain': repr(obj)})
         assert captured.stdout == ''
 
-
-
-class SyntaxErrorTransformer(InputTransformer):
-    def push(self, line):
+def syntax_error_transformer(lines):
+    """Transformer that throws SyntaxError if 'syntaxerror' is in the code."""
+    for line in lines:
         pos = line.find('syntaxerror')
         if pos >= 0:
             e = SyntaxError('input contains "syntaxerror"')
             e.text = line
             e.offset = pos + 1
             raise e
-        return line
+    return lines
 
-    def reset(self):
-        pass
 
 class TerminalMagicsTestCase(unittest.TestCase):
     def test_paste_magics_blankline(self):

IPython/utils/tokenutil.py

@@ -7,7 +7,7 b' from collections import namedtuple'
 from io import StringIO
 from keyword import iskeyword
 
-from . import tokenize2
+import tokenize
 
 
 Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line'])
@@ -15,9 +15,9 b" Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line'])"
 def generate_tokens(readline):
     """wrap generate_tokens to catch EOF errors"""
     try:
-        for token in tokenize2.generate_tokens(readline):
+        for token in tokenize.generate_tokens(readline):
             yield token
-    except tokenize2.TokenError:
+    except tokenize.TokenError:
         # catch EOF error
         return
 
@@ -99,12 +99,12 b' def token_at_cursor(cell, cursor_pos=0):'
             # don't consume it
             break
         
-        if tok.token == tokenize2.NAME and not iskeyword(tok.text):
-            if names and tokens and tokens[-1].token == tokenize2.OP and tokens[-1].text == '.':
+        if tok.token == tokenize.NAME and not iskeyword(tok.text):
+            if names and tokens and tokens[-1].token == tokenize.OP and tokens[-1].text == '.':
                 names[-1] = "%s.%s" % (names[-1], tok.text)
             else:
                 names.append(tok.text)
-        elif tok.token == tokenize2.OP:
+        elif tok.token == tokenize.OP:
             if tok.text == '=' and names:
                 # don't inspect the lhs of an assignment
                 names.pop(-1)

docs/autogen_shortcuts.py

@@ -45,7 +45,7 b' class _DummyTerminal(object):'
     """Used as a buffer to get prompt_toolkit bindings
     """
     handle_return = None
-    input_splitter = None
+    input_transformer_manager = None
     display_completions = None
 
 ipy_bindings = create_ipython_shortcuts(_DummyTerminal()).bindings

docs/source/config/details.rst

@@ -282,8 +282,8 b' IPython configuration::'
             else: # insert a newline with auto-indentation...
 
                 if document.line_count > 1: text = text[:document.cursor_position]
-                indent = shell.input_splitter.check_complete(text + '\n')[1] or 0
-                buffer.insert_text('\n' + ' ' * indent)
+                indent = shell.check_complete(text)[1]
+                buffer.insert_text('\n' + indent)
             
                 # if you just wanted a plain newline without any indentation, you
                 # could use `buffer.insert_text('\n')` instead of the lines above

docs/source/config/inputtransforms.rst

@@ -15,163 +15,52 b' String based transformations'
 
 .. currentmodule:: IPython.core.inputtransforms
 
-When the user enters a line of code, it is first processed as a string. By the
+When the user enters code, it is first processed as a string. By the
 end of this stage, it must be valid Python syntax.
 
-These transformers all subclass :class:`IPython.core.inputtransformer.InputTransformer`,
-and are used by :class:`IPython.core.inputsplitter.IPythonInputSplitter`.
-
-These transformers act in three groups, stored separately as lists of instances
-in attributes of :class:`~IPython.core.inputsplitter.IPythonInputSplitter`:
-
-* ``physical_line_transforms`` act on the lines as the user enters them. For
-  example, these strip Python prompts from examples pasted in.
-* ``logical_line_transforms`` act on lines as connected by explicit line
-  continuations, i.e. ``\`` at the end of physical lines. They are skipped
-  inside multiline Python statements. This is the point where IPython recognises
-  ``%magic`` commands, for instance.
-* ``python_line_transforms`` act on blocks containing complete Python statements.
-  Multi-line strings, lists and function calls are reassembled before being
-  passed to these, but note that function and class *definitions* are still a
-  series of separate statements. IPython does not use any of these by default.
-
-An InteractiveShell instance actually has two
-:class:`~IPython.core.inputsplitter.IPythonInputSplitter` instances, as the
-attributes :attr:`~IPython.core.interactiveshell.InteractiveShell.input_splitter`,
-to tell when a block of input is complete, and
-:attr:`~IPython.core.interactiveshell.InteractiveShell.input_transformer_manager`,
-to transform complete cells. If you add a transformer, you should make sure that
-it gets added to both, e.g.::
-
-    ip.input_splitter.logical_line_transforms.append(my_transformer())
-    ip.input_transformer_manager.logical_line_transforms.append(my_transformer())
+.. versionchanged:: 7.0
 
-These transformers may raise :exc:`SyntaxError` if the input code is invalid, but
-in most cases it is clearer to pass unrecognised code through unmodified and let
-Python's own parser decide whether it is valid.
-
-.. versionchanged:: 2.0
-
-   Added the option to raise :exc:`SyntaxError`.
-
-Stateless transformations
--------------------------
-
-The simplest kind of transformations work one line at a time. Write a function
-which takes a line and returns a line, and decorate it with
-:meth:`StatelessInputTransformer.wrap`::
-
-    @StatelessInputTransformer.wrap
-    def my_special_commands(line):
-        if line.startswith("¬"):
-            return "specialcommand(" + repr(line) + ")"
-        return line
-
-The decorator returns a factory function which will produce instances of
-:class:`~IPython.core.inputtransformer.StatelessInputTransformer` using your
-function.
-
-Transforming a full block
--------------------------
-
-.. warning::
+   The API for string and token-based transformations has been completely
+   redesigned. Any third party code extending input transformation will need to
+   be rewritten. The new API is, hopefully, simpler.
 
-    Transforming a full block at once will break the automatic detection of
-    whether a block of code is complete in interfaces relying on this
-    functionality, such as terminal IPython. You will need to use a
-    shortcut to force-execute your cells.
+String based transformations are functions which accept a list of strings:
+each string is a single line of the input cell, including its line ending.
+The transformation function should return output in the same structure.
 
-Transforming a full block of python code is possible by implementing a
-:class:`~IPython.core.inputtransformer.Inputtransformer` and overwriting the
-``push`` and ``reset`` methods. The reset method should send the full block of
-transformed text. As an example a transformer the reversed the lines from last
-to first.
+These transformations are in two groups, accessible as attributes of
+the :class:`~IPython.core.interactiveshell.InteractiveShell` instance.
+Each group is a list of transformation functions.
 
-    from IPython.core.inputtransformer import InputTransformer
+* ``input_transformers_cleanup`` run first on input, to do things like stripping
+  prompts and leading indents from copied code. It may not be possible at this
+  stage to parse the input as valid Python code.
+* Then IPython runs its own transformations to handle its special syntax, like
+  ``%magics`` and ``!system`` commands. This part does not expose extension
+  points.
+* ``input_transformers_post`` run as the last step, to do things like converting
+  float literals into decimal objects. These may attempt to parse the input as
+  Python code.  
 
-    class ReverseLineTransformer(InputTransformer):
-
-        def __init__(self):
-            self.acc = []
-
-        def push(self, line):
-            self.acc.append(line)
-            return None
-
-        def reset(self):
-            ret = '\n'.join(self.acc[::-1])
-            self.acc = []
-            return ret
-
-
-Coroutine transformers
-----------------------
-
-More advanced transformers can be written as coroutines. The coroutine will be
-sent each line in turn, followed by ``None`` to reset it. It can yield lines, or
-``None`` if it is accumulating text to yield at a later point. When reset, it
-should give up any code it has accumulated.
-
-You may use :meth:`CoroutineInputTransformer.wrap` to simplify the creation of
-such a transformer.
+These transformers may raise :exc:`SyntaxError` if the input code is invalid, but
+in most cases it is clearer to pass unrecognised code through unmodified and let
+Python's own parser decide whether it is valid.
 
-Here is a simple :class:`CoroutineInputTransformer` that can be thought of
-being the identity::
+For example, imagine we want to obfuscate our code by reversing each line, so
+we'd write ``)5(f =+ a`` instead of ``a += f(5)``. Here's how we could swap it
+back the right way before IPython tries to run it::
 
-    from IPython.core.inputtransformer import CoroutineInputTransformer
+    def reverse_line_chars(lines):
+        new_lines = []
+        for line in lines:
+            chars = line[:-1]  # the newline needs to stay at the end
+            new_lines.append(chars[::-1] + '\n')
+        return new_lines
 
-    @CoroutineInputTransformer.wrap
-    def noop():
-        line = ''
-        while True:
-            line = (yield line)
+To start using this::
 
     ip = get_ipython()
-
-    ip.input_splitter.logical_line_transforms.append(noop())
-    ip.input_transformer_manager.logical_line_transforms.append(noop())
-
-This code in IPython strips a constant amount of leading indentation from each
-line in a cell::
-
-    from IPython.core.inputtransformer import CoroutineInputTransformer
-
-    @CoroutineInputTransformer.wrap
-    def leading_indent():
-        """Remove leading indentation.
-        
-        If the first line starts with a spaces or tabs, the same whitespace will be
-        removed from each following line until it is reset.
-        """
-        space_re = re.compile(r'^[ \t]+')
-        line = ''
-        while True:
-            line = (yield line)
-            
-            if line is None:
-                continue
-            
-            m = space_re.match(line)
-            if m:
-                space = m.group(0)
-                while line is not None:
-                    if line.startswith(space):
-                        line = line[len(space):]
-                    line = (yield line)
-            else:
-                # No leading spaces - wait for reset
-                while line is not None:
-                    line = (yield line)
-
-
-Token-based transformers
-------------------------
-
-There is an experimental framework that takes care of tokenizing and
-untokenizing lines of code. Define a function that accepts a list of tokens, and
-returns an iterable of output tokens, and decorate it with
-:meth:`TokenInputTransformer.wrap`. These should only be used in
-``python_line_transforms``.
+    ip.input_transformers_cleanup.append(reverse_line_chars)
 
 AST transformations
 ===================