Backport PR #13625: Remove set-next input when triggering help.

IPython/core/inputtransformer.py

@@ -1,536 +1,534 b''
 """DEPRECATED: Input transformer classes to support IPython special syntax.
 
 This module was deprecated in IPython 7.0, in favour of inputtransformer2.
 
 This includes the machinery to recognise and transform ``%magic`` commands,
 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
 """
 import abc
 import functools
 import re
 import tokenize
 from tokenize import generate_tokens, untokenize, TokenError
 from io import StringIO
 
 from IPython.core.splitinput import LineInfo
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # The escape sequences that define the syntax transformations IPython will
 # apply to user input.  These can NOT be just changed here: many regular
 # expressions and other parts of the code may use their hardcoded values, and
 # for all intents and purposes they constitute the 'IPython syntax', so they
 # should be considered fixed.
 
 ESC_SHELL  = '!'     # Send line to underlying system shell
 ESC_SH_CAP = '!!'    # Send line to system shell and capture output
 ESC_HELP   = '?'     # Find information about object
 ESC_HELP2  = '??'    # Find extra-detailed information about object
 ESC_MAGIC  = '%'     # Call magic function
 ESC_MAGIC2 = '%%'    # Call cell-magic function
 ESC_QUOTE  = ','     # Split args on whitespace, quote each as string and call
 ESC_QUOTE2 = ';'     # Quote all args as a single string, call
 ESC_PAREN  = '/'     # Call first argument with rest of line as arguments
 
 ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\
                  ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\
                  ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ]
 
 
 class InputTransformer(metaclass=abc.ABCMeta):
     """Abstract base class for line-based input transformers."""
     
     @abc.abstractmethod
     def push(self, line):
         """Send a line of input to the transformer, returning the transformed
         input or None if the transformer is waiting for more input.
         
         Must be overridden by subclasses.
 
         Implementations may raise ``SyntaxError`` if the input is invalid. No
         other exceptions may be raised.
         """
         pass
     
     @abc.abstractmethod
     def reset(self):
         """Return, transformed any lines that the transformer has accumulated,
         and reset its internal state.
         
         Must be overridden by subclasses.
         """
         pass
     
     @classmethod
     def wrap(cls, func):
         """Can be used by subclasses as a decorator, to return a factory that
         will allow instantiation with the decorated object.
         """
         @functools.wraps(func)
         def transformer_factory(**kwargs):
             return cls(func, **kwargs)
         
         return transformer_factory
 
 class StatelessInputTransformer(InputTransformer):
     """Wrapper for a stateless input transformer implemented as a function."""
     def __init__(self, func):
         self.func = func
     
     def __repr__(self):
         return "StatelessInputTransformer(func={0!r})".format(self.func)
     
     def push(self, line):
         """Send a line of input to the transformer, returning the
         transformed input."""
         return self.func(line)
     
     def reset(self):
         """No-op - exists for compatibility."""
         pass
 
 class CoroutineInputTransformer(InputTransformer):
     """Wrapper for an input transformer implemented as a coroutine."""
     def __init__(self, coro, **kwargs):
         # Prime it
         self.coro = coro(**kwargs)
         next(self.coro)
     
     def __repr__(self):
         return "CoroutineInputTransformer(coro={0!r})".format(self.coro)
     
     def push(self, line):
         """Send a line of input to the transformer, returning the
         transformed input or None if the transformer is waiting for more
         input.
         """
         return self.coro.send(line)
     
     def reset(self):
         """Return, transformed any lines that the transformer has
         accumulated, and reset its internal state.
         """
         return self.coro.send(None)
 
 class TokenInputTransformer(InputTransformer):
     """Wrapper for a token-based input transformer.
     
     func should accept a list of tokens (5-tuples, see tokenize docs), and
     return an iterable which can be passed to tokenize.untokenize().
     """
     def __init__(self, func):
         self.func = func
         self.buf = []
         self.reset_tokenizer()
 
     def reset_tokenizer(self):
         it = iter(self.buf)
         self.tokenizer = generate_tokens(it.__next__)
 
     def push(self, line):
         self.buf.append(line + '\n')
         if all(l.isspace() for l in self.buf):
             return self.reset()
 
         tokens = []
         stop_at_NL = False
         try:
             for intok in self.tokenizer:
                 tokens.append(intok)
                 t = intok[0]
                 if t == tokenize.NEWLINE or (stop_at_NL and t == tokenize.NL):
                     # Stop before we try to pull a line we don't have yet
                     break
                 elif t == tokenize.ERRORTOKEN:
                     stop_at_NL = True
         except TokenError:
             # Multi-line statement - stop and try again with the next line
             self.reset_tokenizer()
             return None
         
         return self.output(tokens)
     
     def output(self, tokens):
         self.buf.clear()
         self.reset_tokenizer()
         return untokenize(self.func(tokens)).rstrip('\n')
     
     def reset(self):
         l = ''.join(self.buf)
         self.buf.clear()
         self.reset_tokenizer()
         if l:
             return l.rstrip('\n')
 
 class assemble_python_lines(TokenInputTransformer):
     def __init__(self):
         super(assemble_python_lines, self).__init__(None)
     
     def output(self, tokens):
         return self.reset()
 
 @CoroutineInputTransformer.wrap
 def assemble_logical_lines():
     r"""Join lines following explicit line continuations (\)"""
     line = ''
     while True:
         line = (yield line)
         if not line or line.isspace():
             continue
         
         parts = []
         while line is not None:
             if line.endswith('\\') and (not has_comment(line)):
                 parts.append(line[:-1])
                 line = (yield None) # Get another line
             else:
                 parts.append(line)
                 break
         
         # Output
         line = ''.join(parts)
 
 # Utilities
-def _make_help_call(target, esc, lspace, next_input=None):
+def _make_help_call(target, esc, lspace):
     """Prepares a pinfo(2)/psearch call from a target name and the escape
     (i.e. ? or ??)"""
     method  = 'pinfo2' if esc == '??' \
                 else 'psearch' if '*' in target \
                 else 'pinfo'
     arg = " ".join([method, target])
     #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
     t_magic_name, _, t_magic_arg_s = arg.partition(' ')
     t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
-    if next_input is None:
-        return '%sget_ipython().run_line_magic(%r, %r)' % (lspace, t_magic_name, t_magic_arg_s)
-    else:
-        return '%sget_ipython().set_next_input(%r);get_ipython().run_line_magic(%r, %r)' % \
-           (lspace, next_input, t_magic_name, t_magic_arg_s)
-    
+    return "%sget_ipython().run_line_magic(%r, %r)" % (
+        lspace,
+        t_magic_name,
+        t_magic_arg_s,
+    )
+
+
 # These define the transformations for the different escape characters.
 def _tr_system(line_info):
     "Translate lines escaped with: !"
     cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
     return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
 
 def _tr_system2(line_info):
     "Translate lines escaped with: !!"
     cmd = line_info.line.lstrip()[2:]
     return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
 
 def _tr_help(line_info):
     "Translate lines escaped with: ?/??"
     # A naked help line should just fire the intro help screen
     if not line_info.line[1:]:
         return 'get_ipython().show_usage()'
 
     return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
 
 def _tr_magic(line_info):
     "Translate lines escaped with: %"
     tpl = '%sget_ipython().run_line_magic(%r, %r)'
     if line_info.line.startswith(ESC_MAGIC2):
         return line_info.line
     cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
     #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
     t_magic_name, _, t_magic_arg_s = cmd.partition(' ')
     t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
     return tpl % (line_info.pre, t_magic_name, t_magic_arg_s)
 
 def _tr_quote(line_info):
     "Translate lines escaped with: ,"
     return '%s%s("%s")' % (line_info.pre, line_info.ifun,
                          '", "'.join(line_info.the_rest.split()) )
 
 def _tr_quote2(line_info):
     "Translate lines escaped with: ;"
     return '%s%s("%s")' % (line_info.pre, line_info.ifun,
                            line_info.the_rest)
 
 def _tr_paren(line_info):
     "Translate lines escaped with: /"
     return '%s%s(%s)' % (line_info.pre, line_info.ifun,
                          ", ".join(line_info.the_rest.split()))
 
 tr = { ESC_SHELL  : _tr_system,
        ESC_SH_CAP : _tr_system2,
        ESC_HELP   : _tr_help,
        ESC_HELP2  : _tr_help,
        ESC_MAGIC  : _tr_magic,
        ESC_QUOTE  : _tr_quote,
        ESC_QUOTE2 : _tr_quote2,
        ESC_PAREN  : _tr_paren }
 
 @StatelessInputTransformer.wrap
 def escaped_commands(line):
     """Transform escaped commands - %magic, !system, ?help + various autocalls.
     """
     if not line or line.isspace():
         return line
     lineinf = LineInfo(line)
     if lineinf.esc not in tr:
         return line
     
     return tr[lineinf.esc](lineinf)
 
 _initial_space_re = re.compile(r'\s*')
 
 _help_end_re = re.compile(r"""(%{0,2}
                               (?!\d)[\w*]+            # Variable name
                               (\.(?!\d)[\w*]+)*       # .etc.etc
                               )
                               (\?\??)$                # ? or ??
                               """,
                               re.VERBOSE)
 
 # Extra pseudotokens for multiline strings and data structures
 _MULTILINE_STRING = object()
 _MULTILINE_STRUCTURE = object()
 
 def _line_tokens(line):
     """Helper for has_comment and ends_in_comment_or_string."""
     readline = StringIO(line).readline
     toktypes = set()
     try:
         for t in generate_tokens(readline):
             toktypes.add(t[0])
     except TokenError as e:
         # There are only two cases where a TokenError is raised.
         if 'multi-line string' in e.args[0]:
             toktypes.add(_MULTILINE_STRING)
         else:
             toktypes.add(_MULTILINE_STRUCTURE)
     return toktypes
 
 def has_comment(src):
     """Indicate whether an input line has (i.e. ends in, or is) a comment.
 
     This uses tokenize, so it can distinguish comments from # inside strings.
 
     Parameters
     ----------
     src : string
       A single line input string.
 
     Returns
     -------
     comment : bool
         True if source has a comment.
     """
     return (tokenize.COMMENT in _line_tokens(src))
 
 def ends_in_comment_or_string(src):
     """Indicates whether or not an input line ends in a comment or within
     a multiline string.
     
     Parameters
     ----------
     src : string
       A single line input string.
 
     Returns
     -------
     comment : bool
         True if source ends in a comment or multiline string.
     """
     toktypes = _line_tokens(src)
     return (tokenize.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes)
         
 
 @StatelessInputTransformer.wrap
 def help_end(line):
     """Translate lines with ?/?? at the end"""
     m = _help_end_re.search(line)
     if m is None or ends_in_comment_or_string(line):
         return line
     target = m.group(1)
     esc = m.group(3)
     lspace = _initial_space_re.match(line).group(0)
 
-    # If we're mid-command, put it back on the next prompt for the user.
-    next_input = line.rstrip('?') if line.strip() != m.group(0) else None
-
-    return _make_help_call(target, esc, lspace, next_input)
+    return _make_help_call(target, esc, lspace)
 
 
 @CoroutineInputTransformer.wrap
 def cellmagic(end_on_blank_line=False):
     """Captures & transforms cell magics.
     
     After a cell magic is started, this stores up any lines it gets until it is
     reset (sent None).
     """
     tpl = 'get_ipython().run_cell_magic(%r, %r, %r)'
     cellmagic_help_re = re.compile(r'%%\w+\?')
     line = ''
     while True:
         line = (yield line)
         # consume leading empty lines
         while not line:
             line = (yield line)
         
         if not line.startswith(ESC_MAGIC2):
             # This isn't a cell magic, idle waiting for reset then start over
             while line is not None:
                 line = (yield line)
             continue
         
         if cellmagic_help_re.match(line):
             # This case will be handled by help_end
             continue
         
         first = line
         body = []
         line = (yield None)
         while (line is not None) and \
                                 ((line.strip() != '') or not end_on_blank_line):
             body.append(line)
             line = (yield None)
         
         # Output
         magic_name, _, first = first.partition(' ')
         magic_name = magic_name.lstrip(ESC_MAGIC2)
         line = tpl % (magic_name, first, u'\n'.join(body))
 
 
 def _strip_prompts(prompt_re, initial_re=None, turnoff_re=None):
     """Remove matching input prompts from a block of input.
     
     Parameters
     ----------
     prompt_re : regular expression
         A regular expression matching any input prompt (including continuation)
     initial_re : regular expression, optional
         A regular expression matching only the initial prompt, but not continuation.
         If no initial expression is given, prompt_re will be used everywhere.
         Used mainly for plain Python prompts, where the continuation prompt
         ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
     
     If initial_re and prompt_re differ,
     only initial_re will be tested against the first line.
     If any prompt is found on the first two lines,
     prompts will be stripped from the rest of the block.
     """
     if initial_re is None:
         initial_re = prompt_re
     line = ''
     while True:
         line = (yield line)
         
         # First line of cell
         if line is None:
             continue
         out, n1 = initial_re.subn('', line, count=1)
         if turnoff_re and not n1:
             if turnoff_re.match(line):
                 # We're in e.g. a cell magic; disable this transformer for
                 # the rest of the cell.
                 while line is not None:
                     line = (yield line)
                 continue
 
         line = (yield out)
         
         if line is None:
             continue
         # check for any prompt on the second line of the cell,
         # because people often copy from just after the first prompt,
         # so we might not see it in the first line.
         out, n2 = prompt_re.subn('', line, count=1)
         line = (yield out)
         
         if n1 or n2:
             # Found a prompt in the first two lines - check for it in
             # the rest of the cell as well.
             while line is not None:
                 line = (yield prompt_re.sub('', line, count=1))
         
         else:
             # Prompts not in input - wait for reset
             while line is not None:
                 line = (yield line)
 
 @CoroutineInputTransformer.wrap
 def classic_prompt():
     """Strip the >>>/... prompts of the Python interactive shell."""
     # FIXME: non-capturing version (?:...) usable?
     prompt_re = re.compile(r'^(>>>|\.\.\.)( |$)')
     initial_re = re.compile(r'^>>>( |$)')
     # Any %magic/!system is IPython syntax, so we needn't look for >>> prompts
     turnoff_re = re.compile(r'^[%!]')
     return _strip_prompts(prompt_re, initial_re, turnoff_re)
 
 @CoroutineInputTransformer.wrap
 def ipy_prompt():
     """Strip IPython's In [1]:/...: prompts."""
     # FIXME: non-capturing version (?:...) usable?
     prompt_re = re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)')
     # Disable prompt stripping inside cell magics
     turnoff_re = re.compile(r'^%%')
     return _strip_prompts(prompt_re, turnoff_re=turnoff_re)
 
 
 @CoroutineInputTransformer.wrap
 def leading_indent():
     """Remove leading indentation.
     
     If the first line starts with a spaces or tabs, the same whitespace will be
     removed from each following line until it is reset.
     """
     space_re = re.compile(r'^[ \t]+')
     line = ''
     while True:
         line = (yield line)
         
         if line is None:
             continue
         
         m = space_re.match(line)
         if m:
             space = m.group(0)
             while line is not None:
                 if line.startswith(space):
                     line = line[len(space):]
                 line = (yield line)
         else:
             # No leading spaces - wait for reset
             while line is not None:
                 line = (yield line)
 
 
 _assign_pat = \
 r'''(?P<lhs>(\s*)
     ([\w\.]+)                # Initial identifier
     (\s*,\s*
         \*?[\w\.]+)*         # Further identifiers for unpacking
     \s*?,?                   # Trailing comma
     )
     \s*=\s*
 '''
 
 assign_system_re = re.compile(r'{}!\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE)
 assign_system_template = '%s = get_ipython().getoutput(%r)'
 @StatelessInputTransformer.wrap
 def assign_from_system(line):
     """Transform assignment from system commands (e.g. files = !ls)"""
     m = assign_system_re.match(line)
     if m is None:
         return line
     
     return assign_system_template % m.group('lhs', 'cmd')
 
 assign_magic_re = re.compile(r'{}%\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE)
 assign_magic_template = '%s = get_ipython().run_line_magic(%r, %r)'
 @StatelessInputTransformer.wrap
 def assign_from_magic(line):
     """Transform assignment from magic commands (e.g. a = %who_ls)"""
     m = assign_magic_re.match(line)
     if m is None:
         return line
     #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
     m_lhs, m_cmd = m.group('lhs', 'cmd')
     t_magic_name, _, t_magic_arg_s = m_cmd.partition(' ')
     t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
     return assign_magic_template % (m_lhs, t_magic_name, t_magic_arg_s)

IPython/core/inputtransformer2.py

@@ -1,752 +1,744 b''
 """Input transformer machinery to support IPython special syntax.
 
 This includes the machinery to recognise and transform ``%magic`` commands,
 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
 
 Added: IPython 7.0. Replaces inputsplitter and inputtransformer which were
 deprecated in 7.0.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import ast
 import sys
 from codeop import CommandCompiler, Compile
 import re
 import tokenize
 from typing import List, Tuple, Union
 import warnings
 
 _indent_re = re.compile(r'^[ \t]+')
 
 def leading_empty_lines(lines):
     """Remove leading empty lines
 
     If the leading lines are empty or contain only whitespace, they will be
     removed.
     """
     if not lines:
         return lines
     for i, line in enumerate(lines):
         if line and not line.isspace():
             return lines[i:]
     return lines
 
 def leading_indent(lines):
     """Remove leading indentation.
 
     If the first line starts with a spaces or tabs, the same whitespace will be
     removed from each following line in the cell.
     """
     if not lines:
         return lines
     m = _indent_re.match(lines[0])
     if not m:
         return lines
     space = m.group(0)
     n = len(space)
     return [l[n:] if l.startswith(space) else l
             for l in lines]
 
 class PromptStripper:
     """Remove matching input prompts from a block of input.
 
     Parameters
     ----------
     prompt_re : regular expression
         A regular expression matching any input prompt (including continuation,
         e.g. ``...``)
     initial_re : regular expression, optional
         A regular expression matching only the initial prompt, but not continuation.
         If no initial expression is given, prompt_re will be used everywhere.
         Used mainly for plain Python prompts (``>>>``), where the continuation prompt
         ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
 
     Notes
     -----
 
     If initial_re and prompt_re differ,
     only initial_re will be tested against the first line.
     If any prompt is found on the first two lines,
     prompts will be stripped from the rest of the block.
     """
     def __init__(self, prompt_re, initial_re=None):
         self.prompt_re = prompt_re
         self.initial_re = initial_re or prompt_re
 
     def _strip(self, lines):
         return [self.prompt_re.sub('', l, count=1) for l in lines]
 
     def __call__(self, lines):
         if not lines:
             return lines
         if self.initial_re.match(lines[0]) or \
                 (len(lines) > 1 and self.prompt_re.match(lines[1])):
             return self._strip(lines)
         return lines
 
 classic_prompt = PromptStripper(
     prompt_re=re.compile(r'^(>>>|\.\.\.)( |$)'),
     initial_re=re.compile(r'^>>>( |$)')
 )
 
 ipython_prompt = PromptStripper(re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)'))
 
 def cell_magic(lines):
     if not lines or not lines[0].startswith('%%'):
         return lines
     if re.match(r'%%\w+\?', lines[0]):
         # This case will be handled by help_end
         return lines
     magic_name, _, first_line = lines[0][2:].rstrip().partition(' ')
     body = ''.join(lines[1:])
     return ['get_ipython().run_cell_magic(%r, %r, %r)\n'
             % (magic_name, first_line, body)]
 
 
 def _find_assign_op(token_line) -> Union[int, None]:
     """Get the index of the first assignment in the line ('=' not inside brackets)
 
     Note: We don't try to support multiple special assignment (a = b = %foo)
     """
     paren_level = 0
     for i, ti in enumerate(token_line):
         s = ti.string
         if s == '=' and paren_level == 0:
             return i
         if s in {'(','[','{'}:
             paren_level += 1
         elif s in {')', ']', '}'}:
             if paren_level > 0:
                 paren_level -= 1
 
 def find_end_of_continued_line(lines, start_line: int):
     """Find the last line of a line explicitly extended using backslashes.
 
     Uses 0-indexed line numbers.
     """
     end_line = start_line
     while lines[end_line].endswith('\\\n'):
         end_line += 1
         if end_line >= len(lines):
             break
     return end_line
 
 def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):
     r"""Assemble a single line from multiple continued line pieces
 
     Continued lines are lines ending in ``\``, and the line following the last
     ``\`` in the block.
 
     For example, this code continues over multiple lines::
 
         if (assign_ix is not None) \
              and (len(line) >= assign_ix + 2) \
              and (line[assign_ix+1].string == '%') \
              and (line[assign_ix+2].type == tokenize.NAME):
 
     This statement contains four continued line pieces.
     Assembling these pieces into a single line would give::
 
         if (assign_ix is not None) and (len(line) >= assign_ix + 2) and (line[...
 
     This uses 0-indexed line numbers. *start* is (lineno, colno).
 
     Used to allow ``%magic`` and ``!system`` commands to be continued over
     multiple lines.
     """
     parts = [lines[start[0]][start[1]:]] + lines[start[0]+1:end_line+1]
     return ' '.join([p.rstrip()[:-1] for p in parts[:-1]]  # Strip backslash+newline
                     + [parts[-1].rstrip()])         # Strip newline from last line
 
 class TokenTransformBase:
     """Base class for transformations which examine tokens.
 
     Special syntax should not be transformed when it occurs inside strings or
     comments. This is hard to reliably avoid with regexes. The solution is to
     tokenise the code as Python, and recognise the special syntax in the tokens.
 
     IPython's special syntax is not valid Python syntax, so tokenising may go
     wrong after the special syntax starts. These classes therefore find and
     transform *one* instance of special syntax at a time into regular Python
     syntax. After each transformation, tokens are regenerated to find the next
     piece of special syntax.
 
     Subclasses need to implement one class method (find)
     and one regular method (transform).
 
     The priority attribute can select which transformation to apply if multiple
     transformers match in the same place. Lower numbers have higher priority.
     This allows "%magic?" to be turned into a help call rather than a magic call.
     """
     # Lower numbers -> higher priority (for matches in the same location)
     priority = 10
 
     def sortby(self):
         return self.start_line, self.start_col, self.priority
 
     def __init__(self, start):
         self.start_line = start[0] - 1   # Shift from 1-index to 0-index
         self.start_col = start[1]
 
     @classmethod
     def find(cls, tokens_by_line):
         """Find one instance of special syntax in the provided tokens.
 
         Tokens are grouped into logical lines for convenience,
         so it is easy to e.g. look at the first token of each line.
         *tokens_by_line* is a list of lists of tokenize.TokenInfo objects.
 
         This should return an instance of its class, pointing to the start
         position it has found, or None if it found no match.
         """
         raise NotImplementedError
 
     def transform(self, lines: List[str]):
         """Transform one instance of special syntax found by ``find()``
 
         Takes a list of strings representing physical lines,
         returns a similar list of transformed lines.
         """
         raise NotImplementedError
 
 class MagicAssign(TokenTransformBase):
     """Transformer for assignments from magics (a = %foo)"""
     @classmethod
     def find(cls, tokens_by_line):
         """Find the first magic assignment (a = %foo) in the cell.
         """
         for line in tokens_by_line:
             assign_ix = _find_assign_op(line)
             if (assign_ix is not None) \
                     and (len(line) >= assign_ix + 2) \
                     and (line[assign_ix+1].string == '%') \
                     and (line[assign_ix+2].type == tokenize.NAME):
                 return cls(line[assign_ix+1].start)
 
     def transform(self, lines: List[str]):
         """Transform a magic assignment found by the ``find()`` classmethod.
         """
         start_line, start_col = self.start_line, self.start_col
         lhs = lines[start_line][:start_col]
         end_line = find_end_of_continued_line(lines, start_line)
         rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
         assert rhs.startswith('%'), rhs
         magic_name, _, args = rhs[1:].partition(' ')
 
         lines_before = lines[:start_line]
         call = "get_ipython().run_line_magic({!r}, {!r})".format(magic_name, args)
         new_line = lhs + call + '\n'
         lines_after = lines[end_line+1:]
 
         return lines_before + [new_line] + lines_after
 
 
 class SystemAssign(TokenTransformBase):
     """Transformer for assignments from system commands (a = !foo)"""
     @classmethod
     def find(cls, tokens_by_line):
         """Find the first system assignment (a = !foo) in the cell.
         """
         for line in tokens_by_line:
             assign_ix = _find_assign_op(line)
             if (assign_ix is not None) \
                     and not line[assign_ix].line.strip().startswith('=') \
                     and (len(line) >= assign_ix + 2) \
                     and (line[assign_ix + 1].type == tokenize.ERRORTOKEN):
                 ix = assign_ix + 1
 
                 while ix < len(line) and line[ix].type == tokenize.ERRORTOKEN:
                     if line[ix].string == '!':
                         return cls(line[ix].start)
                     elif not line[ix].string.isspace():
                         break
                     ix += 1
 
     def transform(self, lines: List[str]):
         """Transform a system assignment found by the ``find()`` classmethod.
         """
         start_line, start_col = self.start_line, self.start_col
 
         lhs = lines[start_line][:start_col]
         end_line = find_end_of_continued_line(lines, start_line)
         rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
         assert rhs.startswith('!'), rhs
         cmd = rhs[1:]
 
         lines_before = lines[:start_line]
         call = "get_ipython().getoutput({!r})".format(cmd)
         new_line = lhs + call + '\n'
         lines_after = lines[end_line + 1:]
 
         return lines_before + [new_line] + lines_after
 
 # The escape sequences that define the syntax transformations IPython will
 # apply to user input.  These can NOT be just changed here: many regular
 # expressions and other parts of the code may use their hardcoded values, and
 # for all intents and purposes they constitute the 'IPython syntax', so they
 # should be considered fixed.
 
 ESC_SHELL  = '!'     # Send line to underlying system shell
 ESC_SH_CAP = '!!'    # Send line to system shell and capture output
 ESC_HELP   = '?'     # Find information about object
 ESC_HELP2  = '??'    # Find extra-detailed information about object
 ESC_MAGIC  = '%'     # Call magic function
 ESC_MAGIC2 = '%%'    # Call cell-magic function
 ESC_QUOTE  = ','     # Split args on whitespace, quote each as string and call
 ESC_QUOTE2 = ';'     # Quote all args as a single string, call
 ESC_PAREN  = '/'     # Call first argument with rest of line as arguments
 
 ESCAPE_SINGLES = {'!', '?', '%', ',', ';', '/'}
 ESCAPE_DOUBLES = {'!!', '??'}  # %% (cell magic) is handled separately
 
-def _make_help_call(target, esc, next_input=None):
+def _make_help_call(target, esc):
     """Prepares a pinfo(2)/psearch call from a target name and the escape
     (i.e. ? or ??)"""
     method  = 'pinfo2' if esc == '??' \
                 else 'psearch' if '*' in target \
                 else 'pinfo'
     arg = " ".join([method, target])
     #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
     t_magic_name, _, t_magic_arg_s = arg.partition(' ')
     t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
-    if next_input is None:
-        return 'get_ipython().run_line_magic(%r, %r)' % (t_magic_name, t_magic_arg_s)
-    else:
-        return 'get_ipython().set_next_input(%r);get_ipython().run_line_magic(%r, %r)' % \
-           (next_input, t_magic_name, t_magic_arg_s)
+    return "get_ipython().run_line_magic(%r, %r)" % (t_magic_name, t_magic_arg_s)
+
 
 def _tr_help(content):
     """Translate lines escaped with: ?
 
     A naked help line should fire the intro help screen (shell.show_usage())
     """
     if not content:
         return 'get_ipython().show_usage()'
 
     return _make_help_call(content, '?')
 
 def _tr_help2(content):
     """Translate lines escaped with: ??
 
     A naked help line should fire the intro help screen (shell.show_usage())
     """
     if not content:
         return 'get_ipython().show_usage()'
 
     return _make_help_call(content, '??')
 
 def _tr_magic(content):
     "Translate lines escaped with a percent sign: %"
     name, _, args = content.partition(' ')
     return 'get_ipython().run_line_magic(%r, %r)' % (name, args)
 
 def _tr_quote(content):
     "Translate lines escaped with a comma: ,"
     name, _, args = content.partition(' ')
     return '%s("%s")' % (name, '", "'.join(args.split()) )
 
 def _tr_quote2(content):
     "Translate lines escaped with a semicolon: ;"
     name, _, args = content.partition(' ')
     return '%s("%s")' % (name, args)
 
 def _tr_paren(content):
     "Translate lines escaped with a slash: /"
     name, _, args = content.partition(' ')
     return '%s(%s)' % (name, ", ".join(args.split()))
 
 tr = { ESC_SHELL  : 'get_ipython().system({!r})'.format,
        ESC_SH_CAP : 'get_ipython().getoutput({!r})'.format,
        ESC_HELP   : _tr_help,
        ESC_HELP2  : _tr_help2,
        ESC_MAGIC  : _tr_magic,
        ESC_QUOTE  : _tr_quote,
        ESC_QUOTE2 : _tr_quote2,
        ESC_PAREN  : _tr_paren }
 
 class EscapedCommand(TokenTransformBase):
     """Transformer for escaped commands like %foo, !foo, or /foo"""
     @classmethod
     def find(cls, tokens_by_line):
         """Find the first escaped command (%foo, !foo, etc.) in the cell.
         """
         for line in tokens_by_line:
             if not line:
                 continue
             ix = 0
             ll = len(line)
             while ll > ix and line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
                 ix += 1
             if ix >= ll:
                 continue
             if line[ix].string in ESCAPE_SINGLES:
                 return cls(line[ix].start)
 
     def transform(self, lines):
         """Transform an escaped line found by the ``find()`` classmethod.
         """
         start_line, start_col = self.start_line, self.start_col
 
         indent = lines[start_line][:start_col]
         end_line = find_end_of_continued_line(lines, start_line)
         line = assemble_continued_line(lines, (start_line, start_col), end_line)
 
         if len(line) > 1 and line[:2] in ESCAPE_DOUBLES:
             escape, content = line[:2], line[2:]
         else:
             escape, content = line[:1], line[1:]
 
         if escape in tr:
             call = tr[escape](content)
         else:
             call = ''
 
         lines_before = lines[:start_line]
         new_line = indent + call + '\n'
         lines_after = lines[end_line + 1:]
 
         return lines_before + [new_line] + lines_after
 
 _help_end_re = re.compile(r"""(%{0,2}
                               (?!\d)[\w*]+            # Variable name
                               (\.(?!\d)[\w*]+)*       # .etc.etc
                               )
                               (\?\??)$                # ? or ??
                               """,
                               re.VERBOSE)
 
 class HelpEnd(TokenTransformBase):
     """Transformer for help syntax: obj? and obj??"""
     # This needs to be higher priority (lower number) than EscapedCommand so
     # that inspecting magics (%foo?) works.
     priority = 5
 
     def __init__(self, start, q_locn):
         super().__init__(start)
         self.q_line = q_locn[0] - 1  # Shift from 1-indexed to 0-indexed
         self.q_col = q_locn[1]
 
     @classmethod
     def find(cls, tokens_by_line):
         """Find the first help command (foo?) in the cell.
         """
         for line in tokens_by_line:
             # Last token is NEWLINE; look at last but one
             if len(line) > 2 and line[-2].string == '?':
                 # Find the first token that's not INDENT/DEDENT
                 ix = 0
                 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
                     ix += 1
                 return cls(line[ix].start, line[-2].start)
 
     def transform(self, lines):
         """Transform a help command found by the ``find()`` classmethod.
         """
         piece = ''.join(lines[self.start_line:self.q_line+1])
         indent, content = piece[:self.start_col], piece[self.start_col:]
         lines_before = lines[:self.start_line]
         lines_after = lines[self.q_line + 1:]
 
         m = _help_end_re.search(content)
         if not m:
             raise SyntaxError(content)
         assert m is not None, content
         target = m.group(1)
         esc = m.group(3)
 
-        # If we're mid-command, put it back on the next prompt for the user.
-        next_input = None
-        if (not lines_before) and (not lines_after) \
-                and content.strip() != m.group(0):
-            next_input = content.rstrip('?\n')
 
-        call = _make_help_call(target, esc, next_input=next_input)
+        call = _make_help_call(target, esc)
         new_line = indent + call + '\n'
 
         return lines_before + [new_line] + lines_after
 
 def make_tokens_by_line(lines:List[str]):
     """Tokenize a series of lines and group tokens by line.
 
     The tokens for a multiline Python string or expression are grouped as one
     line. All lines except the last lines should keep their line ending ('\\n',
     '\\r\\n') for this to properly work. Use `.splitlines(keeplineending=True)`
     for example when passing block of text to this function.
 
     """
     # NL tokens are used inside multiline expressions, but also after blank
     # lines or comments. This is intentional - see https://bugs.python.org/issue17061
     # We want to group the former case together but split the latter, so we
     # track parentheses level, similar to the internals of tokenize.
     NEWLINE, NL = tokenize.NEWLINE, tokenize.NL
     tokens_by_line = [[]]
     if len(lines) > 1 and not lines[0].endswith(('\n', '\r', '\r\n', '\x0b', '\x0c')):
         warnings.warn("`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified")
     parenlev = 0
     try:
         for token in tokenize.generate_tokens(iter(lines).__next__):
             tokens_by_line[-1].append(token)
             if (token.type == NEWLINE) \
                     or ((token.type == NL) and (parenlev <= 0)):
                 tokens_by_line.append([])
             elif token.string in {'(', '[', '{'}:
                 parenlev += 1
             elif token.string in {')', ']', '}'}:
                 if parenlev > 0:
                     parenlev -= 1
     except tokenize.TokenError:
         # Input ended in a multiline string or expression. That's OK for us.
         pass
 
 
     if not tokens_by_line[-1]:
         tokens_by_line.pop()
 
 
     return tokens_by_line
 
 def show_linewise_tokens(s: str):
     """For investigation and debugging"""
     if not s.endswith('\n'):
         s += '\n'
     lines = s.splitlines(keepends=True)
     for line in make_tokens_by_line(lines):
         print("Line -------")
         for tokinfo in line:
             print(" ", tokinfo)
 
 # Arbitrary limit to prevent getting stuck in infinite loops
 TRANSFORM_LOOP_LIMIT = 500
 
 class TransformerManager:
     """Applies various transformations to a cell or code block.
 
     The key methods for external use are ``transform_cell()``
     and ``check_complete()``.
     """
     def __init__(self):
         self.cleanup_transforms = [
             leading_empty_lines,
             leading_indent,
             classic_prompt,
             ipython_prompt,
         ]
         self.line_transforms = [
             cell_magic,
         ]
         self.token_transformers = [
             MagicAssign,
             SystemAssign,
             EscapedCommand,
             HelpEnd,
         ]
 
     def do_one_token_transform(self, lines):
         """Find and run the transform earliest in the code.
 
         Returns (changed, lines).
 
         This method is called repeatedly until changed is False, indicating
         that all available transformations are complete.
 
         The tokens following IPython special syntax might not be valid, so
         the transformed code is retokenised every time to identify the next
         piece of special syntax. Hopefully long code cells are mostly valid
         Python, not using lots of IPython special syntax, so this shouldn't be
         a performance issue.
         """
         tokens_by_line = make_tokens_by_line(lines)
         candidates = []
         for transformer_cls in self.token_transformers:
             transformer = transformer_cls.find(tokens_by_line)
             if transformer:
                 candidates.append(transformer)
 
         if not candidates:
             # Nothing to transform
             return False, lines
         ordered_transformers = sorted(candidates, key=TokenTransformBase.sortby)
         for transformer in ordered_transformers:
             try:
                 return True, transformer.transform(lines)
             except SyntaxError:
                 pass
         return False, lines
 
     def do_token_transforms(self, lines):
         for _ in range(TRANSFORM_LOOP_LIMIT):
             changed, lines = self.do_one_token_transform(lines)
             if not changed:
                 return lines
 
         raise RuntimeError("Input transformation still changing after "
                            "%d iterations. Aborting." % TRANSFORM_LOOP_LIMIT)
 
     def transform_cell(self, cell: str) -> str:
         """Transforms a cell of input code"""
         if not cell.endswith('\n'):
             cell += '\n'  # Ensure the cell has a trailing newline
         lines = cell.splitlines(keepends=True)
         for transform in self.cleanup_transforms + self.line_transforms:
             lines = transform(lines)
 
         lines = self.do_token_transforms(lines)
         return ''.join(lines)
 
     def check_complete(self, cell: str):
         """Return whether a block of code is ready to execute, or should be continued
 
         Parameters
         ----------
         source : string
           Python input code, which can be multiline.
 
         Returns
         -------
         status : str
           One of 'complete', 'incomplete', or 'invalid' if source is not a
           prefix of valid code.
         indent_spaces : int or None
           The number of spaces by which to indent the next line of code. If
           status is not 'incomplete', this is None.
         """
         # Remember if the lines ends in a new line.
         ends_with_newline = False
         for character in reversed(cell):
             if character == '\n':
                 ends_with_newline = True
                 break
             elif character.strip():
                 break
             else:
                 continue
 
         if not ends_with_newline:
             # Append an newline for consistent tokenization
             # See https://bugs.python.org/issue33899
             cell += '\n'
 
         lines = cell.splitlines(keepends=True)
 
         if not lines:
             return 'complete', None
 
         if lines[-1].endswith('\\'):
             # Explicit backslash continuation
             return 'incomplete', find_last_indent(lines)
 
         try:
             for transform in self.cleanup_transforms:
                 if not getattr(transform, 'has_side_effects', False):
                     lines = transform(lines)
         except SyntaxError:
             return 'invalid', None
 
         if lines[0].startswith('%%'):
             # Special case for cell magics - completion marked by blank line
             if lines[-1].strip():
                 return 'incomplete', find_last_indent(lines)
             else:
                 return 'complete', None
 
         try:
             for transform in self.line_transforms:
                 if not getattr(transform, 'has_side_effects', False):
                     lines = transform(lines)
             lines = self.do_token_transforms(lines)
         except SyntaxError:
             return 'invalid', None
 
         tokens_by_line = make_tokens_by_line(lines)
 
         if not tokens_by_line:
             return 'incomplete', find_last_indent(lines)
 
         if tokens_by_line[-1][-1].type != tokenize.ENDMARKER:
             # We're in a multiline string or expression
             return 'incomplete', find_last_indent(lines)
 
         newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER}
 
         # Pop the last line which only contains DEDENTs and ENDMARKER
         last_token_line = None
         if {t.type for t in tokens_by_line[-1]} in [
             {tokenize.DEDENT, tokenize.ENDMARKER},
             {tokenize.ENDMARKER}
         ] and len(tokens_by_line) > 1:
             last_token_line = tokens_by_line.pop()
 
         while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types:
             tokens_by_line[-1].pop()
 
         if not tokens_by_line[-1]:
             return 'incomplete', find_last_indent(lines)
 
         if tokens_by_line[-1][-1].string == ':':
             # The last line starts a block (e.g. 'if foo:')
             ix = 0
             while tokens_by_line[-1][ix].type in {tokenize.INDENT, tokenize.DEDENT}:
                 ix += 1
 
             indent = tokens_by_line[-1][ix].start[1]
             return 'incomplete', indent + 4
 
         if tokens_by_line[-1][0].line.endswith('\\'):
             return 'incomplete', None
 
         # At this point, our checks think the code is complete (or invalid).
         # We'll use codeop.compile_command to check this with the real parser
         try:
             with warnings.catch_warnings():
                 warnings.simplefilter('error', SyntaxWarning)
                 res = compile_command(''.join(lines), symbol='exec')
         except (SyntaxError, OverflowError, ValueError, TypeError,
                 MemoryError, SyntaxWarning):
             return 'invalid', None
         else:
             if res is None:
                 return 'incomplete', find_last_indent(lines)
 
         if last_token_line and last_token_line[0].type == tokenize.DEDENT:
             if ends_with_newline:
                 return 'complete', None
             return 'incomplete', find_last_indent(lines)
 
         # If there's a blank line at the end, assume we're ready to execute
         if not lines[-1].strip():
             return 'complete', None
 
         return 'complete', None
 
 
 def find_last_indent(lines):
     m = _indent_re.match(lines[-1])
     if not m:
         return 0
     return len(m.group(0).replace('\t', ' '*4))
 
 
 class MaybeAsyncCompile(Compile):
     def __init__(self, extra_flags=0):
         super().__init__()
         self.flags |= extra_flags
 
 
     if sys.version_info < (3,8):
         def __call__(self, *args, **kwds):
             return compile(*args, **kwds)
 
 
 class MaybeAsyncCommandCompiler(CommandCompiler):
     def __init__(self, extra_flags=0):
         self.compiler = MaybeAsyncCompile(extra_flags=extra_flags)
 
 
 if (sys.version_info.major, sys.version_info.minor) >= (3, 8):
     _extra_flags = ast.PyCF_ALLOW_TOP_LEVEL_AWAIT
 else:
     _extra_flags = ast.PyCF_ONLY_AST
 
 compile_command = MaybeAsyncCommandCompiler(extra_flags=_extra_flags)

IPython/core/tests/test_inputtransformer.py

@@ -1,495 +1,499 b''
 import tokenize
 import nose.tools as nt
 
 from IPython.testing import tools as tt
 from IPython.utils import py3compat
 u_fmt = py3compat.u_format
 
 from IPython.core import inputtransformer as ipt
 
 def transform_and_reset(transformer):
     transformer = transformer()
     def transform(inp):
         try:
             return transformer.push(inp)
         finally:
             transformer.reset()
     
     return transform
 
 # Transformer tests
 def transform_checker(tests, transformer, **kwargs):
     """Utility to loop over test inputs"""
     transformer = transformer(**kwargs)
     try:
         for inp, tr in tests:
             if inp is None:
                 out = transformer.reset()
             else:
                 out = transformer.push(inp)
             nt.assert_equal(out, tr)
     finally:
         transformer.reset()
 
 # Data for all the syntax tests in the form of lists of pairs of
 # raw/transformed input.  We store it here as a global dict so that we can use
 # it both within single-function tests and also to validate the behavior of the
 # larger objects
 
-syntax = \
-  dict(assign_system =
-       [(i,py3compat.u_format(o)) for i,o in \
-       [(u'a =! ls', "a = get_ipython().getoutput('ls')"),
-        (u'b = !ls', "b = get_ipython().getoutput('ls')"),
-        (u'c= !ls', "c = get_ipython().getoutput('ls')"),
-        (u'd == !ls', u'd == !ls'), # Invalid syntax, but we leave == alone.
-        ('x=1', 'x=1'), # normal input is unmodified
-        ('    ','    '),  # blank lines are kept intact
-        # Tuple unpacking
-        (u"a, b = !echo 'a\\nb'", u"a, b = get_ipython().getoutput(\"echo 'a\\\\nb'\")"),
-        (u"a,= !echo 'a'", u"a, = get_ipython().getoutput(\"echo 'a'\")"),
-        (u"a, *bc = !echo 'a\\nb\\nc'", u"a, *bc = get_ipython().getoutput(\"echo 'a\\\\nb\\\\nc'\")"),
-        # Tuple unpacking with regular Python expressions, not our syntax.
-        (u"a, b = range(2)", u"a, b = range(2)"),
-        (u"a, = range(1)", u"a, = range(1)"),
-        (u"a, *bc = range(3)", u"a, *bc = range(3)"),
-        ]],
-
-       assign_magic =
-       [(i,py3compat.u_format(o)) for i,o in \
-       [(u'a =% who', "a = get_ipython().run_line_magic('who', '')"),
-        (u'b = %who', "b = get_ipython().run_line_magic('who', '')"),
-        (u'c= %ls', "c = get_ipython().run_line_magic('ls', '')"),
-        (u'd == %ls', u'd == %ls'), # Invalid syntax, but we leave == alone.
-        ('x=1', 'x=1'), # normal input is unmodified
-        ('    ','    '),  # blank lines are kept intact
-        (u"a, b = %foo", u"a, b = get_ipython().run_line_magic('foo', '')"),
-        ]],
-
-       classic_prompt =
-       [('>>> x=1', 'x=1'),
-        ('x=1', 'x=1'), # normal input is unmodified
-        ('    ', '    '),  # blank lines are kept intact
-        ],
-
-       ipy_prompt =
-       [('In [1]: x=1', 'x=1'),
-        ('x=1', 'x=1'), # normal input is unmodified
-        ('    ','    '),  # blank lines are kept intact
-        ],
-
-       # Tests for the escape transformer to leave normal code alone
-       escaped_noesc =
-       [ ('    ', '    '),
-         ('x=1', 'x=1'),
-         ],
-
-       # System calls
-       escaped_shell =
-       [(i,py3compat.u_format(o)) for i,o in \
-       [ (u'!ls', "get_ipython().system('ls')"),
-         # Double-escape shell, this means to capture the output of the
-         # subprocess and return it
-         (u'!!ls', "get_ipython().getoutput('ls')"),
-         ]],
-
-       # Help/object info
-       escaped_help =
-       [(i,py3compat.u_format(o)) for i,o in \
-       [ (u'?', 'get_ipython().show_usage()'),
-         (u'?x1', "get_ipython().run_line_magic('pinfo', 'x1')"),
-         (u'??x2', "get_ipython().run_line_magic('pinfo2', 'x2')"),
-         (u'?a.*s', "get_ipython().run_line_magic('psearch', 'a.*s')"),
-         (u'?%hist1', "get_ipython().run_line_magic('pinfo', '%hist1')"),
-         (u'?%%hist2', "get_ipython().run_line_magic('pinfo', '%%hist2')"),
-         (u'?abc = qwe', "get_ipython().run_line_magic('pinfo', 'abc')"),
-         ]],
-
-      end_help =
-      [(i,py3compat.u_format(o)) for i,o in \
-      [ (u'x3?', "get_ipython().run_line_magic('pinfo', 'x3')"),
-        (u'x4??', "get_ipython().run_line_magic('pinfo2', 'x4')"),
-        (u'%hist1?', "get_ipython().run_line_magic('pinfo', '%hist1')"),
-        (u'%hist2??', "get_ipython().run_line_magic('pinfo2', '%hist2')"),
-        (u'%%hist3?', "get_ipython().run_line_magic('pinfo', '%%hist3')"),
-        (u'%%hist4??', "get_ipython().run_line_magic('pinfo2', '%%hist4')"),
-        (u'π.foo?', "get_ipython().run_line_magic('pinfo', 'π.foo')"),
-        (u'f*?', "get_ipython().run_line_magic('psearch', 'f*')"),
-        (u'ax.*aspe*?', "get_ipython().run_line_magic('psearch', 'ax.*aspe*')"),
-        (u'a = abc?', "get_ipython().set_next_input('a = abc');"
-                      "get_ipython().run_line_magic('pinfo', 'abc')"),
-        (u'a = abc.qe??', "get_ipython().set_next_input('a = abc.qe');"
-                          "get_ipython().run_line_magic('pinfo2', 'abc.qe')"),
-        (u'a = *.items?', "get_ipython().set_next_input('a = *.items');"
-                          "get_ipython().run_line_magic('psearch', '*.items')"),
-        (u'plot(a?', "get_ipython().set_next_input('plot(a');"
-                     "get_ipython().run_line_magic('pinfo', 'a')"),
-        (u'a*2 #comment?', 'a*2 #comment?'),
-        ]],
-
-       # Explicit magic calls
-       escaped_magic =
-       [(i,py3compat.u_format(o)) for i,o in \
-       [ (u'%cd', "get_ipython().run_line_magic('cd', '')"),
-         (u'%cd /home', "get_ipython().run_line_magic('cd', '/home')"),
-         # Backslashes need to be escaped.
-         (u'%cd C:\\User', "get_ipython().run_line_magic('cd', 'C:\\\\User')"),
-         (u'    %magic', "    get_ipython().run_line_magic('magic', '')"),
-         ]],
-
-       # Quoting with separate arguments
-       escaped_quote =
-       [ (',f', 'f("")'),
-         (',f x', 'f("x")'),
-         ('  ,f y', '  f("y")'),
-         (',f a b', 'f("a", "b")'),
-         ],
-
-       # Quoting with single argument
-       escaped_quote2 =
-       [ (';f', 'f("")'),
-         (';f x', 'f("x")'),
-         ('  ;f y', '  f("y")'),
-         (';f a b', 'f("a b")'),
-         ],
-
-       # Simply apply parens
-       escaped_paren =
-       [ ('/f', 'f()'),
-         ('/f x', 'f(x)'),
-         ('  /f y', '  f(y)'),
-         ('/f a b', 'f(a, b)'),
-         ],
-
-       # Check that we transform prompts before other transforms
-       mixed =
-       [(i,py3compat.u_format(o)) for i,o in \
-       [ (u'In [1]: %lsmagic', "get_ipython().run_line_magic('lsmagic', '')"),
-         (u'>>> %lsmagic', "get_ipython().run_line_magic('lsmagic', '')"),
-         (u'In [2]: !ls', "get_ipython().system('ls')"),
-         (u'In [3]: abs?', "get_ipython().run_line_magic('pinfo', 'abs')"),
-         (u'In [4]: b = %who', "b = get_ipython().run_line_magic('who', '')"),
-         ]],
-       )
+syntax = dict(
+    assign_system=[
+        (i, py3compat.u_format(o))
+        for i, o in [
+            (u"a =! ls", "a = get_ipython().getoutput('ls')"),
+            (u"b = !ls", "b = get_ipython().getoutput('ls')"),
+            (u"c= !ls", "c = get_ipython().getoutput('ls')"),
+            (u"d == !ls", u"d == !ls"),  # Invalid syntax, but we leave == alone.
+            ("x=1", "x=1"),  # normal input is unmodified
+            ("    ", "    "),  # blank lines are kept intact
+            # Tuple unpacking
+            (
+                u"a, b = !echo 'a\\nb'",
+                u"a, b = get_ipython().getoutput(\"echo 'a\\\\nb'\")",
+            ),
+            (u"a,= !echo 'a'", u"a, = get_ipython().getoutput(\"echo 'a'\")"),
+            (
+                u"a, *bc = !echo 'a\\nb\\nc'",
+                u"a, *bc = get_ipython().getoutput(\"echo 'a\\\\nb\\\\nc'\")",
+            ),
+            # Tuple unpacking with regular Python expressions, not our syntax.
+            (u"a, b = range(2)", u"a, b = range(2)"),
+            (u"a, = range(1)", u"a, = range(1)"),
+            (u"a, *bc = range(3)", u"a, *bc = range(3)"),
+        ]
+    ],
+    assign_magic=[
+        (i, py3compat.u_format(o))
+        for i, o in [
+            (u"a =% who", "a = get_ipython().run_line_magic('who', '')"),
+            (u"b = %who", "b = get_ipython().run_line_magic('who', '')"),
+            (u"c= %ls", "c = get_ipython().run_line_magic('ls', '')"),
+            (u"d == %ls", u"d == %ls"),  # Invalid syntax, but we leave == alone.
+            ("x=1", "x=1"),  # normal input is unmodified
+            ("    ", "    "),  # blank lines are kept intact
+            (u"a, b = %foo", u"a, b = get_ipython().run_line_magic('foo', '')"),
+        ]
+    ],
+    classic_prompt=[
+        (">>> x=1", "x=1"),
+        ("x=1", "x=1"),  # normal input is unmodified
+        ("    ", "    "),  # blank lines are kept intact
+    ],
+    ipy_prompt=[
+        ("In [1]: x=1", "x=1"),
+        ("x=1", "x=1"),  # normal input is unmodified
+        ("    ", "    "),  # blank lines are kept intact
+    ],
+    # Tests for the escape transformer to leave normal code alone
+    escaped_noesc=[
+        ("    ", "    "),
+        ("x=1", "x=1"),
+    ],
+    # System calls
+    escaped_shell=[
+        (i, py3compat.u_format(o))
+        for i, o in [
+            (u"!ls", "get_ipython().system('ls')"),
+            # Double-escape shell, this means to capture the output of the
+            # subprocess and return it
+            (u"!!ls", "get_ipython().getoutput('ls')"),
+        ]
+    ],
+    # Help/object info
+    escaped_help=[
+        (i, py3compat.u_format(o))
+        for i, o in [
+            (u"?", "get_ipython().show_usage()"),
+            (u"?x1", "get_ipython().run_line_magic('pinfo', 'x1')"),
+            (u"??x2", "get_ipython().run_line_magic('pinfo2', 'x2')"),
+            (u"?a.*s", "get_ipython().run_line_magic('psearch', 'a.*s')"),
+            (u"?%hist1", "get_ipython().run_line_magic('pinfo', '%hist1')"),
+            (u"?%%hist2", "get_ipython().run_line_magic('pinfo', '%%hist2')"),
+            (u"?abc = qwe", "get_ipython().run_line_magic('pinfo', 'abc')"),
+        ]
+    ],
+    end_help=[
+        (i, py3compat.u_format(o))
+        for i, o in [
+            (u"x3?", "get_ipython().run_line_magic('pinfo', 'x3')"),
+            (u"x4??", "get_ipython().run_line_magic('pinfo2', 'x4')"),
+            (u"%hist1?", "get_ipython().run_line_magic('pinfo', '%hist1')"),
+            (u"%hist2??", "get_ipython().run_line_magic('pinfo2', '%hist2')"),
+            (u"%%hist3?", "get_ipython().run_line_magic('pinfo', '%%hist3')"),
+            (u"%%hist4??", "get_ipython().run_line_magic('pinfo2', '%%hist4')"),
+            (u"π.foo?", "get_ipython().run_line_magic('pinfo', 'π.foo')"),
+            (u"f*?", "get_ipython().run_line_magic('psearch', 'f*')"),
+            (u"ax.*aspe*?", "get_ipython().run_line_magic('psearch', 'ax.*aspe*')"),
+            (u"a = abc?", "get_ipython().run_line_magic('pinfo', 'abc')"),
+            (u"a = abc.qe??", "get_ipython().run_line_magic('pinfo2', 'abc.qe')"),
+            (u"a = *.items?", "get_ipython().run_line_magic('psearch', '*.items')"),
+            (u"plot(a?", "get_ipython().run_line_magic('pinfo', 'a')"),
+            (u"a*2 #comment?", "a*2 #comment?"),
+        ]
+    ],
+    # Explicit magic calls
+    escaped_magic=[
+        (i, py3compat.u_format(o))
+        for i, o in [
+            (u"%cd", "get_ipython().run_line_magic('cd', '')"),
+            (u"%cd /home", "get_ipython().run_line_magic('cd', '/home')"),
+            # Backslashes need to be escaped.
+            (u"%cd C:\\User", "get_ipython().run_line_magic('cd', 'C:\\\\User')"),
+            (u"    %magic", "    get_ipython().run_line_magic('magic', '')"),
+        ]
+    ],
+    # Quoting with separate arguments
+    escaped_quote=[
+        (",f", 'f("")'),
+        (",f x", 'f("x")'),
+        ("  ,f y", '  f("y")'),
+        (",f a b", 'f("a", "b")'),
+    ],
+    # Quoting with single argument
+    escaped_quote2=[
+        (";f", 'f("")'),
+        (";f x", 'f("x")'),
+        ("  ;f y", '  f("y")'),
+        (";f a b", 'f("a b")'),
+    ],
+    # Simply apply parens
+    escaped_paren=[
+        ("/f", "f()"),
+        ("/f x", "f(x)"),
+        ("  /f y", "  f(y)"),
+        ("/f a b", "f(a, b)"),
+    ],
+    # Check that we transform prompts before other transforms
+    mixed=[
+        (i, py3compat.u_format(o))
+        for i, o in [
+            (u"In [1]: %lsmagic", "get_ipython().run_line_magic('lsmagic', '')"),
+            (u">>> %lsmagic", "get_ipython().run_line_magic('lsmagic', '')"),
+            (u"In [2]: !ls", "get_ipython().system('ls')"),
+            (u"In [3]: abs?", "get_ipython().run_line_magic('pinfo', 'abs')"),
+            (u"In [4]: b = %who", "b = get_ipython().run_line_magic('who', '')"),
+        ]
+    ],
+)
 
 # multiline syntax examples.  Each of these should be a list of lists, with
 # each entry itself having pairs of raw/transformed input.  The union (with
 # '\n'.join() of the transformed inputs is what the splitter should produce
 # when fed the raw lines one at a time via push.
 syntax_ml = \
   dict(classic_prompt =
        [ [('>>> for i in range(10):','for i in range(10):'),
           ('...     print i','    print i'),
           ('... ', ''),
           ],
          [('>>> a="""','a="""'),
           ('... 123"""','123"""'),
           ],
          [('a="""','a="""'),
           ('... 123','123'),
           ('... 456"""','456"""'),
           ],
          [('a="""','a="""'),
           ('>>> 123','123'),
           ('... 456"""','456"""'),
           ],
          [('a="""','a="""'),
           ('123','123'),
           ('... 456"""','... 456"""'),
           ],
          [('....__class__','....__class__'),
          ],
          [('a=5', 'a=5'),
           ('...', ''),
          ],
          [('>>> def f(x):', 'def f(x):'),
           ('...', ''),
           ('...     return x', '    return x'),
           ],
          [('board = """....', 'board = """....'),
           ('....', '....'),
           ('...."""', '...."""'),
           ],
         ],
 
        ipy_prompt =
        [ [('In [24]: for i in range(10):','for i in range(10):'),
           ('   ....:     print i','    print i'),
           ('   ....: ', ''),
           ],
          [('In [24]: for i in range(10):','for i in range(10):'),
           # Qt console prompts expand with spaces, not dots
           ('    ...:     print i','    print i'),
           ('    ...: ', ''),
           ],
          [('In [24]: for i in range(10):','for i in range(10):'),
           # Sometimes whitespace preceding '...' has been removed
           ('...:     print i','    print i'),
           ('...: ', ''),
           ],
          [('In [24]: for i in range(10):','for i in range(10):'),
           # Space after last continuation prompt has been removed (issue #6674)
           ('...:     print i','    print i'),
           ('...:', ''),
           ],
          [('In [2]: a="""','a="""'),
           ('   ...: 123"""','123"""'),
           ],
          [('a="""','a="""'),
           ('   ...: 123','123'),
           ('   ...: 456"""','456"""'),
           ],
          [('a="""','a="""'),
           ('In [1]: 123','123'),
           ('   ...: 456"""','456"""'),
           ],
          [('a="""','a="""'),
           ('123','123'),
           ('   ...: 456"""','   ...: 456"""'),
           ],
          ],
 
        multiline_datastructure_prompt =
        [ [('>>> a = [1,','a = [1,'),
           ('... 2]','2]'),
          ],
        ],
         
        multiline_datastructure =
        [ [('b = ("%s"', None),
           ('# comment', None),
           ('%foo )', 'b = ("%s"\n# comment\n%foo )'),
          ],
        ],
        
        multiline_string =
        [ [("'''foo?", None),
           ("bar'''", "'''foo?\nbar'''"),
          ],
        ],
        
        leading_indent =
        [ [('    print "hi"','print "hi"'),
           ],
          [('  for a in range(5):','for a in range(5):'),
           ('    a*2','  a*2'),
           ],
          [('    a="""','a="""'),
           ('    123"""','123"""'),
            ],
          [('a="""','a="""'),
           ('    123"""','    123"""'),
           ],
        ],
        
        cellmagic =
        [ [(u'%%foo a', None),
           (None, u_fmt("get_ipython().run_cell_magic('foo', 'a', '')")),
           ],
          [(u'%%bar 123', None),
           (u'hello', None),
           (None , u_fmt("get_ipython().run_cell_magic('bar', '123', 'hello')")),
           ],
          [(u'a=5', 'a=5'),
           (u'%%cellmagic', '%%cellmagic'),
           ],
        ],
        
        escaped =
        [ [('%abc def \\', None),
           ('ghi', u_fmt("get_ipython().run_line_magic('abc', 'def ghi')")),
           ],
          [('%abc def \\', None),
           ('ghi\\', None),
           (None, u_fmt("get_ipython().run_line_magic('abc', 'def ghi')")),
           ],
        ],
        
        assign_magic =
        [ [(u'a = %bc de \\', None),
           (u'fg', u_fmt("a = get_ipython().run_line_magic('bc', 'de fg')")),
           ],
          [(u'a = %bc de \\', None),
           (u'fg\\', None),
           (None, u_fmt("a = get_ipython().run_line_magic('bc', 'de fg')")),
           ],
        ],
        
        assign_system =
        [ [(u'a = !bc de \\', None),
           (u'fg', u_fmt("a = get_ipython().getoutput('bc de fg')")),
           ],
          [(u'a = !bc de \\', None),
           (u'fg\\', None),
           (None, u_fmt("a = get_ipython().getoutput('bc de fg')")),
           ],
        ],
        )
 
 
 def test_assign_system():
     tt.check_pairs(transform_and_reset(ipt.assign_from_system), syntax['assign_system'])
 
 def test_assign_magic():
     tt.check_pairs(transform_and_reset(ipt.assign_from_magic), syntax['assign_magic'])
 
 def test_classic_prompt():
     tt.check_pairs(transform_and_reset(ipt.classic_prompt), syntax['classic_prompt'])
     for example in syntax_ml['classic_prompt']:
         transform_checker(example, ipt.classic_prompt)
     for example in syntax_ml['multiline_datastructure_prompt']:
         transform_checker(example, ipt.classic_prompt)
 
     # Check that we don't transform the second line if the first is obviously
     # IPython syntax
     transform_checker([
         (u'%foo', '%foo'),
         (u'>>> bar', '>>> bar'),
     ], ipt.classic_prompt)
 
 
 def test_ipy_prompt():
     tt.check_pairs(transform_and_reset(ipt.ipy_prompt), syntax['ipy_prompt'])
     for example in syntax_ml['ipy_prompt']:
         transform_checker(example, ipt.ipy_prompt)
 
     # Check that we don't transform the second line if we're inside a cell magic
     transform_checker([
         (u'%%foo', '%%foo'),
         (u'In [1]: bar', 'In [1]: bar'),
     ], ipt.ipy_prompt)
 
 def test_assemble_logical_lines():
     tests = \
     [ [(u"a = \\", None),
        (u"123", u"a = 123"),
       ],
       [(u"a = \\", None),  # Test resetting when within a multi-line string
        (u"12 *\\", None),
        (None, u"a = 12 *"),
       ],
       [(u"# foo\\", u"# foo\\"), # Comments can't be continued like this
       ],
     ]
     for example in tests:
         transform_checker(example, ipt.assemble_logical_lines)
 
 def test_assemble_python_lines():
     tests = \
     [ [(u"a = '''", None),
        (u"abc'''", u"a = '''\nabc'''"),
       ],
       [(u"a = '''", None),  # Test resetting when within a multi-line string
        (u"def", None),
        (None, u"a = '''\ndef"),
       ],
       [(u"a = [1,", None),
        (u"2]", u"a = [1,\n2]"),
       ],
       [(u"a = [1,", None),  # Test resetting when within a multi-line string
        (u"2,", None),
        (None, u"a = [1,\n2,"),
       ],
       [(u"a = '''", None),  # Test line continuation within a multi-line string
        (u"abc\\", None),
        (u"def", None),
        (u"'''", u"a = '''\nabc\\\ndef\n'''"),
       ],
     ] + syntax_ml['multiline_datastructure']
     for example in tests:
         transform_checker(example, ipt.assemble_python_lines)
 
 
 def test_help_end():
     tt.check_pairs(transform_and_reset(ipt.help_end), syntax['end_help'])
 
 def test_escaped_noesc():
     tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_noesc'])
 
 
 def test_escaped_shell():
     tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_shell'])
 
 
 def test_escaped_help():
     tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_help'])
 
 
 def test_escaped_magic():
     tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_magic'])
 
 
 def test_escaped_quote():
     tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote'])
 
 
 def test_escaped_quote2():
     tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote2'])
 
 
 def test_escaped_paren():
     tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_paren'])
 
 
 def test_cellmagic():
     for example in syntax_ml['cellmagic']:
         transform_checker(example, ipt.cellmagic)
     
     line_example = [(u'%%bar 123', None),
                     (u'hello', None),
                     (u'' , u_fmt("get_ipython().run_cell_magic('bar', '123', 'hello')")),
                    ]
     transform_checker(line_example, ipt.cellmagic, end_on_blank_line=True)
 
 def test_has_comment():
     tests = [('text', False),
              ('text #comment', True),
              ('text #comment\n', True),
              ('#comment', True),
              ('#comment\n', True),
              ('a = "#string"', False),
              ('a = "#string" # comment', True),
              ('a #comment not "string"', True),
              ]
     tt.check_pairs(ipt.has_comment, tests)
 
 @ipt.TokenInputTransformer.wrap
 def decistmt(tokens):
     """Substitute Decimals for floats in a string of statements.
 
     Based on an example from the tokenize module docs.
     """
     result = []
     for toknum, tokval, _, _, _  in tokens:
         if toknum == tokenize.NUMBER and '.' in tokval:  # replace NUMBER tokens
             for newtok in [
                 (tokenize.NAME, 'Decimal'),
                 (tokenize.OP, '('),
                 (tokenize.STRING, repr(tokval)),
                 (tokenize.OP, ')')
             ]:
                 yield newtok
         else:
             yield (toknum, tokval)
 
 
 
 def test_token_input_transformer():
     tests = [(u'1.2', u_fmt(u"Decimal ('1.2')")),
              (u'"1.2"', u'"1.2"'),
              ]
     tt.check_pairs(transform_and_reset(decistmt), tests)
     ml_tests = \
     [ [(u"a = 1.2; b = '''x", None),
        (u"y'''", u_fmt(u"a =Decimal ('1.2');b ='''x\ny'''")),
       ],
       [(u"a = [1.2,", None),
        (u"3]", u_fmt(u"a =[Decimal ('1.2'),\n3 ]")),
       ],
       [(u"a = '''foo", None),  # Test resetting when within a multi-line string
        (u"bar", None),
        (None, u"a = '''foo\nbar"),
       ],
     ]
     for example in ml_tests:
         transform_checker(example, decistmt)

IPython/core/tests/test_inputtransformer2.py

@@ -1,358 +1,395 b''
 """Tests for the token-based transformers in IPython.core.inputtransformer2
 
 Line-based transformers are the simpler ones; token-based transformers are
 more complex. See test_inputtransformer2_line for tests for line-based
 transformations.
 """
 import nose.tools as nt
 import string
 import sys
 from textwrap import dedent
 
 import pytest
 
 from IPython.core import inputtransformer2 as ipt2
 from IPython.core.inputtransformer2 import _find_assign_op, make_tokens_by_line
 from IPython.testing.decorators import skip
 
-MULTILINE_MAGIC = ("""\
+MULTILINE_MAGIC = (
+    """\
 a = f()
 %foo \\
 bar
 g()
-""".splitlines(keepends=True), (2, 0), """\
+""".splitlines(
+        keepends=True
+    ),
+    (2, 0),
+    """\
 a = f()
 get_ipython().run_line_magic('foo', ' bar')
 g()
-""".splitlines(keepends=True))
+""".splitlines(
+        keepends=True
+    ),
+)
 
-INDENTED_MAGIC = ("""\
+INDENTED_MAGIC = (
+    """\
 for a in range(5):
     %ls
-""".splitlines(keepends=True), (2, 4), """\
+""".splitlines(
+        keepends=True
+    ),
+    (2, 4),
+    """\
 for a in range(5):
     get_ipython().run_line_magic('ls', '')
-""".splitlines(keepends=True))
+""".splitlines(
+        keepends=True
+    ),
+)
 
-CRLF_MAGIC = ([
-    "a = f()\n",
-    "%ls\r\n",
-    "g()\n"
-], (2, 0), [
-    "a = f()\n",
-    "get_ipython().run_line_magic('ls', '')\n",
-    "g()\n"
-])
-
-MULTILINE_MAGIC_ASSIGN = ("""\
+CRLF_MAGIC = (
+    ["a = f()\n", "%ls\r\n", "g()\n"],
+    (2, 0),
+    ["a = f()\n", "get_ipython().run_line_magic('ls', '')\n", "g()\n"],
+)
+
+MULTILINE_MAGIC_ASSIGN = (
+    """\
 a = f()
 b = %foo \\
   bar
 g()
-""".splitlines(keepends=True), (2, 4), """\
+""".splitlines(
+        keepends=True
+    ),
+    (2, 4),
+    """\
 a = f()
 b = get_ipython().run_line_magic('foo', '   bar')
 g()
-""".splitlines(keepends=True))
+""".splitlines(
+        keepends=True
+    ),
+)
 
 MULTILINE_SYSTEM_ASSIGN = ("""\
 a = f()
 b = !foo \\
   bar
 g()
 """.splitlines(keepends=True), (2, 4), """\
 a = f()
 b = get_ipython().getoutput('foo    bar')
 g()
 """.splitlines(keepends=True))
 
 #####
 
 MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT = ("""\
 def test():
   for i in range(1):
     print(i)
   res =! ls
-""".splitlines(keepends=True), (4, 7), '''\
+""".splitlines(
+        keepends=True
+    ),
+    (4, 7),
+    """\
 def test():
   for i in range(1):
     print(i)
   res =get_ipython().getoutput(\' ls\')
-'''.splitlines(keepends=True))
+""".splitlines(
+        keepends=True
+    ),
+)
 
 ######
 
-AUTOCALL_QUOTE = (
-    [",f 1 2 3\n"], (1, 0),
-    ['f("1", "2", "3")\n']
-)
+AUTOCALL_QUOTE = ([",f 1 2 3\n"], (1, 0), ['f("1", "2", "3")\n'])
 
-AUTOCALL_QUOTE2 = (
-    [";f 1 2 3\n"], (1, 0),
-    ['f("1 2 3")\n']
-)
+AUTOCALL_QUOTE2 = ([";f 1 2 3\n"], (1, 0), ['f("1 2 3")\n'])
 
-AUTOCALL_PAREN = (
-    ["/f 1 2 3\n"], (1, 0),
-    ['f(1, 2, 3)\n']
-)
+AUTOCALL_PAREN = (["/f 1 2 3\n"], (1, 0), ["f(1, 2, 3)\n"])
 
-SIMPLE_HELP = (
-    ["foo?\n"], (1, 0),
-    ["get_ipython().run_line_magic('pinfo', 'foo')\n"]
-)
+SIMPLE_HELP = (["foo?\n"], (1, 0), ["get_ipython().run_line_magic('pinfo', 'foo')\n"])
 
 DETAILED_HELP = (
-    ["foo??\n"], (1, 0),
-    ["get_ipython().run_line_magic('pinfo2', 'foo')\n"]
+    ["foo??\n"],
+    (1, 0),
+    ["get_ipython().run_line_magic('pinfo2', 'foo')\n"],
 )
 
-MAGIC_HELP = (
-    ["%foo?\n"], (1, 0),
-    ["get_ipython().run_line_magic('pinfo', '%foo')\n"]
-)
+MAGIC_HELP = (["%foo?\n"], (1, 0), ["get_ipython().run_line_magic('pinfo', '%foo')\n"])
 
 HELP_IN_EXPR = (
-    ["a = b + c?\n"], (1, 0),
-    ["get_ipython().set_next_input('a = b + c');"
-     "get_ipython().run_line_magic('pinfo', 'c')\n"]
+    ["a = b + c?\n"],
+    (1, 0),
+    ["get_ipython().run_line_magic('pinfo', 'c')\n"],
 )
 
-HELP_CONTINUED_LINE = ("""\
+HELP_CONTINUED_LINE = (
+    """\
 a = \\
 zip?
-""".splitlines(keepends=True), (1, 0),
-[r"get_ipython().set_next_input('a = \\\nzip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
+""".splitlines(
+        keepends=True
+    ),
+    (1, 0),
+    [r"get_ipython().run_line_magic('pinfo', 'zip')" + "\n"],
 )
 
-HELP_MULTILINE = ("""\
+HELP_MULTILINE = (
+    """\
 (a,
 b) = zip?
-""".splitlines(keepends=True), (1, 0),
-[r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
+""".splitlines(
+        keepends=True
+    ),
+    (1, 0),
+    [r"get_ipython().run_line_magic('pinfo', 'zip')" + "\n"],
 )
 
 HELP_UNICODE = (
-    ["π.foo?\n"], (1, 0),
-    ["get_ipython().run_line_magic('pinfo', 'π.foo')\n"]
+    ["π.foo?\n"],
+    (1, 0),
+    ["get_ipython().run_line_magic('pinfo', 'π.foo')\n"],
 )
 
 
 def null_cleanup_transformer(lines):
     """
     A cleanup transform that returns an empty list.
     """
     return []
 
 def check_make_token_by_line_never_ends_empty():
     """
     Check that not sequence of single or double characters ends up leading to en empty list of tokens
     """
     from string import printable
+
     for c in printable:
         nt.assert_not_equal(make_tokens_by_line(c)[-1], [])
         for k in printable:
             nt.assert_not_equal(make_tokens_by_line(c+k)[-1], [])
 
 def check_find(transformer, case, match=True):
-    sample, expected_start, _  = case
+    sample, expected_start, _ = case
     tbl = make_tokens_by_line(sample)
     res = transformer.find(tbl)
     if match:
         # start_line is stored 0-indexed, expected values are 1-indexed
         nt.assert_equal((res.start_line+1, res.start_col), expected_start)
         return res
     else:
         nt.assert_is(res, None)
 
+
 def check_transform(transformer_cls, case):
     lines, start, expected = case
     transformer = transformer_cls(start)
     nt.assert_equal(transformer.transform(lines), expected)
 
+
 def test_continued_line():
     lines = MULTILINE_MAGIC_ASSIGN[0]
     nt.assert_equal(ipt2.find_end_of_continued_line(lines, 1), 2)
 
     nt.assert_equal(ipt2.assemble_continued_line(lines, (1, 5), 2), "foo    bar")
 
+
 def test_find_assign_magic():
     check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
     check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False)
     check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT, match=False)
 
+
 def test_transform_assign_magic():
     check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
 
+
 def test_find_assign_system():
     check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
     check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT)
     check_find(ipt2.SystemAssign, (["a =  !ls\n"], (1, 5), None))
     check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None))
     check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False)
 
+
 def test_transform_assign_system():
     check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
     check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT)
 
+
 def test_find_magic_escape():
     check_find(ipt2.EscapedCommand, MULTILINE_MAGIC)
     check_find(ipt2.EscapedCommand, INDENTED_MAGIC)
     check_find(ipt2.EscapedCommand, MULTILINE_MAGIC_ASSIGN, match=False)
 
+
 def test_transform_magic_escape():
     check_transform(ipt2.EscapedCommand, MULTILINE_MAGIC)
     check_transform(ipt2.EscapedCommand, INDENTED_MAGIC)
     check_transform(ipt2.EscapedCommand, CRLF_MAGIC)
 
+
 def test_find_autocalls():
     for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
         print("Testing %r" % case[0])
         check_find(ipt2.EscapedCommand, case)
 
+
 def test_transform_autocall():
     for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
         print("Testing %r" % case[0])
         check_transform(ipt2.EscapedCommand, case)
 
+
 def test_find_help():
     for case in [SIMPLE_HELP, DETAILED_HELP, MAGIC_HELP, HELP_IN_EXPR]:
         check_find(ipt2.HelpEnd, case)
 
     tf = check_find(ipt2.HelpEnd, HELP_CONTINUED_LINE)
     nt.assert_equal(tf.q_line, 1)
     nt.assert_equal(tf.q_col, 3)
 
     tf = check_find(ipt2.HelpEnd, HELP_MULTILINE)
     nt.assert_equal(tf.q_line, 1)
     nt.assert_equal(tf.q_col, 8)
 
     # ? in a comment does not trigger help
     check_find(ipt2.HelpEnd, (["foo # bar?\n"], None, None), match=False)
     # Nor in a string
     check_find(ipt2.HelpEnd, (["foo = '''bar?\n"], None, None), match=False)
 
+
 def test_transform_help():
     tf = ipt2.HelpEnd((1, 0), (1, 9))
     nt.assert_equal(tf.transform(HELP_IN_EXPR[0]), HELP_IN_EXPR[2])
 
     tf = ipt2.HelpEnd((1, 0), (2, 3))
     nt.assert_equal(tf.transform(HELP_CONTINUED_LINE[0]), HELP_CONTINUED_LINE[2])
 
     tf = ipt2.HelpEnd((1, 0), (2, 8))
     nt.assert_equal(tf.transform(HELP_MULTILINE[0]), HELP_MULTILINE[2])
 
     tf = ipt2.HelpEnd((1, 0), (1, 0))
     nt.assert_equal(tf.transform(HELP_UNICODE[0]), HELP_UNICODE[2])
 
+
 def test_find_assign_op_dedent():
     """
     be careful that empty token like dedent are not counted as parens
     """
+
     class Tk:
         def __init__(self, s):
             self.string = s
 
     nt.assert_equal(_find_assign_op([Tk(s) for s in ('','a','=','b')]), 2)
     nt.assert_equal(_find_assign_op([Tk(s) for s in ('','(', 'a','=','b', ')', '=' ,'5')]), 6)
 
 examples = [
     pytest.param("a = 1", "complete", None),
     pytest.param("for a in range(5):", "incomplete", 4),
     pytest.param("for a in range(5):\n    if a > 0:", "incomplete", 8),
     pytest.param("raise = 2", "invalid", None),
     pytest.param("a = [1,\n2,", "incomplete", 0),
     pytest.param("(\n))", "incomplete", 0),
     pytest.param("\\\r\n", "incomplete", 0),
     pytest.param("a = '''\n   hi", "incomplete", 3),
     pytest.param("def a():\n x=1\n global x", "invalid", None),
     pytest.param(
         "a \\ ",
         "invalid",
         None,
         marks=pytest.mark.xfail(
             reason="Bug in python 3.9.8 – bpo 45738",
             condition=sys.version_info[:3] == (3, 9, 8),
             raises=SystemError,
             strict=True,
         ),
     ),  # Nothing allowed after backslash,
     pytest.param("1\\\n+2", "complete", None),
 ]
 
 
 @skip('Tested on master, skip only on iptest not available on 7.x')
 @pytest.mark.xfail(
     reason="Bug in python 3.9.8 – bpo 45738",
     condition=sys.version_info[:3] == (3, 9, 8),
 )
 def test_check_complete():
     cc = ipt2.TransformerManager().check_complete
 
-    example = dedent("""
+    example = dedent(
+        """
         if True:
-            a=1""" )
+            a=1"""
+    )
 
     nt.assert_equal(cc(example), ('incomplete', 4))
     nt.assert_equal(cc(example+'\n'), ('complete', None))
     nt.assert_equal(cc(example+'\n    '), ('complete', None))
 
     # no need to loop on all the letters/numbers.
-    short = '12abAB'+string.printable[62:]
+    short = "12abAB" + string.printable[62:]
     for c in short:
         # test does not raise:
         cc(c)
         for k in short:
-            cc(c+k)
+            cc(c + k)
 
     nt.assert_equal(cc("def f():\n  x=0\n  \\\n  "), ('incomplete', 2))
 
 def test_check_complete_II():
     """
     Test that multiple line strings are properly handled.
 
     Separate test function for convenience
 
     """
     cc = ipt2.TransformerManager().check_complete
     nt.assert_equal(cc('''def foo():\n    """'''), ('incomplete', 4))
 
 
 def test_null_cleanup_transformer():
     manager = ipt2.TransformerManager()
     manager.cleanup_transforms.insert(0, null_cleanup_transformer)
     assert  manager.transform_cell("") == ""
 
 
-
-
 def test_side_effects_I():
     count = 0
+
     def counter(lines):
         nonlocal count
         count += 1
         return lines
 
     counter.has_side_effects = True
 
     manager = ipt2.TransformerManager()
     manager.cleanup_transforms.insert(0, counter)
-    assert manager.check_complete("a=1\n") == ('complete', None)
+    assert manager.check_complete("a=1\n") == ("complete", None)
     assert count == 0
 
 
-
-
 def test_side_effects_II():
     count = 0
+
     def counter(lines):
         nonlocal count
         count += 1
         return lines
 
     counter.has_side_effects = True
 
     manager = ipt2.TransformerManager()
     manager.line_transforms.insert(0, counter)
-    assert manager.check_complete("b=1\n") == ('complete', None)
+    assert manager.check_complete("b=1\n") == ("complete", None)
     assert count == 0