run formatter

IPython/core/inputsplitter.py

@@ -1,782 +1,787 b''
 """DEPRECATED: Input handling and transformation machinery.
 
 This module was deprecated in IPython 7.0, in favour of inputtransformer2.
 
 The first class in this module, :class:`InputSplitter`, is designed to tell when
 input from a line-oriented frontend is complete and should be executed, and when
 the user should be prompted for another line of code instead. The name 'input
 splitter' is largely for historical reasons.
 
 A companion, :class:`IPythonInputSplitter`, provides the same functionality but
 with full support for the extended IPython syntax (magics, system calls, etc).
 The code to actually do these transformations is in :mod:`IPython.core.inputtransformer`.
 :class:`IPythonInputSplitter` feeds the raw code to the transformers in order
 and stores the results.
 
 For more details, see the class docstrings below.
 """
 
 from warnings import warn
 
 warn('IPython.core.inputsplitter is deprecated since IPython 7 in favor of `IPython.core.inputtransformer2`',
      DeprecationWarning)
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 import ast
 import codeop
 import io
 import re
 import sys
 import tokenize
 import warnings
 
 from typing import List
 
 from IPython.core.inputtransformer import (leading_indent,
                                            classic_prompt,
                                            ipy_prompt,
                                            cellmagic,
                                            assemble_logical_lines,
                                            help_end,
                                            escaped_commands,
                                            assign_from_magic,
                                            assign_from_system,
                                            assemble_python_lines,
                                            )
 from IPython.utils import tokenutil
 
 # These are available in this module for backwards compatibility.
 from IPython.core.inputtransformer import (ESC_SHELL, ESC_SH_CAP, ESC_HELP,
                                         ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,
                                         ESC_QUOTE, ESC_QUOTE2, ESC_PAREN, ESC_SEQUENCES)
 
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
 # FIXME: These are general-purpose utilities that later can be moved to the
 # general ward.  Kept here for now because we're being very strict about test
 # coverage with this code, and this lets us ensure that we keep 100% coverage
 # while developing.
 
 # compiled regexps for autoindent management
 dedent_re = re.compile('|'.join([
     r'^\s+raise(\s.*)?$', # raise statement (+ space + other stuff, maybe)
     r'^\s+raise\([^\)]*\).*$', # wacky raise with immediate open paren
     r'^\s+return(\s.*)?$', # normal return (+ space + other stuff, maybe)
     r'^\s+return\([^\)]*\).*$', # wacky return with immediate open paren
     r'^\s+pass\s*$', # pass (optionally followed by trailing spaces)
     r'^\s+break\s*$', # break (optionally followed by trailing spaces)
     r'^\s+continue\s*$', # continue (optionally followed by trailing spaces)
 ]))
 ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)')
 
 # regexp to match pure comment lines so we don't accidentally insert 'if 1:'
 # before pure comments
 comment_line_re = re.compile(r'^\s*\#')
 
 
 def num_ini_spaces(s):
     """Return the number of initial spaces in a string.
 
     Note that tabs are counted as a single space.  For now, we do *not* support
     mixing of tabs and spaces in the user's input.
 
     Parameters
     ----------
     s : string
 
     Returns
     -------
     n : int
     """
 
     ini_spaces = ini_spaces_re.match(s)
     if ini_spaces:
         return ini_spaces.end()
     else:
         return 0
 
 # Fake token types for partial_tokenize:
 INCOMPLETE_STRING = tokenize.N_TOKENS
 IN_MULTILINE_STATEMENT = tokenize.N_TOKENS + 1
 
 # The 2 classes below have the same API as TokenInfo, but don't try to look up
 # a token type name that they won't find.
 class IncompleteString:
     type = exact_type = INCOMPLETE_STRING
     def __init__(self, s, start, end, line):
         self.s = s
         self.start = start
         self.end = end
         self.line = line
 
 class InMultilineStatement:
     type = exact_type = IN_MULTILINE_STATEMENT
     def __init__(self, pos, line):
         self.s = ''
         self.start = self.end = pos
         self.line = line
 
 def partial_tokens(s):
     """Iterate over tokens from a possibly-incomplete string of code.
 
     This adds two special token types: INCOMPLETE_STRING and
     IN_MULTILINE_STATEMENT. These can only occur as the last token yielded, and
     represent the two main ways for code to be incomplete.
     """
     readline = io.StringIO(s).readline
     token = tokenize.TokenInfo(tokenize.NEWLINE, '', (1, 0), (1, 0), '')
     try:
         for token in tokenutil.generate_tokens_catch_errors(readline):
             yield token
     except tokenize.TokenError as e:
         # catch EOF error
         lines = s.splitlines(keepends=True)
         end = len(lines), len(lines[-1])
         if 'multi-line string' in e.args[0]:
             l, c = start = token.end
             s = lines[l-1][c:] + ''.join(lines[l:])
             yield IncompleteString(s, start, end, lines[-1])
         elif 'multi-line statement' in e.args[0]:
             yield InMultilineStatement(end, lines[-1])
         else:
             raise
 
 def find_next_indent(code):
     """Find the number of spaces for the next line of indentation"""
     tokens = list(partial_tokens(code))
     if tokens[-1].type == tokenize.ENDMARKER:
         tokens.pop()
     if not tokens:
         return 0
 
-    while (tokens[-1].type in {tokenize.DEDENT, tokenize.NEWLINE, tokenize.COMMENT, tokenize.ERRORTOKEN}):
+    while tokens[-1].type in {
+        tokenize.DEDENT,
+        tokenize.NEWLINE,
+        tokenize.COMMENT,
+        tokenize.ERRORTOKEN,
+    }:
         tokens.pop()
 
     # Starting in Python 3.12, the tokenize module adds implicit newlines at the end
     # of input. We need to remove those if we're in a multiline statement
     if tokens[-1].type == IN_MULTILINE_STATEMENT:
         while tokens[-2].type in {tokenize.NL}:
             tokens.pop(-2)
 
 
     if tokens[-1].type == INCOMPLETE_STRING:
         # Inside a multiline string
         return 0
 
     # Find the indents used before
     prev_indents = [0]
     def _add_indent(n):
         if n != prev_indents[-1]:
             prev_indents.append(n)
 
     tokiter = iter(tokens)
     for tok in tokiter:
         if tok.type in {tokenize.INDENT, tokenize.DEDENT}:
             _add_indent(tok.end[1])
         elif (tok.type == tokenize.NL):
             try:
                 _add_indent(next(tokiter).start[1])
             except StopIteration:
                 break
 
     last_indent = prev_indents.pop()
 
     # If we've just opened a multiline statement (e.g. 'a = ['), indent more
     if tokens[-1].type == IN_MULTILINE_STATEMENT:
         if tokens[-2].exact_type in {tokenize.LPAR, tokenize.LSQB, tokenize.LBRACE}:
             return last_indent + 4
         return last_indent
 
     if tokens[-1].exact_type == tokenize.COLON:
         # Line ends with colon - indent
         return last_indent + 4
 
     if last_indent:
         # Examine the last line for dedent cues - statements like return or
         # raise which normally end a block of code.
         last_line_starts = 0
         for i, tok in enumerate(tokens):
             if tok.type == tokenize.NEWLINE:
                 last_line_starts = i + 1
 
         last_line_tokens = tokens[last_line_starts:]
         names = [t.string for t in last_line_tokens if t.type == tokenize.NAME]
         if names and names[0] in {'raise', 'return', 'pass', 'break', 'continue'}:
             # Find the most recent indentation less than the current level
             for indent in reversed(prev_indents):
                 if indent < last_indent:
                     return indent
 
     return last_indent
 
 
 def last_blank(src):
     """Determine if the input source ends in a blank.
 
     A blank is either a newline or a line consisting of whitespace.
 
     Parameters
     ----------
     src : string
         A single or multiline string.
     """
     if not src: return False
     ll  = src.splitlines()[-1]
     return (ll == '') or ll.isspace()
 
 
 last_two_blanks_re = re.compile(r'\n\s*\n\s*$', re.MULTILINE)
 last_two_blanks_re2 = re.compile(r'.+\n\s*\n\s+$', re.MULTILINE)
 
 def last_two_blanks(src):
     """Determine if the input source ends in two blanks.
 
     A blank is either a newline or a line consisting of whitespace.
 
     Parameters
     ----------
     src : string
         A single or multiline string.
     """
     if not src: return False
     # The logic here is tricky: I couldn't get a regexp to work and pass all
     # the tests, so I took a different approach: split the source by lines,
     # grab the last two and prepend '###\n' as a stand-in for whatever was in
     # the body before the last two lines.  Then, with that structure, it's
     # possible to analyze with two regexps.  Not the most elegant solution, but
     # it works.  If anyone tries to change this logic, make sure to validate
     # the whole test suite first!
     new_src = '\n'.join(['###\n'] + src.splitlines()[-2:])
     return (bool(last_two_blanks_re.match(new_src)) or
             bool(last_two_blanks_re2.match(new_src)) )
 
 
 def remove_comments(src):
     """Remove all comments from input source.
 
     Note: comments are NOT recognized inside of strings!
 
     Parameters
     ----------
     src : string
         A single or multiline input string.
 
     Returns
     -------
     String with all Python comments removed.
     """
 
     return re.sub('#.*', '', src)
 
 
 def get_input_encoding():
     """Return the default standard input encoding.
 
     If sys.stdin has no encoding, 'ascii' is returned."""
     # There are strange environments for which sys.stdin.encoding is None. We
     # ensure that a valid encoding is returned.
     encoding = getattr(sys.stdin, 'encoding', None)
     if encoding is None:
         encoding = 'ascii'
     return encoding
 
 #-----------------------------------------------------------------------------
 # Classes and functions for normal Python syntax handling
 #-----------------------------------------------------------------------------
 
 class InputSplitter(object):
     r"""An object that can accumulate lines of Python source before execution.
 
     This object is designed to be fed python source line-by-line, using
     :meth:`push`. It will return on each push whether the currently pushed
     code could be executed already. In addition, it provides a method called
     :meth:`push_accepts_more` that can be used to query whether more input
     can be pushed into a single interactive block.
 
     This is a simple example of how an interactive terminal-based client can use
     this tool::
 
         isp = InputSplitter()
         while isp.push_accepts_more():
             indent = ' '*isp.indent_spaces
             prompt = '>>> ' + indent
             line = indent + raw_input(prompt)
             isp.push(line)
         print 'Input source was:\n', isp.source_reset(),
     """
     # A cache for storing the current indentation
     # The first value stores the most recently processed source input
     # The second value is the number of spaces for the current indentation 
     # If self.source matches the first value, the second value is a valid
     # current indentation. Otherwise, the cache is invalid and the indentation
     # must be recalculated.
     _indent_spaces_cache = None, None
     # String, indicating the default input encoding.  It is computed by default
     # at initialization time via get_input_encoding(), but it can be reset by a
     # client with specific knowledge of the encoding.
     encoding = ''
     # String where the current full source input is stored, properly encoded.
     # Reading this attribute is the normal way of querying the currently pushed
     # source code, that has been properly encoded.
     source = ''
     # Code object corresponding to the current source.  It is automatically
     # synced to the source, so it can be queried at any time to obtain the code
     # object; it will be None if the source doesn't compile to valid Python.
     code = None
 
     # Private attributes
 
     # List with lines of input accumulated so far
     _buffer: List[str]
     # Command compiler
     _compile: codeop.CommandCompiler
     # Boolean indicating whether the current block is complete
     _is_complete = None
     # Boolean indicating whether the current block has an unrecoverable syntax error
     _is_invalid = False
 
     def __init__(self) -> None:
         """Create a new InputSplitter instance."""
         self._buffer = []
         self._compile = codeop.CommandCompiler()
         self.encoding = get_input_encoding()
 
     def reset(self):
         """Reset the input buffer and associated state."""
         self._buffer[:] = []
         self.source = ''
         self.code = None
         self._is_complete = False
         self._is_invalid = False
 
     def source_reset(self):
         """Return the input source and perform a full reset.
         """
         out = self.source
         self.reset()
         return out
 
     def check_complete(self, source):
         """Return whether a block of code is ready to execute, or should be continued
 
         This is a non-stateful API, and will reset the state of this InputSplitter.
 
         Parameters
         ----------
         source : string
             Python input code, which can be multiline.
 
         Returns
         -------
         status : str
             One of 'complete', 'incomplete', or 'invalid' if source is not a
             prefix of valid code.
         indent_spaces : int or None
             The number of spaces by which to indent the next line of code. If
             status is not 'incomplete', this is None.
         """
         self.reset()
         try:
             self.push(source)
         except SyntaxError:
             # Transformers in IPythonInputSplitter can raise SyntaxError,
             # which push() will not catch.
             return 'invalid', None
         else:
             if self._is_invalid:
                 return 'invalid', None
             elif self.push_accepts_more():
                 return 'incomplete', self.get_indent_spaces()
             else:
                 return 'complete', None
         finally:
             self.reset()
 
     def push(self, lines:str) -> bool:
         """Push one or more lines of input.
 
         This stores the given lines and returns a status code indicating
         whether the code forms a complete Python block or not.
 
         Any exceptions generated in compilation are swallowed, but if an
         exception was produced, the method returns True.
 
         Parameters
         ----------
         lines : string
             One or more lines of Python input.
 
         Returns
         -------
         is_complete : boolean
             True if the current input source (the result of the current input
             plus prior inputs) forms a complete Python execution block.  Note that
             this value is also stored as a private attribute (``_is_complete``), so it
             can be queried at any time.
         """
         assert isinstance(lines, str)
         self._store(lines)
         source = self.source
 
         # Before calling _compile(), reset the code object to None so that if an
         # exception is raised in compilation, we don't mislead by having
         # inconsistent code/source attributes.
         self.code, self._is_complete = None, None
         self._is_invalid = False
 
         # Honor termination lines properly
         if source.endswith('\\\n'):
             return False
 
         try:
             with warnings.catch_warnings():
                 warnings.simplefilter('error', SyntaxWarning)
                 self.code = self._compile(source, symbol="exec")
         # Invalid syntax can produce any of a number of different errors from
         # inside the compiler, so we have to catch them all.  Syntax errors
         # immediately produce a 'ready' block, so the invalid Python can be
         # sent to the kernel for evaluation with possible ipython
         # special-syntax conversion.
         except (SyntaxError, OverflowError, ValueError, TypeError,
                 MemoryError, SyntaxWarning):
             self._is_complete = True
             self._is_invalid = True
         else:
             # Compilation didn't produce any exceptions (though it may not have
             # given a complete code object)
             self._is_complete = self.code is not None
 
         return self._is_complete
 
     def push_accepts_more(self):
         """Return whether a block of interactive input can accept more input.
 
         This method is meant to be used by line-oriented frontends, who need to
         guess whether a block is complete or not based solely on prior and
         current input lines.  The InputSplitter considers it has a complete
         interactive block and will not accept more input when either:
 
         * A SyntaxError is raised
 
         * The code is complete and consists of a single line or a single
           non-compound statement
 
         * The code is complete and has a blank line at the end
 
         If the current input produces a syntax error, this method immediately
         returns False but does *not* raise the syntax error exception, as
         typically clients will want to send invalid syntax to an execution
         backend which might convert the invalid syntax into valid Python via
         one of the dynamic IPython mechanisms.
         """
 
         # With incomplete input, unconditionally accept more
         # A syntax error also sets _is_complete to True - see push()
         if not self._is_complete:
             #print("Not complete")  # debug
             return True
         
         # The user can make any (complete) input execute by leaving a blank line
         last_line = self.source.splitlines()[-1]
         if (not last_line) or last_line.isspace():
             #print("Blank line")  # debug
             return False
         
         # If there's just a single line or AST node, and we're flush left, as is
         # the case after a simple statement such as 'a=1', we want to execute it
         # straight away.
         if self.get_indent_spaces() == 0:
             if len(self.source.splitlines()) <= 1:
                 return False
             
             try:
                 code_ast = ast.parse("".join(self._buffer))
             except Exception:
                 #print("Can't parse AST")  # debug
                 return False
             else:
                 if len(code_ast.body) == 1 and \
                                     not hasattr(code_ast.body[0], 'body'):
                     #print("Simple statement")  # debug
                     return False
 
         # General fallback - accept more code
         return True
 
     def get_indent_spaces(self):
         sourcefor, n = self._indent_spaces_cache
         if sourcefor == self.source:
             return n
 
         # self.source always has a trailing newline
         n = find_next_indent(self.source[:-1])
         self._indent_spaces_cache = (self.source, n)
         return n
 
     # Backwards compatibility. I think all code that used .indent_spaces was
     # inside IPython, but we can leave this here until IPython 7 in case any
     # other modules are using it. -TK, November 2017
     indent_spaces = property(get_indent_spaces)
 
     def _store(self, lines, buffer=None, store='source'):
         """Store one or more lines of input.
 
         If input lines are not newline-terminated, a newline is automatically
         appended."""
 
         if buffer is None:
             buffer = self._buffer
 
         if lines.endswith('\n'):
             buffer.append(lines)
         else:
             buffer.append(lines+'\n')
         setattr(self, store, self._set_source(buffer))
 
     def _set_source(self, buffer):
         return u''.join(buffer)
 
 
 class IPythonInputSplitter(InputSplitter):
     """An input splitter that recognizes all of IPython's special syntax."""
 
     # String with raw, untransformed input.
     source_raw = ''
     
     # Flag to track when a transformer has stored input that it hasn't given
     # back yet.
     transformer_accumulating = False
     
     # Flag to track when assemble_python_lines has stored input that it hasn't
     # given back yet.
     within_python_line = False
 
     # Private attributes
 
     # List with lines of raw input accumulated so far.
     _buffer_raw = None
 
     def __init__(self, line_input_checker=True, physical_line_transforms=None,
                     logical_line_transforms=None, python_line_transforms=None):
         super(IPythonInputSplitter, self).__init__()
         self._buffer_raw = []
         self._validate = True
         
         if physical_line_transforms is not None:
             self.physical_line_transforms = physical_line_transforms
         else:
             self.physical_line_transforms = [
                                              leading_indent(),
                                              classic_prompt(),
                                              ipy_prompt(),
                                              cellmagic(end_on_blank_line=line_input_checker),
                                             ]
         
         self.assemble_logical_lines = assemble_logical_lines()
         if logical_line_transforms is not None:
             self.logical_line_transforms = logical_line_transforms
         else:
             self.logical_line_transforms = [
                                             help_end(),
                                             escaped_commands(),
                                             assign_from_magic(),
                                             assign_from_system(),
                                            ]
         
         self.assemble_python_lines = assemble_python_lines()
         if python_line_transforms is not None:
             self.python_line_transforms = python_line_transforms
         else:
             # We don't use any of these at present
             self.python_line_transforms = []
     
     @property
     def transforms(self):
         "Quick access to all transformers."
         return self.physical_line_transforms + \
             [self.assemble_logical_lines] + self.logical_line_transforms + \
             [self.assemble_python_lines]  + self.python_line_transforms
     
     @property
     def transforms_in_use(self):
         """Transformers, excluding logical line transformers if we're in a
         Python line."""
         t = self.physical_line_transforms[:]
         if not self.within_python_line:
             t += [self.assemble_logical_lines] + self.logical_line_transforms
         return t + [self.assemble_python_lines] + self.python_line_transforms
 
     def reset(self):
         """Reset the input buffer and associated state."""
         super(IPythonInputSplitter, self).reset()
         self._buffer_raw[:] = []
         self.source_raw = ''
         self.transformer_accumulating = False
         self.within_python_line = False
 
         for t in self.transforms:
             try:
                 t.reset()
             except SyntaxError:
                 # Nothing that calls reset() expects to handle transformer
                 # errors
                 pass
     
     def flush_transformers(self):
         def _flush(transform, outs):
             """yield transformed lines
 
             always strings, never None
 
             transform: the current transform
             outs: an iterable of previously transformed inputs.
                  Each may be multiline, which will be passed
                  one line at a time to transform.
             """
             for out in outs:
                 for line in out.splitlines():
                     # push one line at a time
                     tmp = transform.push(line)
                     if tmp is not None:
                         yield tmp
             
             # reset the transform
             tmp = transform.reset()
             if tmp is not None:
                 yield tmp
         
         out = []
         for t in self.transforms_in_use:
             out = _flush(t, out)
         
         out = list(out)
         if out:
             self._store('\n'.join(out))
 
     def raw_reset(self):
         """Return raw input only and perform a full reset.
         """
         out = self.source_raw
         self.reset()
         return out
     
     def source_reset(self):
         try:
             self.flush_transformers()
             return self.source
         finally:
             self.reset()
 
     def push_accepts_more(self):
         if self.transformer_accumulating:
             return True
         else:
             return super(IPythonInputSplitter, self).push_accepts_more()
 
     def transform_cell(self, cell):
         """Process and translate a cell of input.
         """
         self.reset()
         try:
             self.push(cell)
             self.flush_transformers()
             return self.source
         finally:
             self.reset()
 
     def push(self, lines:str) -> bool:
         """Push one or more lines of IPython input.
 
         This stores the given lines and returns a status code indicating
         whether the code forms a complete Python block or not, after processing
         all input lines for special IPython syntax.
 
         Any exceptions generated in compilation are swallowed, but if an
         exception was produced, the method returns True.
 
         Parameters
         ----------
         lines : string
             One or more lines of Python input.
 
         Returns
         -------
         is_complete : boolean
             True if the current input source (the result of the current input
             plus prior inputs) forms a complete Python execution block.  Note that
             this value is also stored as a private attribute (_is_complete), so it
             can be queried at any time.
         """
         assert isinstance(lines, str)
         # We must ensure all input is pure unicode
         # ''.splitlines() --> [], but we need to push the empty line to transformers
         lines_list = lines.splitlines()
         if not lines_list:
             lines_list = ['']
 
         # Store raw source before applying any transformations to it.  Note
         # that this must be done *after* the reset() call that would otherwise
         # flush the buffer.
         self._store(lines, self._buffer_raw, 'source_raw')
 
         transformed_lines_list = []
         for line in lines_list:
             transformed = self._transform_line(line)
             if transformed is not None:
                 transformed_lines_list.append(transformed)
 
         if transformed_lines_list:
             transformed_lines = '\n'.join(transformed_lines_list)
             return super(IPythonInputSplitter, self).push(transformed_lines)
         else:
             # Got nothing back from transformers - they must be waiting for
             # more input.
             return False
 
     def _transform_line(self, line):
         """Push a line of input code through the various transformers.
 
         Returns any output from the transformers, or None if a transformer
         is accumulating lines.
 
         Sets self.transformer_accumulating as a side effect.
         """
         def _accumulating(dbg):
             #print(dbg)
             self.transformer_accumulating = True
             return None
 
         for transformer in self.physical_line_transforms:
             line = transformer.push(line)
             if line is None:
                 return _accumulating(transformer)
 
         if not self.within_python_line:
             line = self.assemble_logical_lines.push(line)
             if line is None:
                 return _accumulating('acc logical line')
 
             for transformer in self.logical_line_transforms:
                 line = transformer.push(line)
                 if line is None:
                     return _accumulating(transformer)
 
         line = self.assemble_python_lines.push(line)
         if line is None:
             self.within_python_line = True
             return _accumulating('acc python line')
         else:
             self.within_python_line = False
 
         for transformer in self.python_line_transforms:
             line = transformer.push(line)
             if line is None:
                 return _accumulating(transformer)
 
         #print("transformers clear") #debug
         self.transformer_accumulating = False
         return line
 

IPython/core/inputtransformer2.py

@@ -1,823 +1,827 b''
 """Input transformer machinery to support IPython special syntax.
 
 This includes the machinery to recognise and transform ``%magic`` commands,
 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
 
 Added: IPython 7.0. Replaces inputsplitter and inputtransformer which were
 deprecated in 7.0.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import ast
 from codeop import CommandCompiler, Compile
 import re
 import sys
 import tokenize
 from typing import List, Tuple, Optional, Any
 import warnings
 
 from IPython.utils import tokenutil
 
 _indent_re = re.compile(r'^[ \t]+')
 
 def leading_empty_lines(lines):
     """Remove leading empty lines
 
     If the leading lines are empty or contain only whitespace, they will be
     removed.
     """
     if not lines:
         return lines
     for i, line in enumerate(lines):
         if line and not line.isspace():
             return lines[i:]
     return lines
 
 def leading_indent(lines):
     """Remove leading indentation.
 
     If the first line starts with a spaces or tabs, the same whitespace will be
     removed from each following line in the cell.
     """
     if not lines:
         return lines
     m = _indent_re.match(lines[0])
     if not m:
         return lines
     space = m.group(0)
     n = len(space)
     return [l[n:] if l.startswith(space) else l
             for l in lines]
 
 class PromptStripper:
     """Remove matching input prompts from a block of input.
 
     Parameters
     ----------
     prompt_re : regular expression
         A regular expression matching any input prompt (including continuation,
         e.g. ``...``)
     initial_re : regular expression, optional
         A regular expression matching only the initial prompt, but not continuation.
         If no initial expression is given, prompt_re will be used everywhere.
         Used mainly for plain Python prompts (``>>>``), where the continuation prompt
         ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
 
     Notes
     -----
 
     If initial_re and prompt_re differ,
     only initial_re will be tested against the first line.
     If any prompt is found on the first two lines,
     prompts will be stripped from the rest of the block.
     """
     def __init__(self, prompt_re, initial_re=None):
         self.prompt_re = prompt_re
         self.initial_re = initial_re or prompt_re
 
     def _strip(self, lines):
         return [self.prompt_re.sub('', l, count=1) for l in lines]
 
     def __call__(self, lines):
         if not lines:
             return lines
         if self.initial_re.match(lines[0]) or \
                 (len(lines) > 1 and self.prompt_re.match(lines[1])):
             return self._strip(lines)
         return lines
 
 classic_prompt = PromptStripper(
     prompt_re=re.compile(r'^(>>>|\.\.\.)( |$)'),
     initial_re=re.compile(r'^>>>( |$)')
 )
 
 ipython_prompt = PromptStripper(
     re.compile(
         r"""
         ^(                         # Match from the beginning of a line, either:
 
                                    # 1. First-line prompt:
         ((\[nav\]|\[ins\])?\ )?    # Vi editing mode prompt, if it's there
         In\                        # The 'In' of the prompt, with a space
         \[\d+\]:                   # Command index, as displayed in the prompt
         \                          # With a mandatory trailing space
 
         |                          # ... or ...
 
                                    # 2. The three dots of the multiline prompt
         \s*                        # All leading whitespace characters
         \.{3,}:                    # The three (or more) dots
         \ ?                        # With an optional trailing space
 
         )
         """,
         re.VERBOSE,
     )
 )
 
 
 def cell_magic(lines):
     if not lines or not lines[0].startswith('%%'):
         return lines
     if re.match(r'%%\w+\?', lines[0]):
         # This case will be handled by help_end
         return lines
     magic_name, _, first_line = lines[0][2:].rstrip().partition(' ')
     body = ''.join(lines[1:])
     return ['get_ipython().run_cell_magic(%r, %r, %r)\n'
             % (magic_name, first_line, body)]
 
 
 def _find_assign_op(token_line) -> Optional[int]:
     """Get the index of the first assignment in the line ('=' not inside brackets)
 
     Note: We don't try to support multiple special assignment (a = b = %foo)
     """
     paren_level = 0
     for i, ti in enumerate(token_line):
         s = ti.string
         if s == '=' and paren_level == 0:
             return i
         if s in {'(','[','{'}:
             paren_level += 1
         elif s in {')', ']', '}'}:
             if paren_level > 0:
                 paren_level -= 1
     return None
 
 def find_end_of_continued_line(lines, start_line: int):
     """Find the last line of a line explicitly extended using backslashes.
 
     Uses 0-indexed line numbers.
     """
     end_line = start_line
     while lines[end_line].endswith('\\\n'):
         end_line += 1
         if end_line >= len(lines):
             break
     return end_line
 
 def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):
     r"""Assemble a single line from multiple continued line pieces
 
     Continued lines are lines ending in ``\``, and the line following the last
     ``\`` in the block.
 
     For example, this code continues over multiple lines::
 
         if (assign_ix is not None) \
              and (len(line) >= assign_ix + 2) \
              and (line[assign_ix+1].string == '%') \
              and (line[assign_ix+2].type == tokenize.NAME):
 
     This statement contains four continued line pieces.
     Assembling these pieces into a single line would give::
 
         if (assign_ix is not None) and (len(line) >= assign_ix + 2) and (line[...
 
     This uses 0-indexed line numbers. *start* is (lineno, colno).
 
     Used to allow ``%magic`` and ``!system`` commands to be continued over
     multiple lines.
     """
     parts = [lines[start[0]][start[1]:]] + lines[start[0]+1:end_line+1]
     return ' '.join([p.rstrip()[:-1] for p in parts[:-1]]  # Strip backslash+newline
                     + [parts[-1].rstrip()])         # Strip newline from last line
 
 class TokenTransformBase:
     """Base class for transformations which examine tokens.
 
     Special syntax should not be transformed when it occurs inside strings or
     comments. This is hard to reliably avoid with regexes. The solution is to
     tokenise the code as Python, and recognise the special syntax in the tokens.
 
     IPython's special syntax is not valid Python syntax, so tokenising may go
     wrong after the special syntax starts. These classes therefore find and
     transform *one* instance of special syntax at a time into regular Python
     syntax. After each transformation, tokens are regenerated to find the next
     piece of special syntax.
 
     Subclasses need to implement one class method (find)
     and one regular method (transform).
 
     The priority attribute can select which transformation to apply if multiple
     transformers match in the same place. Lower numbers have higher priority.
     This allows "%magic?" to be turned into a help call rather than a magic call.
     """
     # Lower numbers -> higher priority (for matches in the same location)
     priority = 10
 
     def sortby(self):
         return self.start_line, self.start_col, self.priority
 
     def __init__(self, start):
         self.start_line = start[0] - 1   # Shift from 1-index to 0-index
         self.start_col = start[1]
 
     @classmethod
     def find(cls, tokens_by_line):
         """Find one instance of special syntax in the provided tokens.
 
         Tokens are grouped into logical lines for convenience,
         so it is easy to e.g. look at the first token of each line.
         *tokens_by_line* is a list of lists of tokenize.TokenInfo objects.
 
         This should return an instance of its class, pointing to the start
         position it has found, or None if it found no match.
         """
         raise NotImplementedError
 
     def transform(self, lines: List[str]):
         """Transform one instance of special syntax found by ``find()``
 
         Takes a list of strings representing physical lines,
         returns a similar list of transformed lines.
         """
         raise NotImplementedError
 
 class MagicAssign(TokenTransformBase):
     """Transformer for assignments from magics (a = %foo)"""
     @classmethod
     def find(cls, tokens_by_line):
         """Find the first magic assignment (a = %foo) in the cell.
         """
         for line in tokens_by_line:
             assign_ix = _find_assign_op(line)
             if (assign_ix is not None) \
                     and (len(line) >= assign_ix + 2) \
                     and (line[assign_ix+1].string == '%') \
                     and (line[assign_ix+2].type == tokenize.NAME):
                 return cls(line[assign_ix+1].start)
 
     def transform(self, lines: List[str]):
         """Transform a magic assignment found by the ``find()`` classmethod.
         """
         start_line, start_col = self.start_line, self.start_col
         lhs = lines[start_line][:start_col]
         end_line = find_end_of_continued_line(lines, start_line)
         rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
         assert rhs.startswith('%'), rhs
         magic_name, _, args = rhs[1:].partition(' ')
 
         lines_before = lines[:start_line]
         call = "get_ipython().run_line_magic({!r}, {!r})".format(magic_name, args)
         new_line = lhs + call + '\n'
         lines_after = lines[end_line+1:]
 
         return lines_before + [new_line] + lines_after
 
 
 class SystemAssign(TokenTransformBase):
     """Transformer for assignments from system commands (a = !foo)"""
     @classmethod
     def find_pre_312(cls, tokens_by_line):
         for line in tokens_by_line:
             assign_ix = _find_assign_op(line)
             if (assign_ix is not None) \
                     and not line[assign_ix].line.strip().startswith('=') \
                     and (len(line) >= assign_ix + 2) \
                     and (line[assign_ix + 1].type == tokenize.ERRORTOKEN):
                 ix = assign_ix + 1
 
                 while ix < len(line) and line[ix].type == tokenize.ERRORTOKEN:
                     if line[ix].string == '!':
                         return cls(line[ix].start)
                     elif not line[ix].string.isspace():
                         break
                     ix += 1
 
     @classmethod
     def find_post_312(cls, tokens_by_line):
         for line in tokens_by_line:
             assign_ix = _find_assign_op(line)
-            if (assign_ix is not None) \
-                    and not line[assign_ix].line.strip().startswith('=') \
-                    and (len(line) >= assign_ix + 2) \
-                    and (line[assign_ix + 1].type == tokenize.OP) \
-                    and (line[assign_ix + 1].string == '!'):
+            if (
+                (assign_ix is not None)
+                and not line[assign_ix].line.strip().startswith("=")
+                and (len(line) >= assign_ix + 2)
+                and (line[assign_ix + 1].type == tokenize.OP)
+                and (line[assign_ix + 1].string == "!")
+            ):
                 return cls(line[assign_ix + 1].start)
 
     @classmethod
     def find(cls, tokens_by_line):
-        """Find the first system assignment (a = !foo) in the cell.
-        """
+        """Find the first system assignment (a = !foo) in the cell."""
         if sys.version_info < (3, 12):
             return cls.find_pre_312(tokens_by_line)
         return cls.find_post_312(tokens_by_line)
 
     def transform(self, lines: List[str]):
         """Transform a system assignment found by the ``find()`` classmethod.
         """
         start_line, start_col = self.start_line, self.start_col
 
         lhs = lines[start_line][:start_col]
         end_line = find_end_of_continued_line(lines, start_line)
         rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
         assert rhs.startswith('!'), rhs
         cmd = rhs[1:]
 
         lines_before = lines[:start_line]
         call = "get_ipython().getoutput({!r})".format(cmd)
         new_line = lhs + call + '\n'
         lines_after = lines[end_line + 1:]
 
         return lines_before + [new_line] + lines_after
 
 # The escape sequences that define the syntax transformations IPython will
 # apply to user input.  These can NOT be just changed here: many regular
 # expressions and other parts of the code may use their hardcoded values, and
 # for all intents and purposes they constitute the 'IPython syntax', so they
 # should be considered fixed.
 
 ESC_SHELL  = '!'     # Send line to underlying system shell
 ESC_SH_CAP = '!!'    # Send line to system shell and capture output
 ESC_HELP   = '?'     # Find information about object
 ESC_HELP2  = '??'    # Find extra-detailed information about object
 ESC_MAGIC  = '%'     # Call magic function
 ESC_MAGIC2 = '%%'    # Call cell-magic function
 ESC_QUOTE  = ','     # Split args on whitespace, quote each as string and call
 ESC_QUOTE2 = ';'     # Quote all args as a single string, call
 ESC_PAREN  = '/'     # Call first argument with rest of line as arguments
 
 ESCAPE_SINGLES = {'!', '?', '%', ',', ';', '/'}
 ESCAPE_DOUBLES = {'!!', '??'}  # %% (cell magic) is handled separately
 
 def _make_help_call(target, esc):
     """Prepares a pinfo(2)/psearch call from a target name and the escape
     (i.e. ? or ??)"""
     method  = 'pinfo2' if esc == '??' \
                 else 'psearch' if '*' in target \
                 else 'pinfo'
     arg = " ".join([method, target])
     #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
     t_magic_name, _, t_magic_arg_s = arg.partition(' ')
     t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
     return "get_ipython().run_line_magic(%r, %r)" % (t_magic_name, t_magic_arg_s)
 
 
 def _tr_help(content):
     """Translate lines escaped with: ?
 
     A naked help line should fire the intro help screen (shell.show_usage())
     """
     if not content:
         return 'get_ipython().show_usage()'
 
     return _make_help_call(content, '?')
 
 def _tr_help2(content):
     """Translate lines escaped with: ??
 
     A naked help line should fire the intro help screen (shell.show_usage())
     """
     if not content:
         return 'get_ipython().show_usage()'
 
     return _make_help_call(content, '??')
 
 def _tr_magic(content):
     "Translate lines escaped with a percent sign: %"
     name, _, args = content.partition(' ')
     return 'get_ipython().run_line_magic(%r, %r)' % (name, args)
 
 def _tr_quote(content):
     "Translate lines escaped with a comma: ,"
     name, _, args = content.partition(' ')
     return '%s("%s")' % (name, '", "'.join(args.split()) )
 
 def _tr_quote2(content):
     "Translate lines escaped with a semicolon: ;"
     name, _, args = content.partition(' ')
     return '%s("%s")' % (name, args)
 
 def _tr_paren(content):
     "Translate lines escaped with a slash: /"
     name, _, args = content.partition(' ')
     return '%s(%s)' % (name, ", ".join(args.split()))
 
 tr = { ESC_SHELL  : 'get_ipython().system({!r})'.format,
        ESC_SH_CAP : 'get_ipython().getoutput({!r})'.format,
        ESC_HELP   : _tr_help,
        ESC_HELP2  : _tr_help2,
        ESC_MAGIC  : _tr_magic,
        ESC_QUOTE  : _tr_quote,
        ESC_QUOTE2 : _tr_quote2,
        ESC_PAREN  : _tr_paren }
 
 class EscapedCommand(TokenTransformBase):
     """Transformer for escaped commands like %foo, !foo, or /foo"""
     @classmethod
     def find(cls, tokens_by_line):
         """Find the first escaped command (%foo, !foo, etc.) in the cell.
         """
         for line in tokens_by_line:
             if not line:
                 continue
             ix = 0
             ll = len(line)
             while ll > ix and line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
                 ix += 1
             if ix >= ll:
                 continue
             if line[ix].string in ESCAPE_SINGLES:
                 return cls(line[ix].start)
 
     def transform(self, lines):
         """Transform an escaped line found by the ``find()`` classmethod.
         """
         start_line, start_col = self.start_line, self.start_col
 
         indent = lines[start_line][:start_col]
         end_line = find_end_of_continued_line(lines, start_line)
         line = assemble_continued_line(lines, (start_line, start_col), end_line)
 
         if len(line) > 1 and line[:2] in ESCAPE_DOUBLES:
             escape, content = line[:2], line[2:]
         else:
             escape, content = line[:1], line[1:]
 
         if escape in tr:
             call = tr[escape](content)
         else:
             call = ''
 
         lines_before = lines[:start_line]
         new_line = indent + call + '\n'
         lines_after = lines[end_line + 1:]
 
         return lines_before + [new_line] + lines_after
 
 
 _help_end_re = re.compile(
     r"""(%{0,2}
     (?!\d)[\w*]+            # Variable name
     (\.(?!\d)[\w*]+|\[-?[0-9]+\])*       # .etc.etc or [0], we only support literal integers.
     )
     (\?\??)$                # ? or ??
     """,
     re.VERBOSE,
 )
 
 
 class HelpEnd(TokenTransformBase):
     """Transformer for help syntax: obj? and obj??"""
     # This needs to be higher priority (lower number) than EscapedCommand so
     # that inspecting magics (%foo?) works.
     priority = 5
 
     def __init__(self, start, q_locn):
         super().__init__(start)
         self.q_line = q_locn[0] - 1  # Shift from 1-indexed to 0-indexed
         self.q_col = q_locn[1]
 
     @classmethod
     def find(cls, tokens_by_line):
         """Find the first help command (foo?) in the cell.
         """
         for line in tokens_by_line:
             # Last token is NEWLINE; look at last but one
             if len(line) > 2 and line[-2].string == '?':
                 # Find the first token that's not INDENT/DEDENT
                 ix = 0
                 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
                     ix += 1
                 return cls(line[ix].start, line[-2].start)
 
     def transform(self, lines):
         """Transform a help command found by the ``find()`` classmethod.
         """
 
         piece = "".join(lines[self.start_line : self.q_line + 1])
         indent, content = piece[: self.start_col], piece[self.start_col :]
         lines_before = lines[: self.start_line]
         lines_after = lines[self.q_line + 1 :]
 
         m = _help_end_re.search(content)
         if not m:
             raise SyntaxError(content)
         assert m is not None, content
         target = m.group(1)
         esc = m.group(3)
 
 
         call = _make_help_call(target, esc)
         new_line = indent + call + '\n'
 
         return lines_before + [new_line] + lines_after
 
 def make_tokens_by_line(lines:List[str]):
     """Tokenize a series of lines and group tokens by line.
 
     The tokens for a multiline Python string or expression are grouped as one
     line. All lines except the last lines should keep their line ending ('\\n',
     '\\r\\n') for this to properly work. Use `.splitlines(keeplineending=True)`
     for example when passing block of text to this function.
 
     """
     # NL tokens are used inside multiline expressions, but also after blank
     # lines or comments. This is intentional - see https://bugs.python.org/issue17061
     # We want to group the former case together but split the latter, so we
     # track parentheses level, similar to the internals of tokenize.
 
     #   reexported from token on 3.7+
     NEWLINE, NL = tokenize.NEWLINE, tokenize.NL  # type: ignore
     tokens_by_line: List[List[Any]] = [[]]
     if len(lines) > 1 and not lines[0].endswith(("\n", "\r", "\r\n", "\x0b", "\x0c")):
         warnings.warn(
             "`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified",
             stacklevel=2,
         )
     parenlev = 0
     try:
-        for token in tokenutil.generate_tokens_catch_errors(iter(lines).__next__,
-                                                            extra_errors_to_catch=['expected EOF']):
+        for token in tokenutil.generate_tokens_catch_errors(
+            iter(lines).__next__, extra_errors_to_catch=["expected EOF"]
+        ):
             tokens_by_line[-1].append(token)
             if (token.type == NEWLINE) \
                     or ((token.type == NL) and (parenlev <= 0)):
                 tokens_by_line.append([])
             elif token.string in {'(', '[', '{'}:
                 parenlev += 1
             elif token.string in {')', ']', '}'}:
                 if parenlev > 0:
                     parenlev -= 1
     except tokenize.TokenError:
         # Input ended in a multiline string or expression. That's OK for us.
         pass
 
 
     if not tokens_by_line[-1]:
         tokens_by_line.pop()
 
 
     return tokens_by_line
 
 
 def has_sunken_brackets(tokens: List[tokenize.TokenInfo]):
     """Check if the depth of brackets in the list of tokens drops below 0"""
     parenlev = 0
     for token in tokens:
         if token.string in {"(", "[", "{"}:
             parenlev += 1
         elif token.string in {")", "]", "}"}:
             parenlev -= 1
             if parenlev < 0:
                 return True
     return False
 
 
 def show_linewise_tokens(s: str):
     """For investigation and debugging"""
     warnings.warn(
         "show_linewise_tokens is deprecated since IPython 8.6",
         DeprecationWarning,
         stacklevel=2,
     )
     if not s.endswith("\n"):
         s += "\n"
     lines = s.splitlines(keepends=True)
     for line in make_tokens_by_line(lines):
         print("Line -------")
         for tokinfo in line:
             print(" ", tokinfo)
 
 # Arbitrary limit to prevent getting stuck in infinite loops
 TRANSFORM_LOOP_LIMIT = 500
 
 class TransformerManager:
     """Applies various transformations to a cell or code block.
 
     The key methods for external use are ``transform_cell()``
     and ``check_complete()``.
     """
     def __init__(self):
         self.cleanup_transforms = [
             leading_empty_lines,
             leading_indent,
             classic_prompt,
             ipython_prompt,
         ]
         self.line_transforms = [
             cell_magic,
         ]
         self.token_transformers = [
             MagicAssign,
             SystemAssign,
             EscapedCommand,
             HelpEnd,
         ]
 
     def do_one_token_transform(self, lines):
         """Find and run the transform earliest in the code.
 
         Returns (changed, lines).
 
         This method is called repeatedly until changed is False, indicating
         that all available transformations are complete.
 
         The tokens following IPython special syntax might not be valid, so
         the transformed code is retokenised every time to identify the next
         piece of special syntax. Hopefully long code cells are mostly valid
         Python, not using lots of IPython special syntax, so this shouldn't be
         a performance issue.
         """
         tokens_by_line = make_tokens_by_line(lines)
         candidates = []
         for transformer_cls in self.token_transformers:
             transformer = transformer_cls.find(tokens_by_line)
             if transformer:
                 candidates.append(transformer)
 
         if not candidates:
             # Nothing to transform
             return False, lines
         ordered_transformers = sorted(candidates, key=TokenTransformBase.sortby)
         for transformer in ordered_transformers:
             try:
                 return True, transformer.transform(lines)
             except SyntaxError:
                 pass
         return False, lines
 
     def do_token_transforms(self, lines):
         for _ in range(TRANSFORM_LOOP_LIMIT):
             changed, lines = self.do_one_token_transform(lines)
             if not changed:
                 return lines
 
         raise RuntimeError("Input transformation still changing after "
                            "%d iterations. Aborting." % TRANSFORM_LOOP_LIMIT)
 
     def transform_cell(self, cell: str) -> str:
         """Transforms a cell of input code"""
         if not cell.endswith('\n'):
             cell += '\n'  # Ensure the cell has a trailing newline
         lines = cell.splitlines(keepends=True)
         for transform in self.cleanup_transforms + self.line_transforms:
             lines = transform(lines)
 
         lines = self.do_token_transforms(lines)
         return ''.join(lines)
 
     def check_complete(self, cell: str):
         """Return whether a block of code is ready to execute, or should be continued
 
         Parameters
         ----------
         cell : string
             Python input code, which can be multiline.
 
         Returns
         -------
         status : str
             One of 'complete', 'incomplete', or 'invalid' if source is not a
             prefix of valid code.
         indent_spaces : int or None
             The number of spaces by which to indent the next line of code. If
             status is not 'incomplete', this is None.
         """
         # Remember if the lines ends in a new line.
         ends_with_newline = False
         for character in reversed(cell):
             if character == '\n':
                 ends_with_newline = True
                 break
             elif character.strip():
                 break
             else:
                 continue
 
         if not ends_with_newline:
             # Append an newline for consistent tokenization
             # See https://bugs.python.org/issue33899
             cell += '\n'
 
         lines = cell.splitlines(keepends=True)
 
         if not lines:
             return 'complete', None
 
         for line in reversed(lines):
             if not line.strip():
                 continue
-            elif line.strip('\n').endswith('\\'):
-                return 'incomplete', find_last_indent(lines)
+            elif line.strip("\n").endswith("\\"):
+                return "incomplete", find_last_indent(lines)
             else:
                 break
 
         try:
             for transform in self.cleanup_transforms:
                 if not getattr(transform, 'has_side_effects', False):
                     lines = transform(lines)
         except SyntaxError:
             return 'invalid', None
 
         if lines[0].startswith('%%'):
             # Special case for cell magics - completion marked by blank line
             if lines[-1].strip():
                 return 'incomplete', find_last_indent(lines)
             else:
                 return 'complete', None
 
         try:
             for transform in self.line_transforms:
                 if not getattr(transform, 'has_side_effects', False):
                     lines = transform(lines)
             lines = self.do_token_transforms(lines)
         except SyntaxError:
             return 'invalid', None
 
         tokens_by_line = make_tokens_by_line(lines)
 
         # Bail if we got one line and there are more closing parentheses than
         # the opening ones
         if (
             len(lines) == 1
             and tokens_by_line
             and has_sunken_brackets(tokens_by_line[0])
         ):
             return "invalid", None
 
         if not tokens_by_line:
             return 'incomplete', find_last_indent(lines)
 
-        if (tokens_by_line[-1][-1].type != tokenize.ENDMARKER
-                and tokens_by_line[-1][-1].type != tokenize.ERRORTOKEN):
+        if (
+            tokens_by_line[-1][-1].type != tokenize.ENDMARKER
+            and tokens_by_line[-1][-1].type != tokenize.ERRORTOKEN
+        ):
             # We're in a multiline string or expression
             return 'incomplete', find_last_indent(lines)
 
         newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER} # type: ignore
 
         # Pop the last line which only contains DEDENTs and ENDMARKER
         last_token_line = None
         if {t.type for t in tokens_by_line[-1]} in [
             {tokenize.DEDENT, tokenize.ENDMARKER},
             {tokenize.ENDMARKER}
         ] and len(tokens_by_line) > 1:
             last_token_line = tokens_by_line.pop()
 
         while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types:
             tokens_by_line[-1].pop()
 
         if not tokens_by_line[-1]:
             return 'incomplete', find_last_indent(lines)
 
         if tokens_by_line[-1][-1].string == ':':
             # The last line starts a block (e.g. 'if foo:')
             ix = 0
             while tokens_by_line[-1][ix].type in {tokenize.INDENT, tokenize.DEDENT}:
                 ix += 1
 
             indent = tokens_by_line[-1][ix].start[1]
             return 'incomplete', indent + 4
 
         if tokens_by_line[-1][0].line.endswith('\\'):
             return 'incomplete', None
 
         # At this point, our checks think the code is complete (or invalid).
         # We'll use codeop.compile_command to check this with the real parser
         try:
             with warnings.catch_warnings():
                 warnings.simplefilter('error', SyntaxWarning)
                 res = compile_command(''.join(lines), symbol='exec')
         except (SyntaxError, OverflowError, ValueError, TypeError,
                 MemoryError, SyntaxWarning):
             return 'invalid', None
         else:
             if res is None:
                 return 'incomplete', find_last_indent(lines)
 
         if last_token_line and last_token_line[0].type == tokenize.DEDENT:
             if ends_with_newline:
                 return 'complete', None
             return 'incomplete', find_last_indent(lines)
 
         # If there's a blank line at the end, assume we're ready to execute
         if not lines[-1].strip():
             return 'complete', None
 
         return 'complete', None
 
 
 def find_last_indent(lines):
     m = _indent_re.match(lines[-1])
     if not m:
         return 0
     return len(m.group(0).replace('\t', ' '*4))
 
 
 class MaybeAsyncCompile(Compile):
     def __init__(self, extra_flags=0):
         super().__init__()
         self.flags |= extra_flags
 
 
 class MaybeAsyncCommandCompiler(CommandCompiler):
     def __init__(self, extra_flags=0):
         self.compiler = MaybeAsyncCompile(extra_flags=extra_flags)
 
 
 _extra_flags = ast.PyCF_ALLOW_TOP_LEVEL_AWAIT
 
 compile_command = MaybeAsyncCommandCompiler(extra_flags=_extra_flags)

IPython/core/tests/test_inputtransformer2.py

@@ -1,447 +1,448 b''
 """Tests for the token-based transformers in IPython.core.inputtransformer2
 
 Line-based transformers are the simpler ones; token-based transformers are
 more complex. See test_inputtransformer2_line for tests for line-based
 transformations.
 """
 import platform
 import string
 import sys
 from textwrap import dedent
 
 import pytest
 
 from IPython.core import inputtransformer2 as ipt2
 from IPython.core.inputtransformer2 import _find_assign_op, make_tokens_by_line
 
 MULTILINE_MAGIC = (
     """\
 a = f()
 %foo \\
 bar
 g()
 """.splitlines(
         keepends=True
     ),
     (2, 0),
     """\
 a = f()
 get_ipython().run_line_magic('foo', ' bar')
 g()
 """.splitlines(
         keepends=True
     ),
 )
 
 INDENTED_MAGIC = (
     """\
 for a in range(5):
     %ls
 """.splitlines(
         keepends=True
     ),
     (2, 4),
     """\
 for a in range(5):
     get_ipython().run_line_magic('ls', '')
 """.splitlines(
         keepends=True
     ),
 )
 
 CRLF_MAGIC = (
     ["a = f()\n", "%ls\r\n", "g()\n"],
     (2, 0),
     ["a = f()\n", "get_ipython().run_line_magic('ls', '')\n", "g()\n"],
 )
 
 MULTILINE_MAGIC_ASSIGN = (
     """\
 a = f()
 b = %foo \\
   bar
 g()
 """.splitlines(
         keepends=True
     ),
     (2, 4),
     """\
 a = f()
 b = get_ipython().run_line_magic('foo', '   bar')
 g()
 """.splitlines(
         keepends=True
     ),
 )
 
 MULTILINE_SYSTEM_ASSIGN = ("""\
 a = f()
 b = !foo \\
   bar
 g()
 """.splitlines(keepends=True), (2, 4), """\
 a = f()
 b = get_ipython().getoutput('foo    bar')
 g()
 """.splitlines(keepends=True))
 
 #####
 
 MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT = (
     """\
 def test():
   for i in range(1):
     print(i)
   res =! ls
 """.splitlines(
         keepends=True
     ),
     (4, 7),
     """\
 def test():
   for i in range(1):
     print(i)
   res =get_ipython().getoutput(\' ls\')
 """.splitlines(
         keepends=True
     ),
 )
 
 ######
 
 AUTOCALL_QUOTE = ([",f 1 2 3\n"], (1, 0), ['f("1", "2", "3")\n'])
 
 AUTOCALL_QUOTE2 = ([";f 1 2 3\n"], (1, 0), ['f("1 2 3")\n'])
 
 AUTOCALL_PAREN = (["/f 1 2 3\n"], (1, 0), ["f(1, 2, 3)\n"])
 
 SIMPLE_HELP = (["foo?\n"], (1, 0), ["get_ipython().run_line_magic('pinfo', 'foo')\n"])
 
 DETAILED_HELP = (
     ["foo??\n"],
     (1, 0),
     ["get_ipython().run_line_magic('pinfo2', 'foo')\n"],
 )
 
 MAGIC_HELP = (["%foo?\n"], (1, 0), ["get_ipython().run_line_magic('pinfo', '%foo')\n"])
 
 HELP_IN_EXPR = (
     ["a = b + c?\n"],
     (1, 0),
     ["get_ipython().run_line_magic('pinfo', 'c')\n"],
 )
 
 HELP_CONTINUED_LINE = (
     """\
 a = \\
 zip?
 """.splitlines(
         keepends=True
     ),
     (1, 0),
     [r"get_ipython().run_line_magic('pinfo', 'zip')" + "\n"],
 )
 
 HELP_MULTILINE = (
     """\
 (a,
 b) = zip?
 """.splitlines(
         keepends=True
     ),
     (1, 0),
     [r"get_ipython().run_line_magic('pinfo', 'zip')" + "\n"],
 )
 
 HELP_UNICODE = (
     ["π.foo?\n"],
     (1, 0),
     ["get_ipython().run_line_magic('pinfo', 'π.foo')\n"],
 )
 
 
 def null_cleanup_transformer(lines):
     """
     A cleanup transform that returns an empty list.
     """
     return []
 
 
 def test_check_make_token_by_line_never_ends_empty():
     """
     Check that not sequence of single or double characters ends up leading to en empty list of tokens
     """
     from string import printable
 
     for c in printable:
         assert make_tokens_by_line(c)[-1] != []
         for k in printable:
             assert make_tokens_by_line(c + k)[-1] != []
 
 
 def check_find(transformer, case, match=True):
     sample, expected_start, _ = case
     tbl = make_tokens_by_line(sample)
     res = transformer.find(tbl)
     if match:
         # start_line is stored 0-indexed, expected values are 1-indexed
         assert (res.start_line + 1, res.start_col) == expected_start
         return res
     else:
         assert res is None
 
 
 def check_transform(transformer_cls, case):
     lines, start, expected = case
     transformer = transformer_cls(start)
     assert transformer.transform(lines) == expected
 
 
 def test_continued_line():
     lines = MULTILINE_MAGIC_ASSIGN[0]
     assert ipt2.find_end_of_continued_line(lines, 1) == 2
 
     assert ipt2.assemble_continued_line(lines, (1, 5), 2) == "foo    bar"
 
 
 def test_find_assign_magic():
     check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
     check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False)
     check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT, match=False)
 
 
 def test_transform_assign_magic():
     check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
 
 
 def test_find_assign_system():
     check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
     check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT)
     check_find(ipt2.SystemAssign, (["a =  !ls\n"], (1, 5), None))
     check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None))
     check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False)
 
 
 def test_transform_assign_system():
     check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
     check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT)
 
 
 def test_find_magic_escape():
     check_find(ipt2.EscapedCommand, MULTILINE_MAGIC)
     check_find(ipt2.EscapedCommand, INDENTED_MAGIC)
     check_find(ipt2.EscapedCommand, MULTILINE_MAGIC_ASSIGN, match=False)
 
 
 def test_transform_magic_escape():
     check_transform(ipt2.EscapedCommand, MULTILINE_MAGIC)
     check_transform(ipt2.EscapedCommand, INDENTED_MAGIC)
     check_transform(ipt2.EscapedCommand, CRLF_MAGIC)
 
 
 def test_find_autocalls():
     for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
         print("Testing %r" % case[0])
         check_find(ipt2.EscapedCommand, case)
 
 
 def test_transform_autocall():
     for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
         print("Testing %r" % case[0])
         check_transform(ipt2.EscapedCommand, case)
 
 
 def test_find_help():
     for case in [SIMPLE_HELP, DETAILED_HELP, MAGIC_HELP, HELP_IN_EXPR]:
         check_find(ipt2.HelpEnd, case)
 
     tf = check_find(ipt2.HelpEnd, HELP_CONTINUED_LINE)
     assert tf.q_line == 1
     assert tf.q_col == 3
 
     tf = check_find(ipt2.HelpEnd, HELP_MULTILINE)
     assert tf.q_line == 1
     assert tf.q_col == 8
 
     # ? in a comment does not trigger help
     check_find(ipt2.HelpEnd, (["foo # bar?\n"], None, None), match=False)
     # Nor in a string
     check_find(ipt2.HelpEnd, (["foo = '''bar?\n"], None, None), match=False)
 
 
 def test_transform_help():
     tf = ipt2.HelpEnd((1, 0), (1, 9))
     assert tf.transform(HELP_IN_EXPR[0]) == HELP_IN_EXPR[2]
 
     tf = ipt2.HelpEnd((1, 0), (2, 3))
     assert tf.transform(HELP_CONTINUED_LINE[0]) == HELP_CONTINUED_LINE[2]
 
     tf = ipt2.HelpEnd((1, 0), (2, 8))
     assert tf.transform(HELP_MULTILINE[0]) == HELP_MULTILINE[2]
 
     tf = ipt2.HelpEnd((1, 0), (1, 0))
     assert tf.transform(HELP_UNICODE[0]) == HELP_UNICODE[2]
 
 
 def test_find_assign_op_dedent():
     """
     be careful that empty token like dedent are not counted as parens
     """
 
     class Tk:
         def __init__(self, s):
             self.string = s
 
     assert _find_assign_op([Tk(s) for s in ("", "a", "=", "b")]) == 2
     assert (
         _find_assign_op([Tk(s) for s in ("", "(", "a", "=", "b", ")", "=", "5")]) == 6
     )
 
+
 extra_closing_paren_param = (
     pytest.param("(\n))", "invalid", None)
     if sys.version_info >= (3, 12)
     else pytest.param("(\n))", "incomplete", 0)
 )
 examples = [
     pytest.param("a = 1", "complete", None),
     pytest.param("for a in range(5):", "incomplete", 4),
     pytest.param("for a in range(5):\n    if a > 0:", "incomplete", 8),
     pytest.param("raise = 2", "invalid", None),
     pytest.param("a = [1,\n2,", "incomplete", 0),
     extra_closing_paren_param,
     pytest.param("\\\r\n", "incomplete", 0),
     pytest.param("a = '''\n   hi", "incomplete", 3),
     pytest.param("def a():\n x=1\n global x", "invalid", None),
     pytest.param(
         "a \\ ",
         "invalid",
         None,
         marks=pytest.mark.xfail(
             reason="Bug in python 3.9.8 – bpo 45738",
             condition=sys.version_info
             in [(3, 9, 8, "final", 0), (3, 11, 0, "alpha", 2)],
             raises=SystemError,
             strict=True,
         ),
     ),  # Nothing allowed after backslash,
     pytest.param("1\\\n+2", "complete", None),
 ]
 
 
 @pytest.mark.parametrize("code, expected, number", examples)
 def test_check_complete_param(code, expected, number):
     cc = ipt2.TransformerManager().check_complete
     assert cc(code) == (expected, number)
 
 
 @pytest.mark.xfail(platform.python_implementation() == "PyPy", reason="fail on pypy")
 @pytest.mark.xfail(
     reason="Bug in python 3.9.8 – bpo 45738",
     condition=sys.version_info in [(3, 9, 8, "final", 0), (3, 11, 0, "alpha", 2)],
     raises=SystemError,
     strict=True,
 )
 def test_check_complete():
     cc = ipt2.TransformerManager().check_complete
 
     example = dedent(
         """
         if True:
             a=1"""
     )
 
     assert cc(example) == ("incomplete", 4)
     assert cc(example + "\n") == ("complete", None)
     assert cc(example + "\n    ") == ("complete", None)
 
     # no need to loop on all the letters/numbers.
     short = "12abAB" + string.printable[62:]
     for c in short:
         # test does not raise:
         cc(c)
         for k in short:
             cc(c + k)
 
     assert cc("def f():\n  x=0\n  \\\n  ") == ("incomplete", 2)
 
 
 @pytest.mark.xfail(platform.python_implementation() == "PyPy", reason="fail on pypy")
 @pytest.mark.parametrize(
     "value, expected",
     [
         ('''def foo():\n    """''', ("incomplete", 4)),
         ("""async with example:\n    pass""", ("incomplete", 4)),
         ("""async with example:\n    pass\n    """, ("complete", None)),
     ],
 )
 def test_check_complete_II(value, expected):
     """
     Test that multiple line strings are properly handled.
 
     Separate test function for convenience
 
     """
     cc = ipt2.TransformerManager().check_complete
     assert cc(value) == expected
 
 
 @pytest.mark.parametrize(
     "value, expected",
     [
         (")", ("invalid", None)),
         ("]", ("invalid", None)),
         ("}", ("invalid", None)),
         (")(", ("invalid", None)),
         ("][", ("invalid", None)),
         ("}{", ("invalid", None)),
         ("]()(", ("invalid", None)),
         ("())(", ("invalid", None)),
         (")[](", ("invalid", None)),
         ("()](", ("invalid", None)),
     ],
 )
 def test_check_complete_invalidates_sunken_brackets(value, expected):
     """
     Test that a single line with more closing brackets than the opening ones is
     interpreted as invalid
     """
     cc = ipt2.TransformerManager().check_complete
     assert cc(value) == expected
 
 
 def test_null_cleanup_transformer():
     manager = ipt2.TransformerManager()
     manager.cleanup_transforms.insert(0, null_cleanup_transformer)
     assert manager.transform_cell("") == ""
 
 
 def test_side_effects_I():
     count = 0
 
     def counter(lines):
         nonlocal count
         count += 1
         return lines
 
     counter.has_side_effects = True
 
     manager = ipt2.TransformerManager()
     manager.cleanup_transforms.insert(0, counter)
     assert manager.check_complete("a=1\n") == ("complete", None)
     assert count == 0
 
 
 def test_side_effects_II():
     count = 0
 
     def counter(lines):
         nonlocal count
         count += 1
         return lines
 
     counter.has_side_effects = True
 
     manager = ipt2.TransformerManager()
     manager.line_transforms.insert(0, counter)
     assert manager.check_complete("b=1\n") == ("complete", None)
     assert count == 0

IPython/utils/tokenutil.py

@@ -1,150 +1,155 b''
 """Token-related utilities"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from collections import namedtuple
 from io import StringIO
 from keyword import iskeyword
 
 import tokenize
 
 
 Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line'])
 
 def generate_tokens(readline):
     """wrap generate_tokens to catch EOF errors"""
     try:
         for token in tokenize.generate_tokens(readline):
             yield token
     except tokenize.TokenError:
         # catch EOF error
         return
 
+
 def generate_tokens_catch_errors(readline, extra_errors_to_catch=None):
-    default_errors_to_catch = ['unterminated string literal', 'invalid non-printable character',
-                               'after line continuation character']
+    default_errors_to_catch = [
+        "unterminated string literal",
+        "invalid non-printable character",
+        "after line continuation character",
+    ]
     assert extra_errors_to_catch is None or isinstance(extra_errors_to_catch, list)
     errors_to_catch = default_errors_to_catch + (extra_errors_to_catch or [])
 
     tokens = []
     try:
         for token in tokenize.generate_tokens(readline):
             tokens.append(token)
             yield token
     except tokenize.TokenError as exc:
         if any(error in exc.args[0] for error in errors_to_catch):
             if tokens:
                 start = tokens[-1].start[0], tokens[-1].end[0]
                 end = start
                 line = tokens[-1].line
             else:
                 start = end = (1, 0)
-                line = ''
-            yield tokenize.TokenInfo(tokenize.ERRORTOKEN, '', start, end, line)
+                line = ""
+            yield tokenize.TokenInfo(tokenize.ERRORTOKEN, "", start, end, line)
         else:
             # Catch EOF
             raise
 
+
 def line_at_cursor(cell, cursor_pos=0):
     """Return the line in a cell at a given cursor position
 
     Used for calling line-based APIs that don't support multi-line input, yet.
 
     Parameters
     ----------
     cell : str
         multiline block of text
     cursor_pos : integer
         the cursor position
 
     Returns
     -------
     (line, offset): (string, integer)
         The line with the current cursor, and the character offset of the start of the line.
     """
     offset = 0
     lines = cell.splitlines(True)
     for line in lines:
         next_offset = offset + len(line)
         if not line.endswith('\n'):
             # If the last line doesn't have a trailing newline, treat it as if
             # it does so that the cursor at the end of the line still counts
             # as being on that line.
             next_offset += 1
         if next_offset > cursor_pos:
             break
         offset = next_offset
     else:
         line = ""
     return (line, offset)
 
 def token_at_cursor(cell, cursor_pos=0):
     """Get the token at a given cursor
 
     Used for introspection.
 
     Function calls are prioritized, so the token for the callable will be returned
     if the cursor is anywhere inside the call.
 
     Parameters
     ----------
     cell : unicode
         A block of Python code
     cursor_pos : int
         The location of the cursor in the block where the token should be found
     """
     names = []
     tokens = []
     call_names = []
     
     offsets = {1: 0} # lines start at 1
     for tup in generate_tokens(StringIO(cell).readline):
         
         tok = Token(*tup)
         
         # token, text, start, end, line = tup
         start_line, start_col = tok.start
         end_line, end_col = tok.end
         if end_line + 1 not in offsets:
             # keep track of offsets for each line
             lines = tok.line.splitlines(True)
             for lineno, line in enumerate(lines, start_line + 1):
                 if lineno not in offsets:
                     offsets[lineno] = offsets[lineno-1] + len(line)
         
         offset = offsets[start_line]
         # allow '|foo' to find 'foo' at the beginning of a line
         boundary = cursor_pos + 1 if start_col == 0 else cursor_pos
         if offset + start_col >= boundary:
             # current token starts after the cursor,
             # don't consume it
             break
         
         if tok.token == tokenize.NAME and not iskeyword(tok.text):
             if names and tokens and tokens[-1].token == tokenize.OP and tokens[-1].text == '.':
                 names[-1] = "%s.%s" % (names[-1], tok.text)
             else:
                 names.append(tok.text)
         elif tok.token == tokenize.OP:
             if tok.text == '=' and names:
                 # don't inspect the lhs of an assignment
                 names.pop(-1)
             if tok.text == '(' and names:
                 # if we are inside a function call, inspect the function
                 call_names.append(names[-1])
             elif tok.text == ')' and call_names:
                 call_names.pop(-1)
         
         tokens.append(tok)
         
         if offsets[end_line] + end_col > cursor_pos:
             # we found the cursor, stop reading
             break
         
     if call_names:
         return call_names[-1]
     elif names:
         return names[-1]
     else:
         return ''