Deprecate inputtransformer since 7.0

Actually emit deprecation warnings when using inputtransformer which is
deprecated.

IPython/core/inputtransformer.py

@@ -1,544 +1,577
 """DEPRECATED: Input transformer classes to support IPython special syntax.
 
 This module was deprecated in IPython 7.0, in favour of inputtransformer2.
 
 This includes the machinery to recognise and transform ``%magic`` commands,
 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
 """
 import abc
 import functools
 import re
 import tokenize
+import warnings
 from tokenize import untokenize, TokenError
 from io import StringIO
 
 from IPython.core.splitinput import LineInfo
 from IPython.utils import tokenutil
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # The escape sequences that define the syntax transformations IPython will
 # apply to user input.  These can NOT be just changed here: many regular
 # expressions and other parts of the code may use their hardcoded values, and
 # for all intents and purposes they constitute the 'IPython syntax', so they
 # should be considered fixed.
 
 ESC_SHELL  = '!'     # Send line to underlying system shell
 ESC_SH_CAP = '!!'    # Send line to system shell and capture output
 ESC_HELP   = '?'     # Find information about object
 ESC_HELP2  = '??'    # Find extra-detailed information about object
 ESC_MAGIC  = '%'     # Call magic function
 ESC_MAGIC2 = '%%'    # Call cell-magic function
 ESC_QUOTE  = ','     # Split args on whitespace, quote each as string and call
 ESC_QUOTE2 = ';'     # Quote all args as a single string, call
 ESC_PAREN  = '/'     # Call first argument with rest of line as arguments
 
 ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\
                  ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\
                  ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ]
 
 
 class InputTransformer(metaclass=abc.ABCMeta):
     """Abstract base class for line-based input transformers."""
-    
+
+    def __init__(self):
+        warnings.warn(
+            "`InputTransformer` has been deprecated since IPython 7.0"
+            " and emit a warnig since IPython 8.31, it"
+            " will be removed in the future",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
     @abc.abstractmethod
     def push(self, line):
         """Send a line of input to the transformer, returning the transformed
         input or None if the transformer is waiting for more input.
 
         Must be overridden by subclasses.
 
         Implementations may raise ``SyntaxError`` if the input is invalid. No
         other exceptions may be raised.
         """
         pass
     
     @abc.abstractmethod
     def reset(self):
         """Return, transformed any lines that the transformer has accumulated,
         and reset its internal state.
 
         Must be overridden by subclasses.
         """
         pass
     
     @classmethod
     def wrap(cls, func):
         """Can be used by subclasses as a decorator, to return a factory that
         will allow instantiation with the decorated object.
         """
         @functools.wraps(func)
         def transformer_factory(**kwargs):
             return cls(func, **kwargs)  # type: ignore [call-arg]
 
         return transformer_factory
 
 class StatelessInputTransformer(InputTransformer):
     """Wrapper for a stateless input transformer implemented as a function."""
     def __init__(self, func):
+        super().__init__()
+        warnings.warn(
+            "`StatelessInputTransformer` has been deprecated since IPython 7.0"
+            " and emit a warnig since IPython 8.31, it"
+            " will be removed in the future",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         self.func = func
     
     def __repr__(self):
         return "StatelessInputTransformer(func={0!r})".format(self.func)
     
     def push(self, line):
         """Send a line of input to the transformer, returning the
         transformed input."""
         return self.func(line)
     
     def reset(self):
         """No-op - exists for compatibility."""
         pass
 
 class CoroutineInputTransformer(InputTransformer):
     """Wrapper for an input transformer implemented as a coroutine."""
     def __init__(self, coro, **kwargs):
         # Prime it
+        super().__init__()
+        warnings.warn(
+            "`CoroutineInputTransformer` has been deprecated since IPython 7.0"
+            " and emit a warnig since IPython 8.31, it"
+            " will be removed in the future",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         self.coro = coro(**kwargs)
         next(self.coro)
     
     def __repr__(self):
         return "CoroutineInputTransformer(coro={0!r})".format(self.coro)
     
     def push(self, line):
         """Send a line of input to the transformer, returning the
         transformed input or None if the transformer is waiting for more
         input.
         """
         return self.coro.send(line)
     
     def reset(self):
         """Return, transformed any lines that the transformer has
         accumulated, and reset its internal state.
         """
         return self.coro.send(None)
 
 class TokenInputTransformer(InputTransformer):
     """Wrapper for a token-based input transformer.
     
     func should accept a list of tokens (5-tuples, see tokenize docs), and
     return an iterable which can be passed to tokenize.untokenize().
     """
     def __init__(self, func):
+        warnings.warn(
+            "`CoroutineInputTransformer` has been deprecated since IPython 7.0"
+            " and emit a warnig since IPython 8.31, it"
+            " will be removed in the future",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         self.func = func
         self.buf = []
         self.reset_tokenizer()
 
     def reset_tokenizer(self):
         it = iter(self.buf)
         self.tokenizer = tokenutil.generate_tokens_catch_errors(it.__next__)
 
     def push(self, line):
         self.buf.append(line + '\n')
         if all(l.isspace() for l in self.buf):
             return self.reset()
 
         tokens = []
         stop_at_NL = False
         try:
             for intok in self.tokenizer:
                 tokens.append(intok)
                 t = intok[0]
                 if t == tokenize.NEWLINE or (stop_at_NL and t == tokenize.NL):
                     # Stop before we try to pull a line we don't have yet
                     break
                 elif t == tokenize.ERRORTOKEN:
                     stop_at_NL = True
         except TokenError:
             # Multi-line statement - stop and try again with the next line
             self.reset_tokenizer()
             return None
         
         return self.output(tokens)
     
     def output(self, tokens):
         self.buf.clear()
         self.reset_tokenizer()
         return untokenize(self.func(tokens)).rstrip('\n')
     
     def reset(self):
         l = ''.join(self.buf)
         self.buf.clear()
         self.reset_tokenizer()
         if l:
             return l.rstrip('\n')
 
 class assemble_python_lines(TokenInputTransformer):
     def __init__(self):
-        super(assemble_python_lines, self).__init__(None)
+        super().__init__(None)
     
     def output(self, tokens):
         return self.reset()
 
 @CoroutineInputTransformer.wrap
 def assemble_logical_lines():
     r"""Join lines following explicit line continuations (\)"""
     line = ''
     while True:
         line = (yield line)
         if not line or line.isspace():
             continue
         
         parts = []
         while line is not None:
             if line.endswith('\\') and (not has_comment(line)):
                 parts.append(line[:-1])
                 line = (yield None) # Get another line
             else:
                 parts.append(line)
                 break
         
         # Output
         line = ''.join(parts)
 
 # Utilities
 def _make_help_call(target: str, esc: str, lspace: str) -> str:
     """Prepares a pinfo(2)/psearch call from a target name and the escape
     (i.e. ? or ??)"""
     method  = 'pinfo2' if esc == '??' \
                 else 'psearch' if '*' in target \
                 else 'pinfo'
     arg = " ".join([method, target])
     #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
     t_magic_name, _, t_magic_arg_s = arg.partition(' ')
     t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
     return "%sget_ipython().run_line_magic(%r, %r)" % (
         lspace,
         t_magic_name,
         t_magic_arg_s,
     )
 
 
 # These define the transformations for the different escape characters.
 def _tr_system(line_info: LineInfo):
     "Translate lines escaped with: !"
     cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
     return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
 
 
 def _tr_system2(line_info: LineInfo):
     "Translate lines escaped with: !!"
     cmd = line_info.line.lstrip()[2:]
     return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
 
 
 def _tr_help(line_info: LineInfo):
     "Translate lines escaped with: ?/??"
     # A naked help line should just fire the intro help screen
     if not line_info.line[1:]:
         return 'get_ipython().show_usage()'
 
     return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
 
 
 def _tr_magic(line_info: LineInfo):
     "Translate lines escaped with: %"
     tpl = '%sget_ipython().run_line_magic(%r, %r)'
     if line_info.line.startswith(ESC_MAGIC2):
         return line_info.line
     cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
     #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
     t_magic_name, _, t_magic_arg_s = cmd.partition(' ')
     t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
     return tpl % (line_info.pre, t_magic_name, t_magic_arg_s)
 
 
 def _tr_quote(line_info: LineInfo):
     "Translate lines escaped with: ,"
     return '%s%s("%s")' % (line_info.pre, line_info.ifun,
                          '", "'.join(line_info.the_rest.split()) )
 
 
 def _tr_quote2(line_info: LineInfo):
     "Translate lines escaped with: ;"
     return '%s%s("%s")' % (line_info.pre, line_info.ifun,
                            line_info.the_rest)
 
 
 def _tr_paren(line_info: LineInfo):
     "Translate lines escaped with: /"
     return '%s%s(%s)' % (line_info.pre, line_info.ifun,
                          ", ".join(line_info.the_rest.split()))
 
 tr = { ESC_SHELL  : _tr_system,
        ESC_SH_CAP : _tr_system2,
        ESC_HELP   : _tr_help,
        ESC_HELP2  : _tr_help,
        ESC_MAGIC  : _tr_magic,
        ESC_QUOTE  : _tr_quote,
        ESC_QUOTE2 : _tr_quote2,
        ESC_PAREN  : _tr_paren }
 
 @StatelessInputTransformer.wrap
 def escaped_commands(line: str):
     """Transform escaped commands - %magic, !system, ?help + various autocalls."""
     if not line or line.isspace():
         return line
     lineinf = LineInfo(line)
     if lineinf.esc not in tr:
         return line
     
     return tr[lineinf.esc](lineinf)
 
 _initial_space_re = re.compile(r'\s*')
 
 _help_end_re = re.compile(r"""(%{0,2}
                               (?!\d)[\w*]+            # Variable name
                               (\.(?!\d)[\w*]+)*       # .etc.etc
                               )
                               (\?\??)$                # ? or ??
                               """,
                               re.VERBOSE)
 
 # Extra pseudotokens for multiline strings and data structures
 _MULTILINE_STRING = object()
 _MULTILINE_STRUCTURE = object()
 
 def _line_tokens(line):
     """Helper for has_comment and ends_in_comment_or_string."""
     readline = StringIO(line).readline
     toktypes = set()
     try:
         for t in tokenutil.generate_tokens_catch_errors(readline):
             toktypes.add(t[0])
     except TokenError as e:
         # There are only two cases where a TokenError is raised.
         if 'multi-line string' in e.args[0]:
             toktypes.add(_MULTILINE_STRING)
         else:
             toktypes.add(_MULTILINE_STRUCTURE)
     return toktypes
 
 def has_comment(src):
     """Indicate whether an input line has (i.e. ends in, or is) a comment.
 
     This uses tokenize, so it can distinguish comments from # inside strings.
 
     Parameters
     ----------
     src : string
         A single line input string.
 
     Returns
     -------
     comment : bool
         True if source has a comment.
     """
     return (tokenize.COMMENT in _line_tokens(src))
 
 def ends_in_comment_or_string(src):
     """Indicates whether or not an input line ends in a comment or within
     a multiline string.
 
     Parameters
     ----------
     src : string
         A single line input string.
 
     Returns
     -------
     comment : bool
         True if source ends in a comment or multiline string.
     """
     toktypes = _line_tokens(src)
     return (tokenize.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes)
         
 
 @StatelessInputTransformer.wrap
 def help_end(line: str):
     """Translate lines with ?/?? at the end"""
     m = _help_end_re.search(line)
     if m is None or ends_in_comment_or_string(line):
         return line
     target = m.group(1)
     esc = m.group(3)
     match = _initial_space_re.match(line)
     assert match is not None
     lspace = match.group(0)
 
     return _make_help_call(target, esc, lspace)
 
 
 @CoroutineInputTransformer.wrap
 def cellmagic(end_on_blank_line: bool = False):
     """Captures & transforms cell magics.
 
     After a cell magic is started, this stores up any lines it gets until it is
     reset (sent None).
     """
     tpl = 'get_ipython().run_cell_magic(%r, %r, %r)'
     cellmagic_help_re = re.compile(r'%%\w+\?')
     line = ''
     while True:
         line = (yield line)
         # consume leading empty lines
         while not line:
             line = (yield line)
         
         if not line.startswith(ESC_MAGIC2):
             # This isn't a cell magic, idle waiting for reset then start over
             while line is not None:
                 line = (yield line)
             continue
         
         if cellmagic_help_re.match(line):
             # This case will be handled by help_end
             continue
         
         first = line
         body = []
         line = (yield None)
         while (line is not None) and \
                                 ((line.strip() != '') or not end_on_blank_line):
             body.append(line)
             line = (yield None)
         
         # Output
         magic_name, _, first = first.partition(' ')
         magic_name = magic_name.lstrip(ESC_MAGIC2)
         line = tpl % (magic_name, first, u'\n'.join(body))
 
 
 def _strip_prompts(prompt_re, initial_re=None, turnoff_re=None):
     """Remove matching input prompts from a block of input.
 
     Parameters
     ----------
     prompt_re : regular expression
         A regular expression matching any input prompt (including continuation)
     initial_re : regular expression, optional
         A regular expression matching only the initial prompt, but not continuation.
         If no initial expression is given, prompt_re will be used everywhere.
         Used mainly for plain Python prompts, where the continuation prompt
         ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
 
     Notes
     -----
     If `initial_re` and `prompt_re differ`,
     only `initial_re` will be tested against the first line.
     If any prompt is found on the first two lines,
     prompts will be stripped from the rest of the block.
     """
     if initial_re is None:
         initial_re = prompt_re
     line = ''
     while True:
         line = (yield line)
         
         # First line of cell
         if line is None:
             continue
         out, n1 = initial_re.subn('', line, count=1)
         if turnoff_re and not n1:
             if turnoff_re.match(line):
                 # We're in e.g. a cell magic; disable this transformer for
                 # the rest of the cell.
                 while line is not None:
                     line = (yield line)
                 continue
 
         line = (yield out)
         
         if line is None:
             continue
         # check for any prompt on the second line of the cell,
         # because people often copy from just after the first prompt,
         # so we might not see it in the first line.
         out, n2 = prompt_re.subn('', line, count=1)
         line = (yield out)
         
         if n1 or n2:
             # Found a prompt in the first two lines - check for it in
             # the rest of the cell as well.
             while line is not None:
                 line = (yield prompt_re.sub('', line, count=1))
         
         else:
             # Prompts not in input - wait for reset
             while line is not None:
                 line = (yield line)
 
 @CoroutineInputTransformer.wrap
 def classic_prompt():
     """Strip the >>>/... prompts of the Python interactive shell."""
     # FIXME: non-capturing version (?:...) usable?
     prompt_re = re.compile(r'^(>>>|\.\.\.)( |$)')
     initial_re = re.compile(r'^>>>( |$)')
     # Any %magic/!system is IPython syntax, so we needn't look for >>> prompts
     turnoff_re = re.compile(r'^[%!]')
     return _strip_prompts(prompt_re, initial_re, turnoff_re)
 
 @CoroutineInputTransformer.wrap
 def ipy_prompt():
     """Strip IPython's In [1]:/...: prompts."""
     # FIXME: non-capturing version (?:...) usable?
     prompt_re = re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)')
     # Disable prompt stripping inside cell magics
     turnoff_re = re.compile(r'^%%')
     return _strip_prompts(prompt_re, turnoff_re=turnoff_re)
 
 
 @CoroutineInputTransformer.wrap
 def leading_indent():
     """Remove leading indentation.
 
     If the first line starts with a spaces or tabs, the same whitespace will be
     removed from each following line until it is reset.
     """
     space_re = re.compile(r'^[ \t]+')
     line = ''
     while True:
         line = (yield line)
         
         if line is None:
             continue
         
         m = space_re.match(line)
         if m:
             space = m.group(0)
             while line is not None:
                 if line.startswith(space):
                     line = line[len(space):]
                 line = (yield line)
         else:
             # No leading spaces - wait for reset
             while line is not None:
                 line = (yield line)
 
 
 _assign_pat = \
 r'''(?P<lhs>(\s*)
     ([\w\.]+)                # Initial identifier
     (\s*,\s*
         \*?[\w\.]+)*         # Further identifiers for unpacking
     \s*?,?                   # Trailing comma
     )
     \s*=\s*
 '''
 
 assign_system_re = re.compile(r'{}!\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE)
 assign_system_template = '%s = get_ipython().getoutput(%r)'
 @StatelessInputTransformer.wrap
 def assign_from_system(line):
     """Transform assignment from system commands (e.g. files = !ls)"""
     m = assign_system_re.match(line)
     if m is None:
         return line
     
     return assign_system_template % m.group('lhs', 'cmd')
 
 assign_magic_re = re.compile(r'{}%\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE)
 assign_magic_template = '%s = get_ipython().run_line_magic(%r, %r)'
 @StatelessInputTransformer.wrap
 def assign_from_magic(line):
     """Transform assignment from magic commands (e.g. a = %who_ls)"""
     m = assign_magic_re.match(line)
     if m is None:
         return line
     #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
     m_lhs, m_cmd = m.group('lhs', 'cmd')
     t_magic_name, _, t_magic_arg_s = m_cmd.partition(' ')
     t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
     return assign_magic_template % (m_lhs, t_magic_name, t_magic_arg_s)

IPython/core/tests/test_inputsplitter.py

@@ -1,643 +1,637
 # -*- coding: utf-8 -*-
 """Tests for the inputsplitter module."""
 
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import unittest
 import pytest
 import sys
 
 with pytest.warns(DeprecationWarning, match="inputsplitter"):
     from IPython.core import inputsplitter as isp
 from IPython.core.inputtransformer import InputTransformer
 from IPython.core.tests.test_inputtransformer import syntax, syntax_ml
 from IPython.testing import tools as tt
 
 #-----------------------------------------------------------------------------
 # Semi-complete examples (also used as tests)
 #-----------------------------------------------------------------------------
 
 # Note: at the bottom, there's a slightly more complete version of this that
 # can be useful during development of code here.
 
 def mini_interactive_loop(input_func):
     """Minimal example of the logic of an interactive interpreter loop.
 
     This serves as an example, and it is used by the test system with a fake
     raw_input that simulates interactive input."""
 
     from IPython.core.inputsplitter import InputSplitter
 
     isp = InputSplitter()
     # In practice, this input loop would be wrapped in an outside loop to read
     # input indefinitely, until some exit/quit command was issued.  Here we
     # only illustrate the basic inner loop.
     while isp.push_accepts_more():
         indent = ' '*isp.get_indent_spaces()
         prompt = '>>> ' + indent
         line = indent + input_func(prompt)
         isp.push(line)
 
     # Here we just return input so we can use it in a test suite, but a real
     # interpreter would instead send it for execution somewhere.
     src = isp.source_reset()
     # print('Input source was:\n', src)  # dbg
     return src
 
 #-----------------------------------------------------------------------------
 # Test utilities, just for local use
 #-----------------------------------------------------------------------------
 
 
 def pseudo_input(lines):
     """Return a function that acts like raw_input but feeds the input list."""
     ilines = iter(lines)
     def raw_in(prompt):
         try:
             return next(ilines)
         except StopIteration:
             return ''
     return raw_in
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 def test_spaces():
     tests = [('', 0),
              (' ', 1),
              ('\n', 0),
              (' \n', 1),
              ('x', 0),
              (' x', 1),
              ('  x',2),
              ('    x',4),
              # Note: tabs are counted as a single whitespace!
              ('\tx', 1),
              ('\t x', 2),
              ]
     with pytest.warns(PendingDeprecationWarning):
         tt.check_pairs(isp.num_ini_spaces, tests)
 
 
 def test_remove_comments():
     tests = [('text', 'text'),
              ('text # comment', 'text '),
              ('text # comment\n', 'text \n'),
              ('text # comment \n', 'text \n'),
              ('line # c \nline\n','line \nline\n'),
              ('line # c \nline#c2  \nline\nline #c\n\n',
               'line \nline\nline\nline \n\n'),
              ]
     tt.check_pairs(isp.remove_comments, tests)
 
 
 def test_get_input_encoding():
     encoding = isp.get_input_encoding()
     assert isinstance(encoding, str)
     # simple-minded check that at least encoding a simple string works with the
     # encoding we got.
     assert "test".encode(encoding) == b"test"
 
 
 class NoInputEncodingTestCase(unittest.TestCase):
     def setUp(self):
         self.old_stdin = sys.stdin
         class X: pass
         fake_stdin = X()
         sys.stdin = fake_stdin
 
     def test(self):
         # Verify that if sys.stdin has no 'encoding' attribute we do the right
         # thing
         enc = isp.get_input_encoding()
         self.assertEqual(enc, 'ascii')
 
     def tearDown(self):
         sys.stdin = self.old_stdin
 
 
 class InputSplitterTestCase(unittest.TestCase):
     def setUp(self):
         self.isp = isp.InputSplitter()
 
     def test_reset(self):
         isp = self.isp
         isp.push('x=1')
         isp.reset()
         self.assertEqual(isp._buffer, [])
         self.assertEqual(isp.get_indent_spaces(), 0)
         self.assertEqual(isp.source, '')
         self.assertEqual(isp.code, None)
         self.assertEqual(isp._is_complete, False)
 
     def test_source(self):
         self.isp._store('1')
         self.isp._store('2')
         self.assertEqual(self.isp.source, '1\n2\n')
         self.assertEqual(len(self.isp._buffer)>0, True)
         self.assertEqual(self.isp.source_reset(), '1\n2\n')
         self.assertEqual(self.isp._buffer, [])
         self.assertEqual(self.isp.source, '')
 
     def test_indent(self):
         isp = self.isp # shorthand
         isp.push('x=1')
         self.assertEqual(isp.get_indent_spaces(), 0)
         isp.push('if 1:\n    x=1')
         self.assertEqual(isp.get_indent_spaces(), 4)
         isp.push('y=2\n')
         self.assertEqual(isp.get_indent_spaces(), 0)
 
     def test_indent2(self):
         isp = self.isp
         isp.push('if 1:')
         self.assertEqual(isp.get_indent_spaces(), 4)
         isp.push('    x=1')
         self.assertEqual(isp.get_indent_spaces(), 4)
         # Blank lines shouldn't change the indent level
         isp.push(' '*2)
         self.assertEqual(isp.get_indent_spaces(), 4)
 
     def test_indent3(self):
         isp = self.isp
         # When a multiline statement contains parens or multiline strings, we
         # shouldn't get confused.
         isp.push("if 1:")
         isp.push("    x = (1+\n    2)")
         self.assertEqual(isp.get_indent_spaces(), 4)
 
     def test_indent4(self):
         isp = self.isp
         # whitespace after ':' should not screw up indent level
         isp.push('if 1: \n    x=1')
         self.assertEqual(isp.get_indent_spaces(), 4)
         isp.push('y=2\n')
         self.assertEqual(isp.get_indent_spaces(), 0)
         isp.push('if 1:\t\n    x=1')
         self.assertEqual(isp.get_indent_spaces(), 4)
         isp.push('y=2\n')
         self.assertEqual(isp.get_indent_spaces(), 0)
 
     def test_dedent_pass(self):
         isp = self.isp # shorthand
         # should NOT cause dedent
         isp.push('if 1:\n    passes = 5')
         self.assertEqual(isp.get_indent_spaces(), 4)
         isp.push('if 1:\n     pass')
         self.assertEqual(isp.get_indent_spaces(), 0)
         isp.push('if 1:\n     pass   ')
         self.assertEqual(isp.get_indent_spaces(), 0)
 
     def test_dedent_break(self):
         isp = self.isp # shorthand
         # should NOT cause dedent
         isp.push('while 1:\n    breaks = 5')
         self.assertEqual(isp.get_indent_spaces(), 4)
         isp.push('while 1:\n     break')
         self.assertEqual(isp.get_indent_spaces(), 0)
         isp.push('while 1:\n     break   ')
         self.assertEqual(isp.get_indent_spaces(), 0)
 
     def test_dedent_continue(self):
         isp = self.isp # shorthand
         # should NOT cause dedent
         isp.push('while 1:\n    continues = 5')
         self.assertEqual(isp.get_indent_spaces(), 4)
         isp.push('while 1:\n     continue')
         self.assertEqual(isp.get_indent_spaces(), 0)
         isp.push('while 1:\n     continue   ')
         self.assertEqual(isp.get_indent_spaces(), 0)
 
     def test_dedent_raise(self):
         isp = self.isp # shorthand
         # should NOT cause dedent
         isp.push('if 1:\n    raised = 4')
         self.assertEqual(isp.get_indent_spaces(), 4)
         isp.push('if 1:\n     raise TypeError()')
         self.assertEqual(isp.get_indent_spaces(), 0)
         isp.push('if 1:\n     raise')
         self.assertEqual(isp.get_indent_spaces(), 0)
         isp.push('if 1:\n     raise      ')
         self.assertEqual(isp.get_indent_spaces(), 0)
 
     def test_dedent_return(self):
         isp = self.isp # shorthand
         # should NOT cause dedent
         isp.push('if 1:\n    returning = 4')
         self.assertEqual(isp.get_indent_spaces(), 4)
         isp.push('if 1:\n     return 5 + 493')
         self.assertEqual(isp.get_indent_spaces(), 0)
         isp.push('if 1:\n     return')
         self.assertEqual(isp.get_indent_spaces(), 0)
         isp.push('if 1:\n     return      ')
         self.assertEqual(isp.get_indent_spaces(), 0)
         isp.push('if 1:\n     return(0)')
         self.assertEqual(isp.get_indent_spaces(), 0)
 
     def test_push(self):
         isp = self.isp
         self.assertEqual(isp.push('x=1'), True)
 
     def test_push2(self):
         isp = self.isp
         self.assertEqual(isp.push('if 1:'), False)
         for line in ['  x=1', '# a comment', '  y=2']:
             print(line)
             self.assertEqual(isp.push(line), True)
 
     def test_push3(self):
         isp = self.isp
         isp.push('if True:')
         isp.push('  a = 1')
         self.assertEqual(isp.push('b = [1,'), False)
 
     def test_push_accepts_more(self):
         isp = self.isp
         isp.push('x=1')
         self.assertEqual(isp.push_accepts_more(), False)
 
     def test_push_accepts_more2(self):
         isp = self.isp
         isp.push('if 1:')
         self.assertEqual(isp.push_accepts_more(), True)
         isp.push('  x=1')
         self.assertEqual(isp.push_accepts_more(), True)
         isp.push('')
         self.assertEqual(isp.push_accepts_more(), False)
 
     def test_push_accepts_more3(self):
         isp = self.isp
         isp.push("x = (2+\n3)")
         self.assertEqual(isp.push_accepts_more(), False)
 
     def test_push_accepts_more4(self):
         isp = self.isp
         # When a multiline statement contains parens or multiline strings, we
         # shouldn't get confused.
         # FIXME: we should be able to better handle de-dents in statements like
         # multiline strings and multiline expressions (continued with \ or
         # parens).  Right now we aren't handling the indentation tracking quite
         # correctly with this, though in practice it may not be too much of a
         # problem.  We'll need to see.
         isp.push("if 1:")
         isp.push("    x = (2+")
         isp.push("    3)")
         self.assertEqual(isp.push_accepts_more(), True)
         isp.push("    y = 3")
         self.assertEqual(isp.push_accepts_more(), True)
         isp.push('')
         self.assertEqual(isp.push_accepts_more(), False)
 
     def test_push_accepts_more5(self):
         isp = self.isp
         isp.push('try:')
         isp.push('    a = 5')
         isp.push('except:')
         isp.push('    raise')
         # We want to be able to add an else: block at this point, so it should
         # wait for a blank line.
         self.assertEqual(isp.push_accepts_more(), True)
 
     def test_continuation(self):
         isp = self.isp
         isp.push("import os, \\")
         self.assertEqual(isp.push_accepts_more(), True)
         isp.push("sys")
         self.assertEqual(isp.push_accepts_more(), False)
 
     def test_syntax_error(self):
         isp = self.isp
         # Syntax errors immediately produce a 'ready' block, so the invalid
         # Python can be sent to the kernel for evaluation with possible ipython
         # special-syntax conversion.
         isp.push('run foo')
         self.assertEqual(isp.push_accepts_more(), False)
 
     def test_unicode(self):
         self.isp.push(u"Pérez")
         self.isp.push(u'\xc3\xa9')
         self.isp.push(u"u'\xc3\xa9'")
 
-    @pytest.mark.xfail(
-        reason="Bug in python 3.9.8 – bpo 45738",
-        condition=sys.version_info in [(3, 11, 0, "alpha", 2)],
-        raises=SystemError,
-        strict=True,
-    )
     def test_line_continuation(self):
         """ Test issue #2108."""
         isp = self.isp
         # A blank line after a line continuation should not accept more
         isp.push("1 \\\n\n")
         self.assertEqual(isp.push_accepts_more(), False)
         # Whitespace after a \ is a SyntaxError.  The only way to test that
         # here is to test that push doesn't accept more (as with
         # test_syntax_error() above).
         isp.push(r"1 \ ")
         self.assertEqual(isp.push_accepts_more(), False)
         # Even if the line is continuable (c.f. the regular Python
         # interpreter)
         isp.push(r"(1 \ ")
         self.assertEqual(isp.push_accepts_more(), False)
 
     def test_check_complete(self):
         isp = self.isp
         self.assertEqual(isp.check_complete("a = 1"), ('complete', None))
         self.assertEqual(isp.check_complete("for a in range(5):"), ('incomplete', 4))
         self.assertEqual(isp.check_complete("raise = 2"), ('invalid', None))
         self.assertEqual(isp.check_complete("a = [1,\n2,"), ('incomplete', 0))
         self.assertEqual(isp.check_complete("def a():\n x=1\n global x"), ('invalid', None))
 
 class InteractiveLoopTestCase(unittest.TestCase):
     """Tests for an interactive loop like a python shell.
     """
     def check_ns(self, lines, ns):
         """Validate that the given input lines produce the resulting namespace.
 
         Note: the input lines are given exactly as they would be typed in an
         auto-indenting environment, as mini_interactive_loop above already does
         auto-indenting and prepends spaces to the input.
         """
         src = mini_interactive_loop(pseudo_input(lines))
         test_ns = {}
         exec(src, test_ns)
         # We can't check that the provided ns is identical to the test_ns,
         # because Python fills test_ns with extra keys (copyright, etc).  But
         # we can check that the given dict is *contained* in test_ns
         for k,v in ns.items():
             self.assertEqual(test_ns[k], v)
 
     def test_simple(self):
         self.check_ns(['x=1'], dict(x=1))
 
     def test_simple2(self):
         self.check_ns(['if 1:', 'x=2'], dict(x=2))
 
     def test_xy(self):
         self.check_ns(['x=1; y=2'], dict(x=1, y=2))
 
     def test_abc(self):
         self.check_ns(['if 1:','a=1','b=2','c=3'], dict(a=1, b=2, c=3))
 
     def test_multi(self):
         self.check_ns(['x =(1+','1+','2)'], dict(x=4))
 
 
 class IPythonInputTestCase(InputSplitterTestCase):
     """By just creating a new class whose .isp is a different instance, we
     re-run the same test battery on the new input splitter.
 
     In addition, this runs the tests over the syntax and syntax_ml dicts that
     were tested by individual functions, as part of the OO interface.
 
     It also makes some checks on the raw buffer storage.
     """
 
     def setUp(self):
         self.isp = isp.IPythonInputSplitter()
 
     def test_syntax(self):
         """Call all single-line syntax tests from the main object"""
         isp = self.isp
         for example in syntax.values():
             for raw, out_t in example:
                 if raw.startswith(' '):
                     continue
 
                 isp.push(raw+'\n')
                 out_raw = isp.source_raw
                 out = isp.source_reset()
                 self.assertEqual(out.rstrip(), out_t,
                         tt.pair_fail_msg.format("inputsplitter",raw, out_t, out))
                 self.assertEqual(out_raw.rstrip(), raw.rstrip())
 
     def test_syntax_multiline(self):
         isp = self.isp
         for example in syntax_ml.values():
             for line_pairs in example:
                 out_t_parts = []
                 raw_parts = []
                 for lraw, out_t_part in line_pairs:
                     if out_t_part is not None:
                         out_t_parts.append(out_t_part)
 
                     if lraw is not None:
                         isp.push(lraw)
                         raw_parts.append(lraw)
 
                 out_raw = isp.source_raw
                 out = isp.source_reset()
                 out_t = '\n'.join(out_t_parts).rstrip()
                 raw = '\n'.join(raw_parts).rstrip()
                 self.assertEqual(out.rstrip(), out_t)
                 self.assertEqual(out_raw.rstrip(), raw)
 
     def test_syntax_multiline_cell(self):
         isp = self.isp
         for example in syntax_ml.values():
 
             out_t_parts = []
             for line_pairs in example:
                 raw = '\n'.join(r for r, _ in line_pairs if r is not None)
                 out_t = '\n'.join(t for _,t in line_pairs if t is not None)
                 out = isp.transform_cell(raw)
                 # Match ignoring trailing whitespace
                 self.assertEqual(out.rstrip(), out_t.rstrip())
 
     def test_cellmagic_preempt(self):
         isp = self.isp
         for raw, name, line, cell in [
             ("%%cellm a\nIn[1]:", u'cellm', u'a', u'In[1]:'),
             ("%%cellm \nline\n>>> hi", u'cellm', u'', u'line\n>>> hi'),
             (">>> %%cellm \nline\n>>> hi", u'cellm', u'', u'line\nhi'),
             ("%%cellm \n>>> hi", u'cellm', u'', u'>>> hi'),
             ("%%cellm \nline1\nline2", u'cellm', u'', u'line1\nline2'),
             ("%%cellm \nline1\\\\\nline2", u'cellm', u'', u'line1\\\\\nline2'),
         ]:
             expected = "get_ipython().run_cell_magic(%r, %r, %r)" % (
                 name, line, cell
             )
             out = isp.transform_cell(raw)
             self.assertEqual(out.rstrip(), expected.rstrip())
 
     def test_multiline_passthrough(self):
         isp = self.isp
         class CommentTransformer(InputTransformer):
             def __init__(self):
                 self._lines = []
 
             def push(self, line):
                 self._lines.append(line + '#')
 
             def reset(self):
                 text = '\n'.join(self._lines)
                 self._lines = []
                 return text
 
         isp.physical_line_transforms.insert(0, CommentTransformer())
 
         for raw, expected in [
             ("a=5", "a=5#"),
             ("%ls foo", "get_ipython().run_line_magic(%r, %r)" % (u'ls', u'foo#')),
             ("!ls foo\n%ls bar", "get_ipython().system(%r)\nget_ipython().run_line_magic(%r, %r)" % (
                 u'ls foo#', u'ls', u'bar#'
             )),
             ("1\n2\n3\n%ls foo\n4\n5", "1#\n2#\n3#\nget_ipython().run_line_magic(%r, %r)\n4#\n5#" % (u'ls', u'foo#')),
         ]:
             out = isp.transform_cell(raw)
             self.assertEqual(out.rstrip(), expected.rstrip())
 
 #-----------------------------------------------------------------------------
 # Main - use as a script, mostly for developer experiments
 #-----------------------------------------------------------------------------
 
 if __name__ == '__main__':
     # A simple demo for interactive experimentation.  This code will not get
     # picked up by any test suite.
     from IPython.core.inputsplitter import IPythonInputSplitter
 
     # configure here the syntax to use, prompt and whether to autoindent
     #isp, start_prompt = InputSplitter(), '>>> '
     isp, start_prompt = IPythonInputSplitter(), 'In> '
 
     autoindent = True
     #autoindent = False
 
     try:
         while True:
             prompt = start_prompt
             while isp.push_accepts_more():
                 indent = ' '*isp.get_indent_spaces()
                 if autoindent:
                     line = indent + input(prompt+indent)
                 else:
                     line = input(prompt)
                 isp.push(line)
                 prompt = '... '
 
             # Here we just return input so we can use it in a test suite, but a
             # real interpreter would instead send it for execution somewhere.
             #src = isp.source; raise EOFError # dbg
             raw = isp.source_raw
             src = isp.source_reset()
             print('Input source was:\n', src)
             print('Raw source was:\n', raw)
     except EOFError:
         print('Bye')
 
 # Tests for cell magics support
 
 def test_last_blank():
     assert isp.last_blank("") is False
     assert isp.last_blank("abc") is False
     assert isp.last_blank("abc\n") is False
     assert isp.last_blank("abc\na") is False
 
     assert isp.last_blank("\n") is True
     assert isp.last_blank("\n ") is True
     assert isp.last_blank("abc\n ") is True
     assert isp.last_blank("abc\n\n") is True
     assert isp.last_blank("abc\nd\n\n") is True
     assert isp.last_blank("abc\nd\ne\n\n") is True
     assert isp.last_blank("abc \n \n \n\n") is True
 
 
 def test_last_two_blanks():
     assert isp.last_two_blanks("") is False
     assert isp.last_two_blanks("abc") is False
     assert isp.last_two_blanks("abc\n") is False
     assert isp.last_two_blanks("abc\n\na") is False
     assert isp.last_two_blanks("abc\n \n") is False
     assert isp.last_two_blanks("abc\n\n") is False
 
     assert isp.last_two_blanks("\n\n") is True
     assert isp.last_two_blanks("\n\n ") is True
     assert isp.last_two_blanks("\n \n") is True
     assert isp.last_two_blanks("abc\n\n ") is True
     assert isp.last_two_blanks("abc\n\n\n") is True
     assert isp.last_two_blanks("abc\n\n \n") is True
     assert isp.last_two_blanks("abc\n\n \n ") is True
     assert isp.last_two_blanks("abc\n\n \n \n") is True
     assert isp.last_two_blanks("abc\nd\n\n\n") is True
     assert isp.last_two_blanks("abc\nd\ne\nf\n\n\n") is True
 
 
 class CellMagicsCommon(object):
 
     def test_whole_cell(self):
         src = "%%cellm line\nbody\n"
         out = self.sp.transform_cell(src)
         ref = "get_ipython().run_cell_magic('cellm', 'line', 'body')\n"
         assert out == ref
 
     def test_cellmagic_help(self):
         self.sp.push('%%cellm?')
         assert self.sp.push_accepts_more() is False
 
     def tearDown(self):
         self.sp.reset()
 
 
 class CellModeCellMagics(CellMagicsCommon, unittest.TestCase):
     sp = isp.IPythonInputSplitter(line_input_checker=False)
 
     def test_incremental(self):
         sp = self.sp
         sp.push("%%cellm firstline\n")
         assert sp.push_accepts_more() is True  # 1
         sp.push("line2\n")
         assert sp.push_accepts_more() is True  # 2
         sp.push("\n")
         # This should accept a blank line and carry on until the cell is reset
         assert sp.push_accepts_more() is True  # 3
 
     def test_no_strip_coding(self):
         src = '\n'.join([
             '%%writefile foo.py',
             '# coding: utf-8',
             'print(u"üñîçø∂é")',
         ])
         out = self.sp.transform_cell(src)
         assert "# coding: utf-8" in out
 
 
 class LineModeCellMagics(CellMagicsCommon, unittest.TestCase):
     sp = isp.IPythonInputSplitter(line_input_checker=True)
 
     def test_incremental(self):
         sp = self.sp
         sp.push("%%cellm line2\n")
         assert sp.push_accepts_more() is True  # 1
         sp.push("\n")
         # In this case, a blank line should end the cell magic
         assert sp.push_accepts_more() is False  # 2
 
 
 indentation_samples = [
     ('a = 1', 0),
     ('for a in b:', 4),
     ('def f():', 4),
     ('def f(): #comment', 4),
     ('a = ":#not a comment"', 0),
     ('def f():\n    a = 1', 4),
     ('def f():\n    return 1', 0),
     ('for a in b:\n'
      '   if a < 0:'
      '       continue', 3),
     ('a = {', 4),
     ('a = {\n'
      '     1,', 5),
     ('b = """123', 0),
     ('', 0),
     ('def f():\n    pass', 0),
     ('class Bar:\n    def f():\n        pass', 4),
     ('class Bar:\n    def f():\n        raise', 4),
 ]
 
 def test_find_next_indent():
     for code, exp in indentation_samples:
         res = isp.find_next_indent(code)
         msg = "{!r} != {!r} (expected)\n Code: {!r}".format(res, exp, code)
         assert res == exp, msg