Merge pull request #13630 from Carreau/longerfail

DEBUG: on pexpect failure show 1000 chars buffer

IPython/core/inputtransformer2.py

@@ -1,750 +1,752 b''
 """Input transformer machinery to support IPython special syntax.
 
 This includes the machinery to recognise and transform ``%magic`` commands,
 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
 
 Added: IPython 7.0. Replaces inputsplitter and inputtransformer which were
 deprecated in 7.0.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import ast
 import sys
 from codeop import CommandCompiler, Compile
 import re
 import tokenize
 from typing import List, Tuple, Union
 import warnings
 
 _indent_re = re.compile(r'^[ \t]+')
 
 def leading_empty_lines(lines):
     """Remove leading empty lines
 
     If the leading lines are empty or contain only whitespace, they will be
     removed.
     """
     if not lines:
         return lines
     for i, line in enumerate(lines):
         if line and not line.isspace():
             return lines[i:]
     return lines
 
 def leading_indent(lines):
     """Remove leading indentation.
 
     If the first line starts with a spaces or tabs, the same whitespace will be
     removed from each following line in the cell.
     """
     if not lines:
         return lines
     m = _indent_re.match(lines[0])
     if not m:
         return lines
     space = m.group(0)
     n = len(space)
     return [l[n:] if l.startswith(space) else l
             for l in lines]
 
 class PromptStripper:
     """Remove matching input prompts from a block of input.
 
     Parameters
     ----------
     prompt_re : regular expression
         A regular expression matching any input prompt (including continuation,
         e.g. ``...``)
     initial_re : regular expression, optional
         A regular expression matching only the initial prompt, but not continuation.
         If no initial expression is given, prompt_re will be used everywhere.
         Used mainly for plain Python prompts (``>>>``), where the continuation prompt
         ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
 
     Notes
     -----
 
     If initial_re and prompt_re differ,
     only initial_re will be tested against the first line.
     If any prompt is found on the first two lines,
     prompts will be stripped from the rest of the block.
     """
     def __init__(self, prompt_re, initial_re=None):
         self.prompt_re = prompt_re
         self.initial_re = initial_re or prompt_re
 
     def _strip(self, lines):
         return [self.prompt_re.sub('', l, count=1) for l in lines]
 
     def __call__(self, lines):
         if not lines:
             return lines
         if self.initial_re.match(lines[0]) or \
                 (len(lines) > 1 and self.prompt_re.match(lines[1])):
             return self._strip(lines)
         return lines
 
 classic_prompt = PromptStripper(
     prompt_re=re.compile(r'^(>>>|\.\.\.)( |$)'),
     initial_re=re.compile(r'^>>>( |$)')
 )
 
 ipython_prompt = PromptStripper(re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)'))
 
 def cell_magic(lines):
     if not lines or not lines[0].startswith('%%'):
         return lines
     if re.match(r'%%\w+\?', lines[0]):
         # This case will be handled by help_end
         return lines
     magic_name, _, first_line = lines[0][2:].rstrip().partition(' ')
     body = ''.join(lines[1:])
     return ['get_ipython().run_cell_magic(%r, %r, %r)\n'
             % (magic_name, first_line, body)]
 
 
 def _find_assign_op(token_line) -> Union[int, None]:
     """Get the index of the first assignment in the line ('=' not inside brackets)
 
     Note: We don't try to support multiple special assignment (a = b = %foo)
     """
     paren_level = 0
     for i, ti in enumerate(token_line):
         s = ti.string
         if s == '=' and paren_level == 0:
             return i
         if s in {'(','[','{'}:
             paren_level += 1
         elif s in {')', ']', '}'}:
             if paren_level > 0:
                 paren_level -= 1
 
 def find_end_of_continued_line(lines, start_line: int):
     """Find the last line of a line explicitly extended using backslashes.
 
     Uses 0-indexed line numbers.
     """
     end_line = start_line
     while lines[end_line].endswith('\\\n'):
         end_line += 1
         if end_line >= len(lines):
             break
     return end_line
 
 def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):
     r"""Assemble a single line from multiple continued line pieces
 
     Continued lines are lines ending in ``\``, and the line following the last
     ``\`` in the block.
 
     For example, this code continues over multiple lines::
 
         if (assign_ix is not None) \
              and (len(line) >= assign_ix + 2) \
              and (line[assign_ix+1].string == '%') \
              and (line[assign_ix+2].type == tokenize.NAME):
 
     This statement contains four continued line pieces.
     Assembling these pieces into a single line would give::
 
         if (assign_ix is not None) and (len(line) >= assign_ix + 2) and (line[...
 
     This uses 0-indexed line numbers. *start* is (lineno, colno).
 
     Used to allow ``%magic`` and ``!system`` commands to be continued over
     multiple lines.
     """
     parts = [lines[start[0]][start[1]:]] + lines[start[0]+1:end_line+1]
     return ' '.join([p.rstrip()[:-1] for p in parts[:-1]]  # Strip backslash+newline
                     + [parts[-1].rstrip()])         # Strip newline from last line
 
 class TokenTransformBase:
     """Base class for transformations which examine tokens.
 
     Special syntax should not be transformed when it occurs inside strings or
     comments. This is hard to reliably avoid with regexes. The solution is to
     tokenise the code as Python, and recognise the special syntax in the tokens.
 
     IPython's special syntax is not valid Python syntax, so tokenising may go
     wrong after the special syntax starts. These classes therefore find and
     transform *one* instance of special syntax at a time into regular Python
     syntax. After each transformation, tokens are regenerated to find the next
     piece of special syntax.
 
     Subclasses need to implement one class method (find)
     and one regular method (transform).
 
     The priority attribute can select which transformation to apply if multiple
     transformers match in the same place. Lower numbers have higher priority.
     This allows "%magic?" to be turned into a help call rather than a magic call.
     """
     # Lower numbers -> higher priority (for matches in the same location)
     priority = 10
 
     def sortby(self):
         return self.start_line, self.start_col, self.priority
 
     def __init__(self, start):
         self.start_line = start[0] - 1   # Shift from 1-index to 0-index
         self.start_col = start[1]
 
     @classmethod
     def find(cls, tokens_by_line):
         """Find one instance of special syntax in the provided tokens.
 
         Tokens are grouped into logical lines for convenience,
         so it is easy to e.g. look at the first token of each line.
         *tokens_by_line* is a list of lists of tokenize.TokenInfo objects.
 
         This should return an instance of its class, pointing to the start
         position it has found, or None if it found no match.
         """
         raise NotImplementedError
 
     def transform(self, lines: List[str]):
         """Transform one instance of special syntax found by ``find()``
 
         Takes a list of strings representing physical lines,
         returns a similar list of transformed lines.
         """
         raise NotImplementedError
 
 class MagicAssign(TokenTransformBase):
     """Transformer for assignments from magics (a = %foo)"""
     @classmethod
     def find(cls, tokens_by_line):
         """Find the first magic assignment (a = %foo) in the cell.
         """
         for line in tokens_by_line:
             assign_ix = _find_assign_op(line)
             if (assign_ix is not None) \
                     and (len(line) >= assign_ix + 2) \
                     and (line[assign_ix+1].string == '%') \
                     and (line[assign_ix+2].type == tokenize.NAME):
                 return cls(line[assign_ix+1].start)
 
     def transform(self, lines: List[str]):
         """Transform a magic assignment found by the ``find()`` classmethod.
         """
         start_line, start_col = self.start_line, self.start_col
         lhs = lines[start_line][:start_col]
         end_line = find_end_of_continued_line(lines, start_line)
         rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
         assert rhs.startswith('%'), rhs
         magic_name, _, args = rhs[1:].partition(' ')
 
         lines_before = lines[:start_line]
         call = "get_ipython().run_line_magic({!r}, {!r})".format(magic_name, args)
         new_line = lhs + call + '\n'
         lines_after = lines[end_line+1:]
 
         return lines_before + [new_line] + lines_after
 
 
 class SystemAssign(TokenTransformBase):
     """Transformer for assignments from system commands (a = !foo)"""
     @classmethod
     def find(cls, tokens_by_line):
         """Find the first system assignment (a = !foo) in the cell.
         """
         for line in tokens_by_line:
             assign_ix = _find_assign_op(line)
             if (assign_ix is not None) \
                     and not line[assign_ix].line.strip().startswith('=') \
                     and (len(line) >= assign_ix + 2) \
                     and (line[assign_ix + 1].type == tokenize.ERRORTOKEN):
                 ix = assign_ix + 1
 
                 while ix < len(line) and line[ix].type == tokenize.ERRORTOKEN:
                     if line[ix].string == '!':
                         return cls(line[ix].start)
                     elif not line[ix].string.isspace():
                         break
                     ix += 1
 
     def transform(self, lines: List[str]):
         """Transform a system assignment found by the ``find()`` classmethod.
         """
         start_line, start_col = self.start_line, self.start_col
 
         lhs = lines[start_line][:start_col]
         end_line = find_end_of_continued_line(lines, start_line)
         rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
         assert rhs.startswith('!'), rhs
         cmd = rhs[1:]
 
         lines_before = lines[:start_line]
         call = "get_ipython().getoutput({!r})".format(cmd)
         new_line = lhs + call + '\n'
         lines_after = lines[end_line + 1:]
 
         return lines_before + [new_line] + lines_after
 
 # The escape sequences that define the syntax transformations IPython will
 # apply to user input.  These can NOT be just changed here: many regular
 # expressions and other parts of the code may use their hardcoded values, and
 # for all intents and purposes they constitute the 'IPython syntax', so they
 # should be considered fixed.
 
 ESC_SHELL  = '!'     # Send line to underlying system shell
 ESC_SH_CAP = '!!'    # Send line to system shell and capture output
 ESC_HELP   = '?'     # Find information about object
 ESC_HELP2  = '??'    # Find extra-detailed information about object
 ESC_MAGIC  = '%'     # Call magic function
 ESC_MAGIC2 = '%%'    # Call cell-magic function
 ESC_QUOTE  = ','     # Split args on whitespace, quote each as string and call
 ESC_QUOTE2 = ';'     # Quote all args as a single string, call
 ESC_PAREN  = '/'     # Call first argument with rest of line as arguments
 
 ESCAPE_SINGLES = {'!', '?', '%', ',', ';', '/'}
 ESCAPE_DOUBLES = {'!!', '??'}  # %% (cell magic) is handled separately
 
 def _make_help_call(target, esc, next_input=None):
     """Prepares a pinfo(2)/psearch call from a target name and the escape
     (i.e. ? or ??)"""
     method  = 'pinfo2' if esc == '??' \
                 else 'psearch' if '*' in target \
                 else 'pinfo'
     arg = " ".join([method, target])
     #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
     t_magic_name, _, t_magic_arg_s = arg.partition(' ')
     t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
     if next_input is None:
         return 'get_ipython().run_line_magic(%r, %r)' % (t_magic_name, t_magic_arg_s)
     else:
         return 'get_ipython().set_next_input(%r);get_ipython().run_line_magic(%r, %r)' % \
            (next_input, t_magic_name, t_magic_arg_s)
 
 def _tr_help(content):
     """Translate lines escaped with: ?
 
     A naked help line should fire the intro help screen (shell.show_usage())
     """
     if not content:
         return 'get_ipython().show_usage()'
 
     return _make_help_call(content, '?')
 
 def _tr_help2(content):
     """Translate lines escaped with: ??
 
     A naked help line should fire the intro help screen (shell.show_usage())
     """
     if not content:
         return 'get_ipython().show_usage()'
 
     return _make_help_call(content, '??')
 
 def _tr_magic(content):
     "Translate lines escaped with a percent sign: %"
     name, _, args = content.partition(' ')
     return 'get_ipython().run_line_magic(%r, %r)' % (name, args)
 
 def _tr_quote(content):
     "Translate lines escaped with a comma: ,"
     name, _, args = content.partition(' ')
     return '%s("%s")' % (name, '", "'.join(args.split()) )
 
 def _tr_quote2(content):
     "Translate lines escaped with a semicolon: ;"
     name, _, args = content.partition(' ')
     return '%s("%s")' % (name, args)
 
 def _tr_paren(content):
     "Translate lines escaped with a slash: /"
     name, _, args = content.partition(' ')
     return '%s(%s)' % (name, ", ".join(args.split()))
 
 tr = { ESC_SHELL  : 'get_ipython().system({!r})'.format,
        ESC_SH_CAP : 'get_ipython().getoutput({!r})'.format,
        ESC_HELP   : _tr_help,
        ESC_HELP2  : _tr_help2,
        ESC_MAGIC  : _tr_magic,
        ESC_QUOTE  : _tr_quote,
        ESC_QUOTE2 : _tr_quote2,
        ESC_PAREN  : _tr_paren }
 
 class EscapedCommand(TokenTransformBase):
     """Transformer for escaped commands like %foo, !foo, or /foo"""
     @classmethod
     def find(cls, tokens_by_line):
         """Find the first escaped command (%foo, !foo, etc.) in the cell.
         """
         for line in tokens_by_line:
             if not line:
                 continue
             ix = 0
             ll = len(line)
             while ll > ix and line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
                 ix += 1
             if ix >= ll:
                 continue
             if line[ix].string in ESCAPE_SINGLES:
                 return cls(line[ix].start)
 
     def transform(self, lines):
         """Transform an escaped line found by the ``find()`` classmethod.
         """
         start_line, start_col = self.start_line, self.start_col
 
         indent = lines[start_line][:start_col]
         end_line = find_end_of_continued_line(lines, start_line)
         line = assemble_continued_line(lines, (start_line, start_col), end_line)
 
         if len(line) > 1 and line[:2] in ESCAPE_DOUBLES:
             escape, content = line[:2], line[2:]
         else:
             escape, content = line[:1], line[1:]
 
         if escape in tr:
             call = tr[escape](content)
         else:
             call = ''
 
         lines_before = lines[:start_line]
         new_line = indent + call + '\n'
         lines_after = lines[end_line + 1:]
 
         return lines_before + [new_line] + lines_after
 
 _help_end_re = re.compile(r"""(%{0,2}
                               (?!\d)[\w*]+            # Variable name
                               (\.(?!\d)[\w*]+)*       # .etc.etc
                               )
                               (\?\??)$                # ? or ??
                               """,
                               re.VERBOSE)
 
 class HelpEnd(TokenTransformBase):
     """Transformer for help syntax: obj? and obj??"""
     # This needs to be higher priority (lower number) than EscapedCommand so
     # that inspecting magics (%foo?) works.
     priority = 5
 
     def __init__(self, start, q_locn):
         super().__init__(start)
         self.q_line = q_locn[0] - 1  # Shift from 1-indexed to 0-indexed
         self.q_col = q_locn[1]
 
     @classmethod
     def find(cls, tokens_by_line):
         """Find the first help command (foo?) in the cell.
         """
         for line in tokens_by_line:
             # Last token is NEWLINE; look at last but one
             if len(line) > 2 and line[-2].string == '?':
                 # Find the first token that's not INDENT/DEDENT
                 ix = 0
                 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
                     ix += 1
                 return cls(line[ix].start, line[-2].start)
 
     def transform(self, lines):
         """Transform a help command found by the ``find()`` classmethod.
         """
         piece = ''.join(lines[self.start_line:self.q_line+1])
         indent, content = piece[:self.start_col], piece[self.start_col:]
         lines_before = lines[:self.start_line]
         lines_after = lines[self.q_line + 1:]
 
         m = _help_end_re.search(content)
         if not m:
             raise SyntaxError(content)
         assert m is not None, content
         target = m.group(1)
         esc = m.group(3)
 
         # If we're mid-command, put it back on the next prompt for the user.
         next_input = None
         if (not lines_before) and (not lines_after) \
                 and content.strip() != m.group(0):
             next_input = content.rstrip('?\n')
 
         call = _make_help_call(target, esc, next_input=next_input)
         new_line = indent + call + '\n'
 
         return lines_before + [new_line] + lines_after
 
 def make_tokens_by_line(lines:List[str]):
     """Tokenize a series of lines and group tokens by line.
 
     The tokens for a multiline Python string or expression are grouped as one
     line. All lines except the last lines should keep their line ending ('\\n',
     '\\r\\n') for this to properly work. Use `.splitlines(keeplineending=True)`
     for example when passing block of text to this function.
 
     """
     # NL tokens are used inside multiline expressions, but also after blank
     # lines or comments. This is intentional - see https://bugs.python.org/issue17061
     # We want to group the former case together but split the latter, so we
     # track parentheses level, similar to the internals of tokenize.
     NEWLINE, NL = tokenize.NEWLINE, tokenize.NL
     tokens_by_line = [[]]
     if len(lines) > 1 and not lines[0].endswith(('\n', '\r', '\r\n', '\x0b', '\x0c')):
         warnings.warn("`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified")
     parenlev = 0
     try:
         for token in tokenize.generate_tokens(iter(lines).__next__):
             tokens_by_line[-1].append(token)
             if (token.type == NEWLINE) \
                     or ((token.type == NL) and (parenlev <= 0)):
                 tokens_by_line.append([])
             elif token.string in {'(', '[', '{'}:
                 parenlev += 1
             elif token.string in {')', ']', '}'}:
                 if parenlev > 0:
                     parenlev -= 1
     except tokenize.TokenError:
         # Input ended in a multiline string or expression. That's OK for us.
         pass
 
 
     if not tokens_by_line[-1]:
         tokens_by_line.pop()
 
 
     return tokens_by_line
 
 def show_linewise_tokens(s: str):
     """For investigation and debugging"""
     if not s.endswith('\n'):
         s += '\n'
     lines = s.splitlines(keepends=True)
     for line in make_tokens_by_line(lines):
         print("Line -------")
         for tokinfo in line:
             print(" ", tokinfo)
 
 # Arbitrary limit to prevent getting stuck in infinite loops
 TRANSFORM_LOOP_LIMIT = 500
 
 class TransformerManager:
     """Applies various transformations to a cell or code block.
 
     The key methods for external use are ``transform_cell()``
     and ``check_complete()``.
     """
     def __init__(self):
         self.cleanup_transforms = [
             leading_empty_lines,
             leading_indent,
             classic_prompt,
             ipython_prompt,
         ]
         self.line_transforms = [
             cell_magic,
         ]
         self.token_transformers = [
             MagicAssign,
             SystemAssign,
             EscapedCommand,
             HelpEnd,
         ]
 
     def do_one_token_transform(self, lines):
         """Find and run the transform earliest in the code.
 
         Returns (changed, lines).
 
         This method is called repeatedly until changed is False, indicating
         that all available transformations are complete.
 
         The tokens following IPython special syntax might not be valid, so
         the transformed code is retokenised every time to identify the next
         piece of special syntax. Hopefully long code cells are mostly valid
         Python, not using lots of IPython special syntax, so this shouldn't be
         a performance issue.
         """
         tokens_by_line = make_tokens_by_line(lines)
         candidates = []
         for transformer_cls in self.token_transformers:
             transformer = transformer_cls.find(tokens_by_line)
             if transformer:
                 candidates.append(transformer)
 
         if not candidates:
             # Nothing to transform
             return False, lines
         ordered_transformers = sorted(candidates, key=TokenTransformBase.sortby)
         for transformer in ordered_transformers:
             try:
                 return True, transformer.transform(lines)
             except SyntaxError:
                 pass
         return False, lines
 
     def do_token_transforms(self, lines):
         for _ in range(TRANSFORM_LOOP_LIMIT):
             changed, lines = self.do_one_token_transform(lines)
             if not changed:
                 return lines
 
         raise RuntimeError("Input transformation still changing after "
                            "%d iterations. Aborting." % TRANSFORM_LOOP_LIMIT)
 
     def transform_cell(self, cell: str) -> str:
         """Transforms a cell of input code"""
         if not cell.endswith('\n'):
             cell += '\n'  # Ensure the cell has a trailing newline
         lines = cell.splitlines(keepends=True)
         for transform in self.cleanup_transforms + self.line_transforms:
             lines = transform(lines)
 
         lines = self.do_token_transforms(lines)
         return ''.join(lines)
 
     def check_complete(self, cell: str):
         """Return whether a block of code is ready to execute, or should be continued
 
         Parameters
         ----------
         source : string
           Python input code, which can be multiline.
 
         Returns
         -------
         status : str
           One of 'complete', 'incomplete', or 'invalid' if source is not a
           prefix of valid code.
         indent_spaces : int or None
           The number of spaces by which to indent the next line of code. If
           status is not 'incomplete', this is None.
         """
         # Remember if the lines ends in a new line.
         ends_with_newline = False
         for character in reversed(cell):
             if character == '\n':
                 ends_with_newline = True
                 break
             elif character.strip():
                 break
             else:
                 continue
 
         if not ends_with_newline:
             # Append an newline for consistent tokenization
             # See https://bugs.python.org/issue33899
             cell += '\n'
 
         lines = cell.splitlines(keepends=True)
 
         if not lines:
             return 'complete', None
 
         if lines[-1].endswith('\\'):
             # Explicit backslash continuation
             return 'incomplete', find_last_indent(lines)
 
         try:
             for transform in self.cleanup_transforms:
                 if not getattr(transform, 'has_side_effects', False):
                     lines = transform(lines)
         except SyntaxError:
             return 'invalid', None
 
         if lines[0].startswith('%%'):
             # Special case for cell magics - completion marked by blank line
             if lines[-1].strip():
                 return 'incomplete', find_last_indent(lines)
             else:
                 return 'complete', None
 
         try:
             for transform in self.line_transforms:
                 if not getattr(transform, 'has_side_effects', False):
                     lines = transform(lines)
             lines = self.do_token_transforms(lines)
         except SyntaxError:
             return 'invalid', None
 
         tokens_by_line = make_tokens_by_line(lines)
 
         if not tokens_by_line:
             return 'incomplete', find_last_indent(lines)
 
         if tokens_by_line[-1][-1].type != tokenize.ENDMARKER:
             # We're in a multiline string or expression
             return 'incomplete', find_last_indent(lines)
 
         newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER}
 
         # Pop the last line which only contains DEDENTs and ENDMARKER
         last_token_line = None
         if {t.type for t in tokens_by_line[-1]} in [
             {tokenize.DEDENT, tokenize.ENDMARKER},
             {tokenize.ENDMARKER}
         ] and len(tokens_by_line) > 1:
             last_token_line = tokens_by_line.pop()
 
         while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types:
             tokens_by_line[-1].pop()
 
         if not tokens_by_line[-1]:
             return 'incomplete', find_last_indent(lines)
 
         if tokens_by_line[-1][-1].string == ':':
             # The last line starts a block (e.g. 'if foo:')
             ix = 0
             while tokens_by_line[-1][ix].type in {tokenize.INDENT, tokenize.DEDENT}:
                 ix += 1
 
             indent = tokens_by_line[-1][ix].start[1]
             return 'incomplete', indent + 4
 
         if tokens_by_line[-1][0].line.endswith('\\'):
             return 'incomplete', None
 
         # At this point, our checks think the code is complete (or invalid).
         # We'll use codeop.compile_command to check this with the real parser
         try:
             with warnings.catch_warnings():
                 warnings.simplefilter('error', SyntaxWarning)
                 res = compile_command(''.join(lines), symbol='exec')
         except (SyntaxError, OverflowError, ValueError, TypeError,
                 MemoryError, SyntaxWarning):
             return 'invalid', None
         else:
             if res is None:
                 return 'incomplete', find_last_indent(lines)
 
         if last_token_line and last_token_line[0].type == tokenize.DEDENT:
             if ends_with_newline:
                 return 'complete', None
             return 'incomplete', find_last_indent(lines)
 
         # If there's a blank line at the end, assume we're ready to execute
         if not lines[-1].strip():
             return 'complete', None
 
         return 'complete', None
 
 
 def find_last_indent(lines):
     m = _indent_re.match(lines[-1])
     if not m:
         return 0
     return len(m.group(0).replace('\t', ' '*4))
 
 
 class MaybeAsyncCompile(Compile):
     def __init__(self, extra_flags=0):
         super().__init__()
         self.flags |= extra_flags
 
-    def __call__(self, *args, **kwds):
-        return compile(*args, **kwds)
+
+    if sys.version_info < (3,8):
+        def __call__(self, *args, **kwds):
+            return compile(*args, **kwds)
 
 
 class MaybeAsyncCommandCompiler(CommandCompiler):
     def __init__(self, extra_flags=0):
         self.compiler = MaybeAsyncCompile(extra_flags=extra_flags)
 
 
 if (sys.version_info.major, sys.version_info.minor) >= (3, 8):
     _extra_flags = ast.PyCF_ALLOW_TOP_LEVEL_AWAIT
 else:
     _extra_flags = ast.PyCF_ONLY_AST
 
 compile_command = MaybeAsyncCommandCompiler(extra_flags=_extra_flags)

IPython/core/tests/test_debugger.py

@@ -1,579 +1,580 b''
 """Tests for debugging machinery.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import bdb
 import builtins
 import os
 import signal
 import subprocess
 import sys
 import time
 import warnings
 
 from subprocess import PIPE, CalledProcessError, check_output
 from tempfile import NamedTemporaryFile
 from textwrap import dedent
 from unittest.mock import patch
 
 import nose.tools as nt
 
 from IPython.core import debugger
 from IPython.testing import IPYTHON_TESTING_TIMEOUT_SCALE
 from IPython.testing.decorators import skip_win32
 
 #-----------------------------------------------------------------------------
 # Helper classes, from CPython's Pdb test suite
 #-----------------------------------------------------------------------------
 
 class _FakeInput(object):
     """
     A fake input stream for pdb's interactive debugger.  Whenever a
     line is read, print it (to simulate the user typing it), and then
     return it.  The set of lines to return is specified in the
     constructor; they should not have trailing newlines.
     """
     def __init__(self, lines):
         self.lines = iter(lines)
 
     def readline(self):
         line = next(self.lines)
         print(line)
         return line+'\n'
 
 class PdbTestInput(object):
     """Context manager that makes testing Pdb in doctests easier."""
 
     def __init__(self, input):
         self.input = input
 
     def __enter__(self):
         self.real_stdin = sys.stdin
         sys.stdin = _FakeInput(self.input)
 
     def __exit__(self, *exc):
         sys.stdin = self.real_stdin
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
 def test_longer_repr():
     try:
         from reprlib import repr as trepr  # Py 3
     except ImportError:
         from repr import repr as trepr  # Py 2
     
     a = '1234567890'* 7
     ar = "'1234567890123456789012345678901234567890123456789012345678901234567890'"
     a_trunc = "'123456789012...8901234567890'"
     nt.assert_equal(trepr(a), a_trunc)
     # The creation of our tracer modifies the repr module's repr function
     # in-place, since that global is used directly by the stdlib's pdb module.
     with warnings.catch_warnings():
         warnings.simplefilter('ignore', DeprecationWarning)
         debugger.Tracer()
     nt.assert_equal(trepr(a), ar)
 
 def test_ipdb_magics():
     '''Test calling some IPython magics from ipdb.
 
     First, set up some test functions and classes which we can inspect.
 
     >>> class ExampleClass(object):
     ...    """Docstring for ExampleClass."""
     ...    def __init__(self):
     ...        """Docstring for ExampleClass.__init__"""
     ...        pass
     ...    def __str__(self):
     ...        return "ExampleClass()"
 
     >>> def example_function(x, y, z="hello"):
     ...     """Docstring for example_function."""
     ...     pass
 
     >>> old_trace = sys.gettrace()
 
     Create a function which triggers ipdb.
 
     >>> def trigger_ipdb():
     ...    a = ExampleClass()
     ...    debugger.Pdb().set_trace()
 
     >>> with PdbTestInput([
     ...    'pdef example_function',
     ...    'pdoc ExampleClass',
     ...    'up',
     ...    'down',
     ...    'list',
     ...    'pinfo a',
     ...    'll',
     ...    'continue',
     ... ]):
     ...     trigger_ipdb()
     --Return--
     None
     > <doctest ...>(3)trigger_ipdb()
           1 def trigger_ipdb():
           2    a = ExampleClass()
     ----> 3    debugger.Pdb().set_trace()
     <BLANKLINE>
     ipdb> pdef example_function
      example_function(x, y, z='hello')
      ipdb> pdoc ExampleClass
     Class docstring:
         Docstring for ExampleClass.
     Init docstring:
         Docstring for ExampleClass.__init__
     ipdb> up
     > <doctest ...>(11)<module>()
           7    'pinfo a',
           8    'll',
           9    'continue',
          10 ]):
     ---> 11     trigger_ipdb()
     <BLANKLINE>
     ipdb> down
     None
     > <doctest ...>(3)trigger_ipdb()
           1 def trigger_ipdb():
           2    a = ExampleClass()
     ----> 3    debugger.Pdb().set_trace()
     <BLANKLINE>
     ipdb> list
           1 def trigger_ipdb():
           2    a = ExampleClass()
     ----> 3    debugger.Pdb().set_trace()
     <BLANKLINE>
     ipdb> pinfo a
     Type:           ExampleClass
     String form:    ExampleClass()
     Namespace:      Local...
     Docstring:      Docstring for ExampleClass.
     Init docstring: Docstring for ExampleClass.__init__
     ipdb> ll
           1 def trigger_ipdb():
           2    a = ExampleClass()
     ----> 3    debugger.Pdb().set_trace()
     <BLANKLINE>
     ipdb> continue
     
     Restore previous trace function, e.g. for coverage.py    
     
     >>> sys.settrace(old_trace)
     '''
 
 def test_ipdb_magics2():
     '''Test ipdb with a very short function.
     
     >>> old_trace = sys.gettrace()
 
     >>> def bar():
     ...     pass
 
     Run ipdb.
 
     >>> with PdbTestInput([
     ...    'continue',
     ... ]):
     ...     debugger.Pdb().runcall(bar)
     > <doctest ...>(2)bar()
           1 def bar():
     ----> 2    pass
     <BLANKLINE>
     ipdb> continue
     
     Restore previous trace function, e.g. for coverage.py    
     
     >>> sys.settrace(old_trace)
     '''
 
 def can_quit():
     '''Test that quit work in ipydb
 
     >>> old_trace = sys.gettrace()
 
     >>> def bar():
     ...     pass
 
     >>> with PdbTestInput([
     ...    'quit',
     ... ]):
     ...     debugger.Pdb().runcall(bar)
     > <doctest ...>(2)bar()
             1 def bar():
     ----> 2    pass
     <BLANKLINE>
     ipdb> quit
 
     Restore previous trace function, e.g. for coverage.py
 
     >>> sys.settrace(old_trace)
     '''
 
 
 def can_exit():
     '''Test that quit work in ipydb
 
     >>> old_trace = sys.gettrace()
 
     >>> def bar():
     ...     pass
 
     >>> with PdbTestInput([
     ...    'exit',
     ... ]):
     ...     debugger.Pdb().runcall(bar)
     > <doctest ...>(2)bar()
             1 def bar():
     ----> 2    pass
     <BLANKLINE>
     ipdb> exit
 
     Restore previous trace function, e.g. for coverage.py
 
     >>> sys.settrace(old_trace)
     '''
 
 
 def test_interruptible_core_debugger():
     """The debugger can be interrupted.
 
     The presumption is there is some mechanism that causes a KeyboardInterrupt
     (this is implemented in ipykernel).  We want to ensure the
     KeyboardInterrupt cause debugging to cease.
     """
     def raising_input(msg="", called=[0]):
         called[0] += 1
         if called[0] == 1:
             raise KeyboardInterrupt()
         else:
             raise AssertionError("input() should only be called once!")
 
     with patch.object(builtins, "input", raising_input):
         debugger.InterruptiblePdb().set_trace()
         # The way this test will fail is by set_trace() never exiting,
         # resulting in a timeout by the test runner. The alternative
         # implementation would involve a subprocess, but that adds issues with
         # interrupting subprocesses that are rather complex, so it's simpler
         # just to do it this way.
 
 @skip_win32
 def test_xmode_skip():
     """that xmode skip frames
 
     Not as a doctest as pytest does not run doctests.
     """
     import pexpect
     env = os.environ.copy()
     env["IPY_TEST_SIMPLE_PROMPT"] = "1"
 
     child = pexpect.spawn(
         sys.executable, ["-m", "IPython", "--colors=nocolor"], env=env
     )
     child.timeout = 15 * IPYTHON_TESTING_TIMEOUT_SCALE
 
     child.expect("IPython")
     child.expect("\n")
     child.expect_exact("In [1]")
 
     block = dedent(
         """
     def f():
         __tracebackhide__ = True
         g()
 
     def g():
         raise ValueError
 
     f()
     """
     )
 
     for line in block.splitlines():
         child.sendline(line)
         child.expect_exact(line)
     child.expect_exact("skipping")
 
     block = dedent(
         """
     def f():
         __tracebackhide__ = True
         g()
 
     def g():
         from IPython.core.debugger import set_trace
         set_trace()
 
     f()
     """
     )
 
     for line in block.splitlines():
         child.sendline(line)
         child.expect_exact(line)
 
     child.expect("ipdb>")
     child.sendline("w")
     child.expect("hidden")
     child.expect("ipdb>")
     child.sendline("skip_hidden false")
     child.sendline("w")
     child.expect("__traceba")
     child.expect("ipdb>")
 
     child.close()
 
 
 skip_decorators_blocks = (
     """
     def helpers_helper():
         pass # should not stop here except breakpoint
     """,
     """
     def helper_1():
         helpers_helper() # should not stop here
     """,
     """
     def helper_2():
         pass # should not stop here
     """,
     """
     def pdb_skipped_decorator2(function):
         def wrapped_fn(*args, **kwargs):
             __debuggerskip__ = True
             helper_2()
             __debuggerskip__ = False
             result = function(*args, **kwargs)
             __debuggerskip__ = True
             helper_2()
             return result
         return wrapped_fn
     """,
     """
     def pdb_skipped_decorator(function):
         def wrapped_fn(*args, **kwargs):
             __debuggerskip__ = True
             helper_1()
             __debuggerskip__ = False
             result = function(*args, **kwargs)
             __debuggerskip__ = True
             helper_2()
             return result
         return wrapped_fn
     """,
     """
     @pdb_skipped_decorator
     @pdb_skipped_decorator2
     def bar(x, y):
         return x * y
     """,
     """import IPython.terminal.debugger as ipdb""",
     """
     def f():
         ipdb.set_trace()
         bar(3, 4)
     """,
     """
     f()
     """,
 )
 
 
 def _decorator_skip_setup():
     import pexpect
 
     env = os.environ.copy()
     env["IPY_TEST_SIMPLE_PROMPT"] = "1"
 
     child = pexpect.spawn(
         sys.executable, ["-m", "IPython", "--colors=nocolor"], env=env
     )
+    child.str_last_chars = 1000
     child.timeout = 5 * IPYTHON_TESTING_TIMEOUT_SCALE
 
     child.expect("IPython")
     child.expect("\n")
 
     dedented_blocks = [dedent(b).strip() for b in skip_decorators_blocks]
     in_prompt_number = 1
     for cblock in dedented_blocks:
         child.expect_exact(f"In [{in_prompt_number}]:")
         in_prompt_number += 1
         for line in cblock.splitlines():
             child.sendline(line)
             child.expect_exact(line)
         child.sendline("")
     return child
 
 
 @skip_win32
 def test_decorator_skip():
     """test that decorator frames can be skipped."""
 
     child = _decorator_skip_setup()
 
     child.expect_exact("3     bar(3, 4)")
     child.expect("ipdb>")
 
     child.expect("ipdb>")
     child.sendline("step")
     child.expect_exact("step")
 
     child.expect_exact("1 @pdb_skipped_decorator")
 
     child.sendline("s")
     child.expect_exact("return x * y")
 
     child.close()
 
 
 @skip_win32
 def test_decorator_skip_disabled():
     """test that decorator frame skipping can be disabled"""
 
     child = _decorator_skip_setup()
 
     child.expect_exact("3     bar(3, 4)")
 
     for input_, expected in [
         ("skip_predicates debuggerskip False", ""),
         ("skip_predicates", "debuggerskip : False"),
         ("step", "---> 2     def wrapped_fn"),
         ("step", "----> 3         __debuggerskip__"),
         ("step", "----> 4         helper_1()"),
         ("step", "---> 1 def helper_1():"),
         ("next", "----> 2     helpers_helper()"),
         ("next", "--Return--"),
         ("next", "----> 5         __debuggerskip__ = False"),
     ]:
         child.expect("ipdb>")
         child.sendline(input_)
         child.expect_exact(input_)
         child.expect_exact(expected)
 
     child.close()
 
 
 @skip_win32
 def test_decorator_skip_with_breakpoint():
     """test that decorator frame skipping can be disabled"""
 
     import pexpect
 
     env = os.environ.copy()
     env["IPY_TEST_SIMPLE_PROMPT"] = "1"
 
     child = pexpect.spawn(
         sys.executable, ["-m", "IPython", "--colors=nocolor"], env=env
     )
     child.timeout = 5 * IPYTHON_TESTING_TIMEOUT_SCALE
 
     child.expect("IPython")
     child.expect("\n")
 
     ### we need a filename, so we need to exec the full block with a filename
     with NamedTemporaryFile(suffix=".py", dir=".", delete=True) as tf:
 
         name = tf.name[:-3].split("/")[-1]
         tf.write("\n".join([dedent(x) for x in skip_decorators_blocks[:-1]]).encode())
         tf.flush()
         codeblock = f"from {name} import f"
 
         dedented_blocks = [
             codeblock,
             "f()",
         ]
 
         in_prompt_number = 1
         for cblock in dedented_blocks:
             child.expect_exact(f"In [{in_prompt_number}]:")
             in_prompt_number += 1
             for line in cblock.splitlines():
                 child.sendline(line)
                 child.expect_exact(line)
             child.sendline("")
 
         # as the filename does not exists, we'll rely on the filename prompt
         child.expect_exact("47     bar(3, 4)")
 
         for input_, expected in [
             (f"b {name}.py:3", ""),
             ("step", "1---> 3     pass # should not stop here except"),
             ("step", "---> 38 @pdb_skipped_decorator"),
             ("continue", ""),
         ]:
             child.expect("ipdb>")
             child.sendline(input_)
             child.expect_exact(input_)
             child.expect_exact(expected)
 
     child.close()
 
 
 @skip_win32
 def test_where_erase_value():
     """Test that `where` does not access f_locals and erase values."""
     import pexpect
 
     env = os.environ.copy()
     env["IPY_TEST_SIMPLE_PROMPT"] = "1"
 
     child = pexpect.spawn(
         sys.executable, ["-m", "IPython", "--colors=nocolor"], env=env
     )
     child.timeout = 15 * IPYTHON_TESTING_TIMEOUT_SCALE
 
     child.expect("IPython")
     child.expect("\n")
     child.expect_exact("In [1]")
 
     block = dedent(
         """
     def simple_f():
          myvar = 1
          print(myvar)
          1/0
          print(myvar)
     simple_f()    """
     )
 
     for line in block.splitlines():
         child.sendline(line)
         child.expect_exact(line)
     child.expect_exact("ZeroDivisionError")
     child.expect_exact("In [2]:")
 
     child.sendline("%debug")
 
     ##
     child.expect("ipdb>")
 
     child.sendline("myvar")
     child.expect("1")
 
     ##
     child.expect("ipdb>")
 
     child.sendline("myvar = 2")
 
     ##
     child.expect_exact("ipdb>")
 
     child.sendline("myvar")
 
     child.expect_exact("2")
 
     ##
     child.expect("ipdb>")
     child.sendline("where")
 
     ##
     child.expect("ipdb>")
     child.sendline("myvar")
 
     child.expect_exact("2")
     child.expect("ipdb>")
 
     child.close()