IPython/core/inputtransformer2.py

"""Input transformer machinery to support IPython special syntax.

This includes the machinery to recognise and transform ``%magic`` commands,
``!system`` commands, ``help?`` querying, prompt stripping, and so forth.

Added: IPython 7.0. Replaces inputsplitter and inputtransformer which were
deprecated in 7.0.
"""

# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.

import ast
from codeop import CommandCompiler, Compile
import re
import sys
import tokenize
from typing import List, Tuple, Optional, Any
import warnings

from IPython.utils import tokenutil

_indent_re = re.compile(r'^[ \t]+')

def leading_empty_lines(lines):
    """Remove leading empty lines

    If the leading lines are empty or contain only whitespace, they will be
    removed.
    """
    if not lines:
        return lines
    for i, line in enumerate(lines):
        if line and not line.isspace():
            return lines[i:]
    return lines

def leading_indent(lines):
    """Remove leading indentation.

    If the first line starts with a spaces or tabs, the same whitespace will be
    removed from each following line in the cell.
    """
    if not lines:
        return lines
    m = _indent_re.match(lines[0])
    if not m:
        return lines
    space = m.group(0)
    n = len(space)
    return [l[n:] if l.startswith(space) else l
            for l in lines]

class PromptStripper:
    """Remove matching input prompts from a block of input.

    Parameters
    ----------
    prompt_re : regular expression
        A regular expression matching any input prompt (including continuation,
        e.g. ``...``)
    initial_re : regular expression, optional
        A regular expression matching only the initial prompt, but not continuation.
        If no initial expression is given, prompt_re will be used everywhere.
        Used mainly for plain Python prompts (``>>>``), where the continuation prompt
        ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.

    Notes
    -----

    If initial_re and prompt_re differ,
    only initial_re will be tested against the first line.
    If any prompt is found on the first two lines,
    prompts will be stripped from the rest of the block.
    """
    def __init__(self, prompt_re, initial_re=None):
        self.prompt_re = prompt_re
        self.initial_re = initial_re or prompt_re

    def _strip(self, lines):
        return [self.prompt_re.sub('', l, count=1) for l in lines]

    def __call__(self, lines):
        if not lines:
            return lines
        if self.initial_re.match(lines[0]) or \
                (len(lines) > 1 and self.prompt_re.match(lines[1])):
            return self._strip(lines)
        return lines

classic_prompt = PromptStripper(
    prompt_re=re.compile(r'^(>>>|\.\.\.)( |$)'),
    initial_re=re.compile(r'^>>>( |$)')
)

ipython_prompt = PromptStripper(
    re.compile(
        r"""
        ^(                         # Match from the beginning of a line, either:

                                   # 1. First-line prompt:
        ((\[nav\]|\[ins\])?\ )?    # Vi editing mode prompt, if it's there
        In\                        # The 'In' of the prompt, with a space
        \[\d+\]:                   # Command index, as displayed in the prompt
        \                          # With a mandatory trailing space

        |                          # ... or ...

                                   # 2. The three dots of the multiline prompt
        \s*                        # All leading whitespace characters
        \.{3,}:                    # The three (or more) dots
        \ ?                        # With an optional trailing space

        )
        """,
        re.VERBOSE,
    )
)


def cell_magic(lines):
    if not lines or not lines[0].startswith('%%'):
        return lines
    if re.match(r'%%\w+\?', lines[0]):
        # This case will be handled by help_end
        return lines
    magic_name, _, first_line = lines[0][2:].rstrip().partition(' ')
    body = ''.join(lines[1:])
    return ['get_ipython().run_cell_magic(%r, %r, %r)\n'
            % (magic_name, first_line, body)]


def _find_assign_op(token_line) -> Optional[int]:
    """Get the index of the first assignment in the line ('=' not inside brackets)

    Note: We don't try to support multiple special assignment (a = b = %foo)
    """
    paren_level = 0
    for i, ti in enumerate(token_line):
        s = ti.string
        if s == '=' and paren_level == 0:
            return i
        if s in {'(','[','{'}:
            paren_level += 1
        elif s in {')', ']', '}'}:
            if paren_level > 0:
                paren_level -= 1
    return None

def find_end_of_continued_line(lines, start_line: int):
    """Find the last line of a line explicitly extended using backslashes.

    Uses 0-indexed line numbers.
    """
    end_line = start_line
    while lines[end_line].endswith('\\\n'):
        end_line += 1
        if end_line >= len(lines):
            break
    return end_line

def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):
    r"""Assemble a single line from multiple continued line pieces

    Continued lines are lines ending in ``\``, and the line following the last
    ``\`` in the block.

    For example, this code continues over multiple lines::

        if (assign_ix is not None) \
             and (len(line) >= assign_ix + 2) \
             and (line[assign_ix+1].string == '%') \
             and (line[assign_ix+2].type == tokenize.NAME):

    This statement contains four continued line pieces.
    Assembling these pieces into a single line would give::

        if (assign_ix is not None) and (len(line) >= assign_ix + 2) and (line[...

    This uses 0-indexed line numbers. *start* is (lineno, colno).

    Used to allow ``%magic`` and ``!system`` commands to be continued over
    multiple lines.
    """
    parts = [lines[start[0]][start[1]:]] + lines[start[0]+1:end_line+1]
    return ' '.join([p.rstrip()[:-1] for p in parts[:-1]]  # Strip backslash+newline
                    + [parts[-1].rstrip()])         # Strip newline from last line

class TokenTransformBase:
    """Base class for transformations which examine tokens.

    Special syntax should not be transformed when it occurs inside strings or
    comments. This is hard to reliably avoid with regexes. The solution is to
    tokenise the code as Python, and recognise the special syntax in the tokens.

    IPython's special syntax is not valid Python syntax, so tokenising may go
    wrong after the special syntax starts. These classes therefore find and
    transform *one* instance of special syntax at a time into regular Python
    syntax. After each transformation, tokens are regenerated to find the next
    piece of special syntax.

    Subclasses need to implement one class method (find)
    and one regular method (transform).

    The priority attribute can select which transformation to apply if multiple
    transformers match in the same place. Lower numbers have higher priority.
    This allows "%magic?" to be turned into a help call rather than a magic call.
    """
    # Lower numbers -> higher priority (for matches in the same location)
    priority = 10

    def sortby(self):
        return self.start_line, self.start_col, self.priority

    def __init__(self, start):
        self.start_line = start[0] - 1   # Shift from 1-index to 0-index
        self.start_col = start[1]

    @classmethod
    def find(cls, tokens_by_line):
        """Find one instance of special syntax in the provided tokens.

        Tokens are grouped into logical lines for convenience,
        so it is easy to e.g. look at the first token of each line.
        *tokens_by_line* is a list of lists of tokenize.TokenInfo objects.

        This should return an instance of its class, pointing to the start
        position it has found, or None if it found no match.
        """
        raise NotImplementedError

    def transform(self, lines: List[str]):
        """Transform one instance of special syntax found by ``find()``

        Takes a list of strings representing physical lines,
        returns a similar list of transformed lines.
        """
        raise NotImplementedError

class MagicAssign(TokenTransformBase):
    """Transformer for assignments from magics (a = %foo)"""
    @classmethod
    def find(cls, tokens_by_line):
        """Find the first magic assignment (a = %foo) in the cell.
        """
        for line in tokens_by_line:
            assign_ix = _find_assign_op(line)
            if (assign_ix is not None) \
                    and (len(line) >= assign_ix + 2) \
                    and (line[assign_ix+1].string == '%') \
                    and (line[assign_ix+2].type == tokenize.NAME):
                return cls(line[assign_ix+1].start)

    def transform(self, lines: List[str]):
        """Transform a magic assignment found by the ``find()`` classmethod.
        """
        start_line, start_col = self.start_line, self.start_col
        lhs = lines[start_line][:start_col]
        end_line = find_end_of_continued_line(lines, start_line)
        rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
        assert rhs.startswith('%'), rhs
        magic_name, _, args = rhs[1:].partition(' ')

        lines_before = lines[:start_line]
        call = "get_ipython().run_line_magic({!r}, {!r})".format(magic_name, args)
        new_line = lhs + call + '\n'
        lines_after = lines[end_line+1:]

        return lines_before + [new_line] + lines_after


class SystemAssign(TokenTransformBase):
    """Transformer for assignments from system commands (a = !foo)"""
    @classmethod
    def find_pre_312(cls, tokens_by_line):
        for line in tokens_by_line:
            assign_ix = _find_assign_op(line)
            if (assign_ix is not None) \
                    and not line[assign_ix].line.strip().startswith('=') \
                    and (len(line) >= assign_ix + 2) \
                    and (line[assign_ix + 1].type == tokenize.ERRORTOKEN):
                ix = assign_ix + 1

                while ix < len(line) and line[ix].type == tokenize.ERRORTOKEN:
                    if line[ix].string == '!':
                        return cls(line[ix].start)
                    elif not line[ix].string.isspace():
                        break
                    ix += 1

    @classmethod
    def find_post_312(cls, tokens_by_line):
        for line in tokens_by_line:
            assign_ix = _find_assign_op(line)
            if (
                (assign_ix is not None)
                and not line[assign_ix].line.strip().startswith("=")
                and (len(line) >= assign_ix + 2)
                and (line[assign_ix + 1].type == tokenize.OP)
                and (line[assign_ix + 1].string == "!")
            ):
                return cls(line[assign_ix + 1].start)

    @classmethod
    def find(cls, tokens_by_line):
        """Find the first system assignment (a = !foo) in the cell."""
        if sys.version_info < (3, 12):
            return cls.find_pre_312(tokens_by_line)
        return cls.find_post_312(tokens_by_line)

    def transform(self, lines: List[str]):
        """Transform a system assignment found by the ``find()`` classmethod.
        """
        start_line, start_col = self.start_line, self.start_col

        lhs = lines[start_line][:start_col]
        end_line = find_end_of_continued_line(lines, start_line)
        rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
        assert rhs.startswith('!'), rhs
        cmd = rhs[1:]

        lines_before = lines[:start_line]
        call = "get_ipython().getoutput({!r})".format(cmd)
        new_line = lhs + call + '\n'
        lines_after = lines[end_line + 1:]

        return lines_before + [new_line] + lines_after

# The escape sequences that define the syntax transformations IPython will
# apply to user input.  These can NOT be just changed here: many regular
# expressions and other parts of the code may use their hardcoded values, and
# for all intents and purposes they constitute the 'IPython syntax', so they
# should be considered fixed.

ESC_SHELL  = '!'     # Send line to underlying system shell
ESC_SH_CAP = '!!'    # Send line to system shell and capture output
ESC_HELP   = '?'     # Find information about object
ESC_HELP2  = '??'    # Find extra-detailed information about object
ESC_MAGIC  = '%'     # Call magic function
ESC_MAGIC2 = '%%'    # Call cell-magic function
ESC_QUOTE  = ','     # Split args on whitespace, quote each as string and call
ESC_QUOTE2 = ';'     # Quote all args as a single string, call
ESC_PAREN  = '/'     # Call first argument with rest of line as arguments

ESCAPE_SINGLES = {'!', '?', '%', ',', ';', '/'}
ESCAPE_DOUBLES = {'!!', '??'}  # %% (cell magic) is handled separately

def _make_help_call(target, esc):
    """Prepares a pinfo(2)/psearch call from a target name and the escape
    (i.e. ? or ??)"""
    method  = 'pinfo2' if esc == '??' \
                else 'psearch' if '*' in target \
                else 'pinfo'
    arg = " ".join([method, target])
    #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
    t_magic_name, _, t_magic_arg_s = arg.partition(' ')
    t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
    return "get_ipython().run_line_magic(%r, %r)" % (t_magic_name, t_magic_arg_s)


def _tr_help(content):
    """Translate lines escaped with: ?

    A naked help line should fire the intro help screen (shell.show_usage())
    """
    if not content:
        return 'get_ipython().show_usage()'

    return _make_help_call(content, '?')

def _tr_help2(content):
    """Translate lines escaped with: ??

    A naked help line should fire the intro help screen (shell.show_usage())
    """
    if not content:
        return 'get_ipython().show_usage()'

    return _make_help_call(content, '??')

def _tr_magic(content):
    "Translate lines escaped with a percent sign: %"
    name, _, args = content.partition(' ')
    return 'get_ipython().run_line_magic(%r, %r)' % (name, args)

def _tr_quote(content):
    "Translate lines escaped with a comma: ,"
    name, _, args = content.partition(' ')
    return '%s("%s")' % (name, '", "'.join(args.split()) )

def _tr_quote2(content):
    "Translate lines escaped with a semicolon: ;"
    name, _, args = content.partition(' ')
    return '%s("%s")' % (name, args)

def _tr_paren(content):
    "Translate lines escaped with a slash: /"
    name, _, args = content.partition(" ")
    if name == "":
        raise SyntaxError(f'"{ESC_SHELL}" must be followed by a callable name')

    return '%s(%s)' % (name, ", ".join(args.split()))

tr = { ESC_SHELL  : 'get_ipython().system({!r})'.format,
       ESC_SH_CAP : 'get_ipython().getoutput({!r})'.format,
       ESC_HELP   : _tr_help,
       ESC_HELP2  : _tr_help2,
       ESC_MAGIC  : _tr_magic,
       ESC_QUOTE  : _tr_quote,
       ESC_QUOTE2 : _tr_quote2,
       ESC_PAREN  : _tr_paren }

class EscapedCommand(TokenTransformBase):
    """Transformer for escaped commands like %foo, !foo, or /foo"""
    @classmethod
    def find(cls, tokens_by_line):
        """Find the first escaped command (%foo, !foo, etc.) in the cell.
        """
        for line in tokens_by_line:
            if not line:
                continue
            ix = 0
            ll = len(line)
            while ll > ix and line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
                ix += 1
            if ix >= ll:
                continue
            if line[ix].string in ESCAPE_SINGLES:
                return cls(line[ix].start)

    def transform(self, lines):
        """Transform an escaped line found by the ``find()`` classmethod.
        """
        start_line, start_col = self.start_line, self.start_col

        indent = lines[start_line][:start_col]
        end_line = find_end_of_continued_line(lines, start_line)
        line = assemble_continued_line(lines, (start_line, start_col), end_line)

        if len(line) > 1 and line[:2] in ESCAPE_DOUBLES:
            escape, content = line[:2], line[2:]
        else:
            escape, content = line[:1], line[1:]

        if escape in tr:
            call = tr[escape](content)
        else:
            call = ''

        lines_before = lines[:start_line]
        new_line = indent + call + '\n'
        lines_after = lines[end_line + 1:]

        return lines_before + [new_line] + lines_after


_help_end_re = re.compile(
    r"""(%{0,2}
    (?!\d)[\w*]+            # Variable name
    (\.(?!\d)[\w*]+|\[-?[0-9]+\])*       # .etc.etc or [0], we only support literal integers.
    )
    (\?\??)$                # ? or ??
    """,
    re.VERBOSE,
)


class HelpEnd(TokenTransformBase):
    """Transformer for help syntax: obj? and obj??"""
    # This needs to be higher priority (lower number) than EscapedCommand so
    # that inspecting magics (%foo?) works.
    priority = 5

    def __init__(self, start, q_locn):
        super().__init__(start)
        self.q_line = q_locn[0] - 1  # Shift from 1-indexed to 0-indexed
        self.q_col = q_locn[1]

    @classmethod
    def find(cls, tokens_by_line):
        """Find the first help command (foo?) in the cell.
        """
        for line in tokens_by_line:
            # Last token is NEWLINE; look at last but one
            if len(line) > 2 and line[-2].string == '?':
                # Find the first token that's not INDENT/DEDENT
                ix = 0
                while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
                    ix += 1
                return cls(line[ix].start, line[-2].start)

    def transform(self, lines):
        """Transform a help command found by the ``find()`` classmethod.
        """

        piece = "".join(lines[self.start_line : self.q_line + 1])
        indent, content = piece[: self.start_col], piece[self.start_col :]
        lines_before = lines[: self.start_line]
        lines_after = lines[self.q_line + 1 :]

        m = _help_end_re.search(content)
        if not m:
            raise SyntaxError(content)
        assert m is not None, content
        target = m.group(1)
        esc = m.group(3)


        call = _make_help_call(target, esc)
        new_line = indent + call + '\n'

        return lines_before + [new_line] + lines_after

def make_tokens_by_line(lines:List[str]):
    """Tokenize a series of lines and group tokens by line.

    The tokens for a multiline Python string or expression are grouped as one
    line. All lines except the last lines should keep their line ending ('\\n',
    '\\r\\n') for this to properly work. Use `.splitlines(keeplineending=True)`
    for example when passing block of text to this function.

    """
    # NL tokens are used inside multiline expressions, but also after blank
    # lines or comments. This is intentional - see https://bugs.python.org/issue17061
    # We want to group the former case together but split the latter, so we
    # track parentheses level, similar to the internals of tokenize.

    #   reexported from token on 3.7+
    NEWLINE, NL = tokenize.NEWLINE, tokenize.NL  # type: ignore
    tokens_by_line: List[List[Any]] = [[]]
    if len(lines) > 1 and not lines[0].endswith(("\n", "\r", "\r\n", "\x0b", "\x0c")):
        warnings.warn(
            "`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified",
            stacklevel=2,
        )
    parenlev = 0
    try:
        for token in tokenutil.generate_tokens_catch_errors(
            iter(lines).__next__, extra_errors_to_catch=["expected EOF"]
        ):
            tokens_by_line[-1].append(token)
            if (token.type == NEWLINE) \
                    or ((token.type == NL) and (parenlev <= 0)):
                tokens_by_line.append([])
            elif token.string in {'(', '[', '{'}:
                parenlev += 1
            elif token.string in {')', ']', '}'}:
                if parenlev > 0:
                    parenlev -= 1
    except tokenize.TokenError:
        # Input ended in a multiline string or expression. That's OK for us.
        pass


    if not tokens_by_line[-1]:
        tokens_by_line.pop()


    return tokens_by_line


def has_sunken_brackets(tokens: List[tokenize.TokenInfo]):
    """Check if the depth of brackets in the list of tokens drops below 0"""
    parenlev = 0
    for token in tokens:
        if token.string in {"(", "[", "{"}:
            parenlev += 1
        elif token.string in {")", "]", "}"}:
            parenlev -= 1
            if parenlev < 0:
                return True
    return False


def show_linewise_tokens(s: str):
    """For investigation and debugging"""
    warnings.warn(
        "show_linewise_tokens is deprecated since IPython 8.6",
        DeprecationWarning,
        stacklevel=2,
    )
    if not s.endswith("\n"):
        s += "\n"
    lines = s.splitlines(keepends=True)
    for line in make_tokens_by_line(lines):
        print("Line -------")
        for tokinfo in line:
            print(" ", tokinfo)

# Arbitrary limit to prevent getting stuck in infinite loops
TRANSFORM_LOOP_LIMIT = 500

class TransformerManager:
    """Applies various transformations to a cell or code block.

    The key methods for external use are ``transform_cell()``
    and ``check_complete()``.
    """
    def __init__(self):
        self.cleanup_transforms = [
            leading_empty_lines,
            leading_indent,
            classic_prompt,
            ipython_prompt,
        ]
        self.line_transforms = [
            cell_magic,
        ]
        self.token_transformers = [
            MagicAssign,
            SystemAssign,
            EscapedCommand,
            HelpEnd,
        ]

    def do_one_token_transform(self, lines):
        """Find and run the transform earliest in the code.

        Returns (changed, lines).

        This method is called repeatedly until changed is False, indicating
        that all available transformations are complete.

        The tokens following IPython special syntax might not be valid, so
        the transformed code is retokenised every time to identify the next
        piece of special syntax. Hopefully long code cells are mostly valid
        Python, not using lots of IPython special syntax, so this shouldn't be
        a performance issue.
        """
        tokens_by_line = make_tokens_by_line(lines)
        candidates = []
        for transformer_cls in self.token_transformers:
            transformer = transformer_cls.find(tokens_by_line)
            if transformer:
                candidates.append(transformer)

        if not candidates:
            # Nothing to transform
            return False, lines
        ordered_transformers = sorted(candidates, key=TokenTransformBase.sortby)
        for transformer in ordered_transformers:
            try:
                return True, transformer.transform(lines)
            except SyntaxError:
                pass
        return False, lines

    def do_token_transforms(self, lines):
        for _ in range(TRANSFORM_LOOP_LIMIT):
            changed, lines = self.do_one_token_transform(lines)
            if not changed:
                return lines

        raise RuntimeError("Input transformation still changing after "
                           "%d iterations. Aborting." % TRANSFORM_LOOP_LIMIT)

    def transform_cell(self, cell: str) -> str:
        """Transforms a cell of input code"""
        if not cell.endswith('\n'):
            cell += '\n'  # Ensure the cell has a trailing newline
        lines = cell.splitlines(keepends=True)
        for transform in self.cleanup_transforms + self.line_transforms:
            lines = transform(lines)

        lines = self.do_token_transforms(lines)
        return ''.join(lines)

    def check_complete(self, cell: str):
        """Return whether a block of code is ready to execute, or should be continued

        Parameters
        ----------
        cell : string
            Python input code, which can be multiline.

        Returns
        -------
        status : str
            One of 'complete', 'incomplete', or 'invalid' if source is not a
            prefix of valid code.
        indent_spaces : int or None
            The number of spaces by which to indent the next line of code. If
            status is not 'incomplete', this is None.
        """
        # Remember if the lines ends in a new line.
        ends_with_newline = False
        for character in reversed(cell):
            if character == '\n':
                ends_with_newline = True
                break
            elif character.strip():
                break
            else:
                continue

        if not ends_with_newline:
            # Append an newline for consistent tokenization
            # See https://bugs.python.org/issue33899
            cell += '\n'

        lines = cell.splitlines(keepends=True)

        if not lines:
            return 'complete', None

        for line in reversed(lines):
            if not line.strip():
                continue
            elif line.strip("\n").endswith("\\"):
                return "incomplete", find_last_indent(lines)
            else:
                break

        try:
            for transform in self.cleanup_transforms:
                if not getattr(transform, 'has_side_effects', False):
                    lines = transform(lines)
        except SyntaxError:
            return 'invalid', None

        if lines[0].startswith('%%'):
            # Special case for cell magics - completion marked by blank line
            if lines[-1].strip():
                return 'incomplete', find_last_indent(lines)
            else:
                return 'complete', None

        try:
            for transform in self.line_transforms:
                if not getattr(transform, 'has_side_effects', False):
                    lines = transform(lines)
            lines = self.do_token_transforms(lines)
        except SyntaxError:
            return 'invalid', None

        tokens_by_line = make_tokens_by_line(lines)

        # Bail if we got one line and there are more closing parentheses than
        # the opening ones
        if (
            len(lines) == 1
            and tokens_by_line
            and has_sunken_brackets(tokens_by_line[0])
        ):
            return "invalid", None

        if not tokens_by_line:
            return 'incomplete', find_last_indent(lines)

        if (
            tokens_by_line[-1][-1].type != tokenize.ENDMARKER
            and tokens_by_line[-1][-1].type != tokenize.ERRORTOKEN
        ):
            # We're in a multiline string or expression
            return 'incomplete', find_last_indent(lines)

        newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER} # type: ignore

        # Pop the last line which only contains DEDENTs and ENDMARKER
        last_token_line = None
        if {t.type for t in tokens_by_line[-1]} in [
            {tokenize.DEDENT, tokenize.ENDMARKER},
            {tokenize.ENDMARKER}
        ] and len(tokens_by_line) > 1:
            last_token_line = tokens_by_line.pop()

        while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types:
            tokens_by_line[-1].pop()

        if not tokens_by_line[-1]:
            return 'incomplete', find_last_indent(lines)

        if tokens_by_line[-1][-1].string == ':':
            # The last line starts a block (e.g. 'if foo:')
            ix = 0
            while tokens_by_line[-1][ix].type in {tokenize.INDENT, tokenize.DEDENT}:
                ix += 1

            indent = tokens_by_line[-1][ix].start[1]
            return 'incomplete', indent + 4

        if tokens_by_line[-1][0].line.endswith('\\'):
            return 'incomplete', None

        # At this point, our checks think the code is complete (or invalid).
        # We'll use codeop.compile_command to check this with the real parser
        try:
            with warnings.catch_warnings():
                warnings.simplefilter('error', SyntaxWarning)
                res = compile_command(''.join(lines), symbol='exec')
        except (SyntaxError, OverflowError, ValueError, TypeError,
                MemoryError, SyntaxWarning):
            return 'invalid', None
        else:
            if res is None:
                return 'incomplete', find_last_indent(lines)

        if last_token_line and last_token_line[0].type == tokenize.DEDENT:
            if ends_with_newline:
                return 'complete', None
            return 'incomplete', find_last_indent(lines)

        # If there's a blank line at the end, assume we're ready to execute
        if not lines[-1].strip():
            return 'complete', None

        return 'complete', None


def find_last_indent(lines):
    m = _indent_re.match(lines[-1])
    if not m:
        return 0
    return len(m.group(0).replace('\t', ' '*4))


class MaybeAsyncCompile(Compile):
    def __init__(self, extra_flags=0):
        super().__init__()
        self.flags |= extra_flags


class MaybeAsyncCommandCompiler(CommandCompiler):
    def __init__(self, extra_flags=0):
        self.compiler = MaybeAsyncCompile(extra_flags=extra_flags)


_extra_flags = ast.PyCF_ALLOW_TOP_LEVEL_AWAIT

compile_command = MaybeAsyncCommandCompiler(extra_flags=_extra_flags)