diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py deleted file mode 100644 index b20dfb1..0000000 --- a/IPython/core/inputsplitter.py +++ /dev/null @@ -1,799 +0,0 @@ -"""DEPRECATED: Input handling and transformation machinery. - -This module was deprecated in IPython 7.0, in favour of inputtransformer2. - -The first class in this module, :class:`InputSplitter`, is designed to tell when -input from a line-oriented frontend is complete and should be executed, and when -the user should be prompted for another line of code instead. The name 'input -splitter' is largely for historical reasons. - -A companion, :class:`IPythonInputSplitter`, provides the same functionality but -with full support for the extended IPython syntax (magics, system calls, etc). -The code to actually do these transformations is in :mod:`IPython.core.inputtransformer`. -:class:`IPythonInputSplitter` feeds the raw code to the transformers in order -and stores the results. - -For more details, see the class docstrings below. -""" - -from __future__ import annotations - -from warnings import warn - -warn('IPython.core.inputsplitter is deprecated since IPython 7 in favor of `IPython.core.inputtransformer2`', - DeprecationWarning) - -# Copyright (c) IPython Development Team. -# Distributed under the terms of the Modified BSD License. -import ast -import codeop -import io -import re -import sys -import tokenize -import warnings - -from typing import List, Tuple, Union, Optional, TYPE_CHECKING -from types import CodeType - -from IPython.core.inputtransformer import (leading_indent, - classic_prompt, - ipy_prompt, - cellmagic, - assemble_logical_lines, - help_end, - escaped_commands, - assign_from_magic, - assign_from_system, - assemble_python_lines, - ) -from IPython.utils import tokenutil - -# These are available in this module for backwards compatibility. -from IPython.core.inputtransformer import (ESC_SHELL, ESC_SH_CAP, ESC_HELP, - ESC_HELP2, ESC_MAGIC, ESC_MAGIC2, - ESC_QUOTE, ESC_QUOTE2, ESC_PAREN, ESC_SEQUENCES) - -if TYPE_CHECKING: - from typing_extensions import Self -#----------------------------------------------------------------------------- -# Utilities -#----------------------------------------------------------------------------- - -# FIXME: These are general-purpose utilities that later can be moved to the -# general ward. Kept here for now because we're being very strict about test -# coverage with this code, and this lets us ensure that we keep 100% coverage -# while developing. - -# compiled regexps for autoindent management -dedent_re = re.compile('|'.join([ - r'^\s+raise(\s.*)?$', # raise statement (+ space + other stuff, maybe) - r'^\s+raise\([^\)]*\).*$', # wacky raise with immediate open paren - r'^\s+return(\s.*)?$', # normal return (+ space + other stuff, maybe) - r'^\s+return\([^\)]*\).*$', # wacky return with immediate open paren - r'^\s+pass\s*$', # pass (optionally followed by trailing spaces) - r'^\s+break\s*$', # break (optionally followed by trailing spaces) - r'^\s+continue\s*$', # continue (optionally followed by trailing spaces) -])) -ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)') - -# regexp to match pure comment lines so we don't accidentally insert 'if 1:' -# before pure comments -comment_line_re = re.compile(r'^\s*\#') - - -def num_ini_spaces(s): - """Return the number of initial spaces in a string. - - Note that tabs are counted as a single space. For now, we do *not* support - mixing of tabs and spaces in the user's input. - - Parameters - ---------- - s : string - - Returns - ------- - n : int - """ - warnings.warn( - "`num_ini_spaces` is Pending Deprecation since IPython 8.17." - "It is considered for removal in in future version. " - "Please open an issue if you believe it should be kept.", - stacklevel=2, - category=PendingDeprecationWarning, - ) - ini_spaces = ini_spaces_re.match(s) - if ini_spaces: - return ini_spaces.end() - else: - return 0 - -# Fake token types for partial_tokenize: -INCOMPLETE_STRING = tokenize.N_TOKENS -IN_MULTILINE_STATEMENT = tokenize.N_TOKENS + 1 - -# The 2 classes below have the same API as TokenInfo, but don't try to look up -# a token type name that they won't find. -class IncompleteString: - type = exact_type = INCOMPLETE_STRING - def __init__(self, s, start, end, line): - self.s = s - self.start = start - self.end = end - self.line = line - -class InMultilineStatement: - type = exact_type = IN_MULTILINE_STATEMENT - def __init__(self, pos, line): - self.s = '' - self.start = self.end = pos - self.line = line - -def partial_tokens(s): - """Iterate over tokens from a possibly-incomplete string of code. - - This adds two special token types: INCOMPLETE_STRING and - IN_MULTILINE_STATEMENT. These can only occur as the last token yielded, and - represent the two main ways for code to be incomplete. - """ - readline = io.StringIO(s).readline - token = tokenize.TokenInfo(tokenize.NEWLINE, '', (1, 0), (1, 0), '') - try: - for token in tokenutil.generate_tokens_catch_errors(readline): - yield token - except tokenize.TokenError as e: - # catch EOF error - lines = s.splitlines(keepends=True) - end = len(lines), len(lines[-1]) - if 'multi-line string' in e.args[0]: - l, c = start = token.end - s = lines[l-1][c:] + ''.join(lines[l:]) - yield IncompleteString(s, start, end, lines[-1]) - elif 'multi-line statement' in e.args[0]: - yield InMultilineStatement(end, lines[-1]) - else: - raise - -def find_next_indent(code) -> int: - """Find the number of spaces for the next line of indentation""" - tokens = list(partial_tokens(code)) - if tokens[-1].type == tokenize.ENDMARKER: - tokens.pop() - if not tokens: - return 0 - - while tokens[-1].type in { - tokenize.DEDENT, - tokenize.NEWLINE, - tokenize.COMMENT, - tokenize.ERRORTOKEN, - }: - tokens.pop() - - # Starting in Python 3.12, the tokenize module adds implicit newlines at the end - # of input. We need to remove those if we're in a multiline statement - if tokens[-1].type == IN_MULTILINE_STATEMENT: - while tokens[-2].type in {tokenize.NL}: - tokens.pop(-2) - - - if tokens[-1].type == INCOMPLETE_STRING: - # Inside a multiline string - return 0 - - # Find the indents used before - prev_indents = [0] - def _add_indent(n): - if n != prev_indents[-1]: - prev_indents.append(n) - - tokiter = iter(tokens) - for tok in tokiter: - if tok.type in {tokenize.INDENT, tokenize.DEDENT}: - _add_indent(tok.end[1]) - elif (tok.type == tokenize.NL): - try: - _add_indent(next(tokiter).start[1]) - except StopIteration: - break - - last_indent = prev_indents.pop() - - # If we've just opened a multiline statement (e.g. 'a = ['), indent more - if tokens[-1].type == IN_MULTILINE_STATEMENT: - if tokens[-2].exact_type in {tokenize.LPAR, tokenize.LSQB, tokenize.LBRACE}: - return last_indent + 4 - return last_indent - - if tokens[-1].exact_type == tokenize.COLON: - # Line ends with colon - indent - return last_indent + 4 - - if last_indent: - # Examine the last line for dedent cues - statements like return or - # raise which normally end a block of code. - last_line_starts = 0 - for i, tok in enumerate(tokens): - if tok.type == tokenize.NEWLINE: - last_line_starts = i + 1 - - last_line_tokens = tokens[last_line_starts:] - names = [t.string for t in last_line_tokens if t.type == tokenize.NAME] - if names and names[0] in {'raise', 'return', 'pass', 'break', 'continue'}: - # Find the most recent indentation less than the current level - for indent in reversed(prev_indents): - if indent < last_indent: - return indent - - return last_indent - - -def last_blank(src): - """Determine if the input source ends in a blank. - - A blank is either a newline or a line consisting of whitespace. - - Parameters - ---------- - src : string - A single or multiline string. - """ - if not src: return False - ll = src.splitlines()[-1] - return (ll == '') or ll.isspace() - - -last_two_blanks_re = re.compile(r'\n\s*\n\s*$', re.MULTILINE) -last_two_blanks_re2 = re.compile(r'.+\n\s*\n\s+$', re.MULTILINE) - -def last_two_blanks(src): - """Determine if the input source ends in two blanks. - - A blank is either a newline or a line consisting of whitespace. - - Parameters - ---------- - src : string - A single or multiline string. - """ - if not src: return False - # The logic here is tricky: I couldn't get a regexp to work and pass all - # the tests, so I took a different approach: split the source by lines, - # grab the last two and prepend '###\n' as a stand-in for whatever was in - # the body before the last two lines. Then, with that structure, it's - # possible to analyze with two regexps. Not the most elegant solution, but - # it works. If anyone tries to change this logic, make sure to validate - # the whole test suite first! - new_src = '\n'.join(['###\n'] + src.splitlines()[-2:]) - return (bool(last_two_blanks_re.match(new_src)) or - bool(last_two_blanks_re2.match(new_src)) ) - - -def remove_comments(src): - """Remove all comments from input source. - - Note: comments are NOT recognized inside of strings! - - Parameters - ---------- - src : string - A single or multiline input string. - - Returns - ------- - String with all Python comments removed. - """ - - return re.sub('#.*', '', src) - - -def get_input_encoding(): - """Return the default standard input encoding. - - If sys.stdin has no encoding, 'ascii' is returned.""" - # There are strange environments for which sys.stdin.encoding is None. We - # ensure that a valid encoding is returned. - encoding = getattr(sys.stdin, 'encoding', None) - if encoding is None: - encoding = 'ascii' - return encoding - -#----------------------------------------------------------------------------- -# Classes and functions for normal Python syntax handling -#----------------------------------------------------------------------------- - -class InputSplitter(object): - r"""An object that can accumulate lines of Python source before execution. - - This object is designed to be fed python source line-by-line, using - :meth:`push`. It will return on each push whether the currently pushed - code could be executed already. In addition, it provides a method called - :meth:`push_accepts_more` that can be used to query whether more input - can be pushed into a single interactive block. - - This is a simple example of how an interactive terminal-based client can use - this tool:: - - isp = InputSplitter() - while isp.push_accepts_more(): - indent = ' '*isp.indent_spaces - prompt = '>>> ' + indent - line = indent + raw_input(prompt) - isp.push(line) - print('Input source was:\n', isp.source_reset()) - """ - # A cache for storing the current indentation - # The first value stores the most recently processed source input - # The second value is the number of spaces for the current indentation - # If self.source matches the first value, the second value is a valid - # current indentation. Otherwise, the cache is invalid and the indentation - # must be recalculated. - _indent_spaces_cache: Union[Tuple[None, None], Tuple[str, int]] = None, None - # String, indicating the default input encoding. It is computed by default - # at initialization time via get_input_encoding(), but it can be reset by a - # client with specific knowledge of the encoding. - encoding = '' - # String where the current full source input is stored, properly encoded. - # Reading this attribute is the normal way of querying the currently pushed - # source code, that has been properly encoded. - source: str = "" - # Code object corresponding to the current source. It is automatically - # synced to the source, so it can be queried at any time to obtain the code - # object; it will be None if the source doesn't compile to valid Python. - code: Optional[CodeType] = None - - # Private attributes - - # List with lines of input accumulated so far - _buffer: List[str] - # Command compiler - _compile: codeop.CommandCompiler - # Boolean indicating whether the current block is complete - _is_complete: Optional[bool] = None - # Boolean indicating whether the current block has an unrecoverable syntax error - _is_invalid: bool = False - - def __init__(self) -> None: - """Create a new InputSplitter instance.""" - self._buffer = [] - self._compile = codeop.CommandCompiler() - self.encoding = get_input_encoding() - - def reset(self): - """Reset the input buffer and associated state.""" - self._buffer[:] = [] - self.source = '' - self.code = None - self._is_complete = False - self._is_invalid = False - - def source_reset(self): - """Return the input source and perform a full reset. - """ - out = self.source - self.reset() - return out - - def check_complete(self, source): - """Return whether a block of code is ready to execute, or should be continued - - This is a non-stateful API, and will reset the state of this InputSplitter. - - Parameters - ---------- - source : string - Python input code, which can be multiline. - - Returns - ------- - status : str - One of 'complete', 'incomplete', or 'invalid' if source is not a - prefix of valid code. - indent_spaces : int or None - The number of spaces by which to indent the next line of code. If - status is not 'incomplete', this is None. - """ - self.reset() - try: - self.push(source) - except SyntaxError: - # Transformers in IPythonInputSplitter can raise SyntaxError, - # which push() will not catch. - return 'invalid', None - else: - if self._is_invalid: - return 'invalid', None - elif self.push_accepts_more(): - return 'incomplete', self.get_indent_spaces() - else: - return 'complete', None - finally: - self.reset() - - def push(self, lines:str) -> bool: - """Push one or more lines of input. - - This stores the given lines and returns a status code indicating - whether the code forms a complete Python block or not. - - Any exceptions generated in compilation are swallowed, but if an - exception was produced, the method returns True. - - Parameters - ---------- - lines : string - One or more lines of Python input. - - Returns - ------- - is_complete : boolean - True if the current input source (the result of the current input - plus prior inputs) forms a complete Python execution block. Note that - this value is also stored as a private attribute (``_is_complete``), so it - can be queried at any time. - """ - assert isinstance(lines, str) - self._store(lines) - source = self.source - - # Before calling _compile(), reset the code object to None so that if an - # exception is raised in compilation, we don't mislead by having - # inconsistent code/source attributes. - self.code, self._is_complete = None, None - self._is_invalid = False - - # Honor termination lines properly - if source.endswith('\\\n'): - return False - - try: - with warnings.catch_warnings(): - warnings.simplefilter('error', SyntaxWarning) - self.code = self._compile(source, symbol="exec") - # Invalid syntax can produce any of a number of different errors from - # inside the compiler, so we have to catch them all. Syntax errors - # immediately produce a 'ready' block, so the invalid Python can be - # sent to the kernel for evaluation with possible ipython - # special-syntax conversion. - except (SyntaxError, OverflowError, ValueError, TypeError, - MemoryError, SyntaxWarning): - self._is_complete = True - self._is_invalid = True - else: - # Compilation didn't produce any exceptions (though it may not have - # given a complete code object) - self._is_complete = self.code is not None - - return self._is_complete - - def push_accepts_more(self): - """Return whether a block of interactive input can accept more input. - - This method is meant to be used by line-oriented frontends, who need to - guess whether a block is complete or not based solely on prior and - current input lines. The InputSplitter considers it has a complete - interactive block and will not accept more input when either: - - * A SyntaxError is raised - - * The code is complete and consists of a single line or a single - non-compound statement - - * The code is complete and has a blank line at the end - - If the current input produces a syntax error, this method immediately - returns False but does *not* raise the syntax error exception, as - typically clients will want to send invalid syntax to an execution - backend which might convert the invalid syntax into valid Python via - one of the dynamic IPython mechanisms. - """ - - # With incomplete input, unconditionally accept more - # A syntax error also sets _is_complete to True - see push() - if not self._is_complete: - #print("Not complete") # debug - return True - - # The user can make any (complete) input execute by leaving a blank line - last_line = self.source.splitlines()[-1] - if (not last_line) or last_line.isspace(): - #print("Blank line") # debug - return False - - # If there's just a single line or AST node, and we're flush left, as is - # the case after a simple statement such as 'a=1', we want to execute it - # straight away. - if self.get_indent_spaces() == 0: - if len(self.source.splitlines()) <= 1: - return False - - try: - code_ast = ast.parse("".join(self._buffer)) - except Exception: - #print("Can't parse AST") # debug - return False - else: - if len(code_ast.body) == 1 and \ - not hasattr(code_ast.body[0], 'body'): - #print("Simple statement") # debug - return False - - # General fallback - accept more code - return True - - def get_indent_spaces(self) -> int: - sourcefor, n = self._indent_spaces_cache - if sourcefor == self.source: - assert n is not None - return n - - # self.source always has a trailing newline - n = find_next_indent(self.source[:-1]) - self._indent_spaces_cache = (self.source, n) - return n - - # Backwards compatibility. I think all code that used .indent_spaces was - # inside IPython, but we can leave this here until IPython 7 in case any - # other modules are using it. -TK, November 2017 - indent_spaces = property(get_indent_spaces) - - def _store(self, lines, buffer=None, store='source'): - """Store one or more lines of input. - - If input lines are not newline-terminated, a newline is automatically - appended.""" - - if buffer is None: - buffer = self._buffer - - if lines.endswith('\n'): - buffer.append(lines) - else: - buffer.append(lines+'\n') - setattr(self, store, self._set_source(buffer)) - - def _set_source(self, buffer): - return u''.join(buffer) - - -class IPythonInputSplitter(InputSplitter): - """An input splitter that recognizes all of IPython's special syntax.""" - - # String with raw, untransformed input. - source_raw = '' - - # Flag to track when a transformer has stored input that it hasn't given - # back yet. - transformer_accumulating = False - - # Flag to track when assemble_python_lines has stored input that it hasn't - # given back yet. - within_python_line = False - - # Private attributes - - # List with lines of raw input accumulated so far. - _buffer_raw: List[str] - - def __init__(self, line_input_checker=True, physical_line_transforms=None, - logical_line_transforms=None, python_line_transforms=None): - super(IPythonInputSplitter, self).__init__() - self._buffer_raw = [] - self._validate = True - - if physical_line_transforms is not None: - self.physical_line_transforms = physical_line_transforms - else: - self.physical_line_transforms = [ - leading_indent(), - classic_prompt(), - ipy_prompt(), - cellmagic(end_on_blank_line=line_input_checker), - ] - - self.assemble_logical_lines = assemble_logical_lines() - if logical_line_transforms is not None: - self.logical_line_transforms = logical_line_transforms - else: - self.logical_line_transforms = [ - help_end(), - escaped_commands(), - assign_from_magic(), - assign_from_system(), - ] - - self.assemble_python_lines = assemble_python_lines() - if python_line_transforms is not None: - self.python_line_transforms = python_line_transforms - else: - # We don't use any of these at present - self.python_line_transforms = [] - - @property - def transforms(self): - "Quick access to all transformers." - return self.physical_line_transforms + \ - [self.assemble_logical_lines] + self.logical_line_transforms + \ - [self.assemble_python_lines] + self.python_line_transforms - - @property - def transforms_in_use(self): - """Transformers, excluding logical line transformers if we're in a - Python line.""" - t = self.physical_line_transforms[:] - if not self.within_python_line: - t += [self.assemble_logical_lines] + self.logical_line_transforms - return t + [self.assemble_python_lines] + self.python_line_transforms - - def reset(self): - """Reset the input buffer and associated state.""" - super(IPythonInputSplitter, self).reset() - self._buffer_raw[:] = [] - self.source_raw = '' - self.transformer_accumulating = False - self.within_python_line = False - - for t in self.transforms: - try: - t.reset() - except SyntaxError: - # Nothing that calls reset() expects to handle transformer - # errors - pass - - def flush_transformers(self: Self): - def _flush(transform, outs: List[str]): - """yield transformed lines - - always strings, never None - - transform: the current transform - outs: an iterable of previously transformed inputs. - Each may be multiline, which will be passed - one line at a time to transform. - """ - for out in outs: - for line in out.splitlines(): - # push one line at a time - tmp = transform.push(line) - if tmp is not None: - yield tmp - - # reset the transform - tmp = transform.reset() - if tmp is not None: - yield tmp - - out: List[str] = [] - for t in self.transforms_in_use: - out = _flush(t, out) - - out = list(out) - if out: - self._store('\n'.join(out)) - - def raw_reset(self): - """Return raw input only and perform a full reset. - """ - out = self.source_raw - self.reset() - return out - - def source_reset(self): - try: - self.flush_transformers() - return self.source - finally: - self.reset() - - def push_accepts_more(self): - if self.transformer_accumulating: - return True - else: - return super(IPythonInputSplitter, self).push_accepts_more() - - def transform_cell(self, cell): - """Process and translate a cell of input. - """ - self.reset() - try: - self.push(cell) - self.flush_transformers() - return self.source - finally: - self.reset() - - def push(self, lines:str) -> bool: - """Push one or more lines of IPython input. - - This stores the given lines and returns a status code indicating - whether the code forms a complete Python block or not, after processing - all input lines for special IPython syntax. - - Any exceptions generated in compilation are swallowed, but if an - exception was produced, the method returns True. - - Parameters - ---------- - lines : string - One or more lines of Python input. - - Returns - ------- - is_complete : boolean - True if the current input source (the result of the current input - plus prior inputs) forms a complete Python execution block. Note that - this value is also stored as a private attribute (_is_complete), so it - can be queried at any time. - """ - assert isinstance(lines, str) - # We must ensure all input is pure unicode - # ''.splitlines() --> [], but we need to push the empty line to transformers - lines_list = lines.splitlines() - if not lines_list: - lines_list = [''] - - # Store raw source before applying any transformations to it. Note - # that this must be done *after* the reset() call that would otherwise - # flush the buffer. - self._store(lines, self._buffer_raw, 'source_raw') - - transformed_lines_list = [] - for line in lines_list: - transformed = self._transform_line(line) - if transformed is not None: - transformed_lines_list.append(transformed) - - if transformed_lines_list: - transformed_lines = '\n'.join(transformed_lines_list) - return super(IPythonInputSplitter, self).push(transformed_lines) - else: - # Got nothing back from transformers - they must be waiting for - # more input. - return False - - def _transform_line(self, line): - """Push a line of input code through the various transformers. - - Returns any output from the transformers, or None if a transformer - is accumulating lines. - - Sets self.transformer_accumulating as a side effect. - """ - def _accumulating(dbg): - #print(dbg) - self.transformer_accumulating = True - return None - - for transformer in self.physical_line_transforms: - line = transformer.push(line) - if line is None: - return _accumulating(transformer) - - if not self.within_python_line: - line = self.assemble_logical_lines.push(line) - if line is None: - return _accumulating('acc logical line') - - for transformer in self.logical_line_transforms: - line = transformer.push(line) - if line is None: - return _accumulating(transformer) - - line = self.assemble_python_lines.push(line) - if line is None: - self.within_python_line = True - return _accumulating('acc python line') - else: - self.within_python_line = False - - for transformer in self.python_line_transforms: - line = transformer.push(line) - if line is None: - return _accumulating(transformer) - - #print("transformers clear") #debug - self.transformer_accumulating = False - return line - diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py deleted file mode 100644 index 1ec48da..0000000 --- a/IPython/core/tests/test_inputsplitter.py +++ /dev/null @@ -1,637 +0,0 @@ -# -*- coding: utf-8 -*- -"""Tests for the inputsplitter module.""" - - -# Copyright (c) IPython Development Team. -# Distributed under the terms of the Modified BSD License. - -import unittest -import pytest -import sys - -with pytest.warns(DeprecationWarning, match="inputsplitter"): - from IPython.core import inputsplitter as isp -from IPython.core.inputtransformer import InputTransformer -from IPython.core.tests.test_inputtransformer import syntax, syntax_ml -from IPython.testing import tools as tt - -#----------------------------------------------------------------------------- -# Semi-complete examples (also used as tests) -#----------------------------------------------------------------------------- - -# Note: at the bottom, there's a slightly more complete version of this that -# can be useful during development of code here. - -def mini_interactive_loop(input_func): - """Minimal example of the logic of an interactive interpreter loop. - - This serves as an example, and it is used by the test system with a fake - raw_input that simulates interactive input.""" - - from IPython.core.inputsplitter import InputSplitter - - isp = InputSplitter() - # In practice, this input loop would be wrapped in an outside loop to read - # input indefinitely, until some exit/quit command was issued. Here we - # only illustrate the basic inner loop. - while isp.push_accepts_more(): - indent = ' '*isp.get_indent_spaces() - prompt = '>>> ' + indent - line = indent + input_func(prompt) - isp.push(line) - - # Here we just return input so we can use it in a test suite, but a real - # interpreter would instead send it for execution somewhere. - src = isp.source_reset() - # print('Input source was:\n', src) # dbg - return src - -#----------------------------------------------------------------------------- -# Test utilities, just for local use -#----------------------------------------------------------------------------- - - -def pseudo_input(lines): - """Return a function that acts like raw_input but feeds the input list.""" - ilines = iter(lines) - def raw_in(prompt): - try: - return next(ilines) - except StopIteration: - return '' - return raw_in - -#----------------------------------------------------------------------------- -# Tests -#----------------------------------------------------------------------------- -def test_spaces(): - tests = [('', 0), - (' ', 1), - ('\n', 0), - (' \n', 1), - ('x', 0), - (' x', 1), - (' x',2), - (' x',4), - # Note: tabs are counted as a single whitespace! - ('\tx', 1), - ('\t x', 2), - ] - with pytest.warns(PendingDeprecationWarning): - tt.check_pairs(isp.num_ini_spaces, tests) - - -def test_remove_comments(): - tests = [('text', 'text'), - ('text # comment', 'text '), - ('text # comment\n', 'text \n'), - ('text # comment \n', 'text \n'), - ('line # c \nline\n','line \nline\n'), - ('line # c \nline#c2 \nline\nline #c\n\n', - 'line \nline\nline\nline \n\n'), - ] - tt.check_pairs(isp.remove_comments, tests) - - -def test_get_input_encoding(): - encoding = isp.get_input_encoding() - assert isinstance(encoding, str) - # simple-minded check that at least encoding a simple string works with the - # encoding we got. - assert "test".encode(encoding) == b"test" - - -class NoInputEncodingTestCase(unittest.TestCase): - def setUp(self): - self.old_stdin = sys.stdin - class X: pass - fake_stdin = X() - sys.stdin = fake_stdin - - def test(self): - # Verify that if sys.stdin has no 'encoding' attribute we do the right - # thing - enc = isp.get_input_encoding() - self.assertEqual(enc, 'ascii') - - def tearDown(self): - sys.stdin = self.old_stdin - - -class InputSplitterTestCase(unittest.TestCase): - def setUp(self): - self.isp = isp.InputSplitter() - - def test_reset(self): - isp = self.isp - isp.push('x=1') - isp.reset() - self.assertEqual(isp._buffer, []) - self.assertEqual(isp.get_indent_spaces(), 0) - self.assertEqual(isp.source, '') - self.assertEqual(isp.code, None) - self.assertEqual(isp._is_complete, False) - - def test_source(self): - self.isp._store('1') - self.isp._store('2') - self.assertEqual(self.isp.source, '1\n2\n') - self.assertEqual(len(self.isp._buffer)>0, True) - self.assertEqual(self.isp.source_reset(), '1\n2\n') - self.assertEqual(self.isp._buffer, []) - self.assertEqual(self.isp.source, '') - - def test_indent(self): - isp = self.isp # shorthand - isp.push('x=1') - self.assertEqual(isp.get_indent_spaces(), 0) - isp.push('if 1:\n x=1') - self.assertEqual(isp.get_indent_spaces(), 4) - isp.push('y=2\n') - self.assertEqual(isp.get_indent_spaces(), 0) - - def test_indent2(self): - isp = self.isp - isp.push('if 1:') - self.assertEqual(isp.get_indent_spaces(), 4) - isp.push(' x=1') - self.assertEqual(isp.get_indent_spaces(), 4) - # Blank lines shouldn't change the indent level - isp.push(' '*2) - self.assertEqual(isp.get_indent_spaces(), 4) - - def test_indent3(self): - isp = self.isp - # When a multiline statement contains parens or multiline strings, we - # shouldn't get confused. - isp.push("if 1:") - isp.push(" x = (1+\n 2)") - self.assertEqual(isp.get_indent_spaces(), 4) - - def test_indent4(self): - isp = self.isp - # whitespace after ':' should not screw up indent level - isp.push('if 1: \n x=1') - self.assertEqual(isp.get_indent_spaces(), 4) - isp.push('y=2\n') - self.assertEqual(isp.get_indent_spaces(), 0) - isp.push('if 1:\t\n x=1') - self.assertEqual(isp.get_indent_spaces(), 4) - isp.push('y=2\n') - self.assertEqual(isp.get_indent_spaces(), 0) - - def test_dedent_pass(self): - isp = self.isp # shorthand - # should NOT cause dedent - isp.push('if 1:\n passes = 5') - self.assertEqual(isp.get_indent_spaces(), 4) - isp.push('if 1:\n pass') - self.assertEqual(isp.get_indent_spaces(), 0) - isp.push('if 1:\n pass ') - self.assertEqual(isp.get_indent_spaces(), 0) - - def test_dedent_break(self): - isp = self.isp # shorthand - # should NOT cause dedent - isp.push('while 1:\n breaks = 5') - self.assertEqual(isp.get_indent_spaces(), 4) - isp.push('while 1:\n break') - self.assertEqual(isp.get_indent_spaces(), 0) - isp.push('while 1:\n break ') - self.assertEqual(isp.get_indent_spaces(), 0) - - def test_dedent_continue(self): - isp = self.isp # shorthand - # should NOT cause dedent - isp.push('while 1:\n continues = 5') - self.assertEqual(isp.get_indent_spaces(), 4) - isp.push('while 1:\n continue') - self.assertEqual(isp.get_indent_spaces(), 0) - isp.push('while 1:\n continue ') - self.assertEqual(isp.get_indent_spaces(), 0) - - def test_dedent_raise(self): - isp = self.isp # shorthand - # should NOT cause dedent - isp.push('if 1:\n raised = 4') - self.assertEqual(isp.get_indent_spaces(), 4) - isp.push('if 1:\n raise TypeError()') - self.assertEqual(isp.get_indent_spaces(), 0) - isp.push('if 1:\n raise') - self.assertEqual(isp.get_indent_spaces(), 0) - isp.push('if 1:\n raise ') - self.assertEqual(isp.get_indent_spaces(), 0) - - def test_dedent_return(self): - isp = self.isp # shorthand - # should NOT cause dedent - isp.push('if 1:\n returning = 4') - self.assertEqual(isp.get_indent_spaces(), 4) - isp.push('if 1:\n return 5 + 493') - self.assertEqual(isp.get_indent_spaces(), 0) - isp.push('if 1:\n return') - self.assertEqual(isp.get_indent_spaces(), 0) - isp.push('if 1:\n return ') - self.assertEqual(isp.get_indent_spaces(), 0) - isp.push('if 1:\n return(0)') - self.assertEqual(isp.get_indent_spaces(), 0) - - def test_push(self): - isp = self.isp - self.assertEqual(isp.push('x=1'), True) - - def test_push2(self): - isp = self.isp - self.assertEqual(isp.push('if 1:'), False) - for line in [' x=1', '# a comment', ' y=2']: - print(line) - self.assertEqual(isp.push(line), True) - - def test_push3(self): - isp = self.isp - isp.push('if True:') - isp.push(' a = 1') - self.assertEqual(isp.push('b = [1,'), False) - - def test_push_accepts_more(self): - isp = self.isp - isp.push('x=1') - self.assertEqual(isp.push_accepts_more(), False) - - def test_push_accepts_more2(self): - isp = self.isp - isp.push('if 1:') - self.assertEqual(isp.push_accepts_more(), True) - isp.push(' x=1') - self.assertEqual(isp.push_accepts_more(), True) - isp.push('') - self.assertEqual(isp.push_accepts_more(), False) - - def test_push_accepts_more3(self): - isp = self.isp - isp.push("x = (2+\n3)") - self.assertEqual(isp.push_accepts_more(), False) - - def test_push_accepts_more4(self): - isp = self.isp - # When a multiline statement contains parens or multiline strings, we - # shouldn't get confused. - # FIXME: we should be able to better handle de-dents in statements like - # multiline strings and multiline expressions (continued with \ or - # parens). Right now we aren't handling the indentation tracking quite - # correctly with this, though in practice it may not be too much of a - # problem. We'll need to see. - isp.push("if 1:") - isp.push(" x = (2+") - isp.push(" 3)") - self.assertEqual(isp.push_accepts_more(), True) - isp.push(" y = 3") - self.assertEqual(isp.push_accepts_more(), True) - isp.push('') - self.assertEqual(isp.push_accepts_more(), False) - - def test_push_accepts_more5(self): - isp = self.isp - isp.push('try:') - isp.push(' a = 5') - isp.push('except:') - isp.push(' raise') - # We want to be able to add an else: block at this point, so it should - # wait for a blank line. - self.assertEqual(isp.push_accepts_more(), True) - - def test_continuation(self): - isp = self.isp - isp.push("import os, \\") - self.assertEqual(isp.push_accepts_more(), True) - isp.push("sys") - self.assertEqual(isp.push_accepts_more(), False) - - def test_syntax_error(self): - isp = self.isp - # Syntax errors immediately produce a 'ready' block, so the invalid - # Python can be sent to the kernel for evaluation with possible ipython - # special-syntax conversion. - isp.push('run foo') - self.assertEqual(isp.push_accepts_more(), False) - - def test_unicode(self): - self.isp.push(u"Pérez") - self.isp.push(u'\xc3\xa9') - self.isp.push(u"u'\xc3\xa9'") - - def test_line_continuation(self): - """ Test issue #2108.""" - isp = self.isp - # A blank line after a line continuation should not accept more - isp.push("1 \\\n\n") - self.assertEqual(isp.push_accepts_more(), False) - # Whitespace after a \ is a SyntaxError. The only way to test that - # here is to test that push doesn't accept more (as with - # test_syntax_error() above). - isp.push(r"1 \ ") - self.assertEqual(isp.push_accepts_more(), False) - # Even if the line is continuable (c.f. the regular Python - # interpreter) - isp.push(r"(1 \ ") - self.assertEqual(isp.push_accepts_more(), False) - - def test_check_complete(self): - isp = self.isp - self.assertEqual(isp.check_complete("a = 1"), ('complete', None)) - self.assertEqual(isp.check_complete("for a in range(5):"), ('incomplete', 4)) - self.assertEqual(isp.check_complete("raise = 2"), ('invalid', None)) - self.assertEqual(isp.check_complete("a = [1,\n2,"), ('incomplete', 0)) - self.assertEqual(isp.check_complete("def a():\n x=1\n global x"), ('invalid', None)) - -class InteractiveLoopTestCase(unittest.TestCase): - """Tests for an interactive loop like a python shell. - """ - def check_ns(self, lines, ns): - """Validate that the given input lines produce the resulting namespace. - - Note: the input lines are given exactly as they would be typed in an - auto-indenting environment, as mini_interactive_loop above already does - auto-indenting and prepends spaces to the input. - """ - src = mini_interactive_loop(pseudo_input(lines)) - test_ns = {} - exec(src, test_ns) - # We can't check that the provided ns is identical to the test_ns, - # because Python fills test_ns with extra keys (copyright, etc). But - # we can check that the given dict is *contained* in test_ns - for k,v in ns.items(): - self.assertEqual(test_ns[k], v) - - def test_simple(self): - self.check_ns(['x=1'], dict(x=1)) - - def test_simple2(self): - self.check_ns(['if 1:', 'x=2'], dict(x=2)) - - def test_xy(self): - self.check_ns(['x=1; y=2'], dict(x=1, y=2)) - - def test_abc(self): - self.check_ns(['if 1:','a=1','b=2','c=3'], dict(a=1, b=2, c=3)) - - def test_multi(self): - self.check_ns(['x =(1+','1+','2)'], dict(x=4)) - - -class IPythonInputTestCase(InputSplitterTestCase): - """By just creating a new class whose .isp is a different instance, we - re-run the same test battery on the new input splitter. - - In addition, this runs the tests over the syntax and syntax_ml dicts that - were tested by individual functions, as part of the OO interface. - - It also makes some checks on the raw buffer storage. - """ - - def setUp(self): - self.isp = isp.IPythonInputSplitter() - - def test_syntax(self): - """Call all single-line syntax tests from the main object""" - isp = self.isp - for example in syntax.values(): - for raw, out_t in example: - if raw.startswith(' '): - continue - - isp.push(raw+'\n') - out_raw = isp.source_raw - out = isp.source_reset() - self.assertEqual(out.rstrip(), out_t, - tt.pair_fail_msg.format("inputsplitter",raw, out_t, out)) - self.assertEqual(out_raw.rstrip(), raw.rstrip()) - - def test_syntax_multiline(self): - isp = self.isp - for example in syntax_ml.values(): - for line_pairs in example: - out_t_parts = [] - raw_parts = [] - for lraw, out_t_part in line_pairs: - if out_t_part is not None: - out_t_parts.append(out_t_part) - - if lraw is not None: - isp.push(lraw) - raw_parts.append(lraw) - - out_raw = isp.source_raw - out = isp.source_reset() - out_t = '\n'.join(out_t_parts).rstrip() - raw = '\n'.join(raw_parts).rstrip() - self.assertEqual(out.rstrip(), out_t) - self.assertEqual(out_raw.rstrip(), raw) - - def test_syntax_multiline_cell(self): - isp = self.isp - for example in syntax_ml.values(): - - out_t_parts = [] - for line_pairs in example: - raw = '\n'.join(r for r, _ in line_pairs if r is not None) - out_t = '\n'.join(t for _,t in line_pairs if t is not None) - out = isp.transform_cell(raw) - # Match ignoring trailing whitespace - self.assertEqual(out.rstrip(), out_t.rstrip()) - - def test_cellmagic_preempt(self): - isp = self.isp - for raw, name, line, cell in [ - ("%%cellm a\nIn[1]:", u'cellm', u'a', u'In[1]:'), - ("%%cellm \nline\n>>> hi", u'cellm', u'', u'line\n>>> hi'), - (">>> %%cellm \nline\n>>> hi", u'cellm', u'', u'line\nhi'), - ("%%cellm \n>>> hi", u'cellm', u'', u'>>> hi'), - ("%%cellm \nline1\nline2", u'cellm', u'', u'line1\nline2'), - ("%%cellm \nline1\\\\\nline2", u'cellm', u'', u'line1\\\\\nline2'), - ]: - expected = "get_ipython().run_cell_magic(%r, %r, %r)" % ( - name, line, cell - ) - out = isp.transform_cell(raw) - self.assertEqual(out.rstrip(), expected.rstrip()) - - def test_multiline_passthrough(self): - isp = self.isp - class CommentTransformer(InputTransformer): - def __init__(self): - self._lines = [] - - def push(self, line): - self._lines.append(line + '#') - - def reset(self): - text = '\n'.join(self._lines) - self._lines = [] - return text - - isp.physical_line_transforms.insert(0, CommentTransformer()) - - for raw, expected in [ - ("a=5", "a=5#"), - ("%ls foo", "get_ipython().run_line_magic(%r, %r)" % (u'ls', u'foo#')), - ("!ls foo\n%ls bar", "get_ipython().system(%r)\nget_ipython().run_line_magic(%r, %r)" % ( - u'ls foo#', u'ls', u'bar#' - )), - ("1\n2\n3\n%ls foo\n4\n5", "1#\n2#\n3#\nget_ipython().run_line_magic(%r, %r)\n4#\n5#" % (u'ls', u'foo#')), - ]: - out = isp.transform_cell(raw) - self.assertEqual(out.rstrip(), expected.rstrip()) - -#----------------------------------------------------------------------------- -# Main - use as a script, mostly for developer experiments -#----------------------------------------------------------------------------- - -if __name__ == '__main__': - # A simple demo for interactive experimentation. This code will not get - # picked up by any test suite. - from IPython.core.inputsplitter import IPythonInputSplitter - - # configure here the syntax to use, prompt and whether to autoindent - #isp, start_prompt = InputSplitter(), '>>> ' - isp, start_prompt = IPythonInputSplitter(), 'In> ' - - autoindent = True - #autoindent = False - - try: - while True: - prompt = start_prompt - while isp.push_accepts_more(): - indent = ' '*isp.get_indent_spaces() - if autoindent: - line = indent + input(prompt+indent) - else: - line = input(prompt) - isp.push(line) - prompt = '... ' - - # Here we just return input so we can use it in a test suite, but a - # real interpreter would instead send it for execution somewhere. - #src = isp.source; raise EOFError # dbg - raw = isp.source_raw - src = isp.source_reset() - print('Input source was:\n', src) - print('Raw source was:\n', raw) - except EOFError: - print('Bye') - -# Tests for cell magics support - -def test_last_blank(): - assert isp.last_blank("") is False - assert isp.last_blank("abc") is False - assert isp.last_blank("abc\n") is False - assert isp.last_blank("abc\na") is False - - assert isp.last_blank("\n") is True - assert isp.last_blank("\n ") is True - assert isp.last_blank("abc\n ") is True - assert isp.last_blank("abc\n\n") is True - assert isp.last_blank("abc\nd\n\n") is True - assert isp.last_blank("abc\nd\ne\n\n") is True - assert isp.last_blank("abc \n \n \n\n") is True - - -def test_last_two_blanks(): - assert isp.last_two_blanks("") is False - assert isp.last_two_blanks("abc") is False - assert isp.last_two_blanks("abc\n") is False - assert isp.last_two_blanks("abc\n\na") is False - assert isp.last_two_blanks("abc\n \n") is False - assert isp.last_two_blanks("abc\n\n") is False - - assert isp.last_two_blanks("\n\n") is True - assert isp.last_two_blanks("\n\n ") is True - assert isp.last_two_blanks("\n \n") is True - assert isp.last_two_blanks("abc\n\n ") is True - assert isp.last_two_blanks("abc\n\n\n") is True - assert isp.last_two_blanks("abc\n\n \n") is True - assert isp.last_two_blanks("abc\n\n \n ") is True - assert isp.last_two_blanks("abc\n\n \n \n") is True - assert isp.last_two_blanks("abc\nd\n\n\n") is True - assert isp.last_two_blanks("abc\nd\ne\nf\n\n\n") is True - - -class CellMagicsCommon(object): - - def test_whole_cell(self): - src = "%%cellm line\nbody\n" - out = self.sp.transform_cell(src) - ref = "get_ipython().run_cell_magic('cellm', 'line', 'body')\n" - assert out == ref - - def test_cellmagic_help(self): - self.sp.push('%%cellm?') - assert self.sp.push_accepts_more() is False - - def tearDown(self): - self.sp.reset() - - -class CellModeCellMagics(CellMagicsCommon, unittest.TestCase): - sp = isp.IPythonInputSplitter(line_input_checker=False) - - def test_incremental(self): - sp = self.sp - sp.push("%%cellm firstline\n") - assert sp.push_accepts_more() is True # 1 - sp.push("line2\n") - assert sp.push_accepts_more() is True # 2 - sp.push("\n") - # This should accept a blank line and carry on until the cell is reset - assert sp.push_accepts_more() is True # 3 - - def test_no_strip_coding(self): - src = '\n'.join([ - '%%writefile foo.py', - '# coding: utf-8', - 'print(u"üñîçø∂é")', - ]) - out = self.sp.transform_cell(src) - assert "# coding: utf-8" in out - - -class LineModeCellMagics(CellMagicsCommon, unittest.TestCase): - sp = isp.IPythonInputSplitter(line_input_checker=True) - - def test_incremental(self): - sp = self.sp - sp.push("%%cellm line2\n") - assert sp.push_accepts_more() is True # 1 - sp.push("\n") - # In this case, a blank line should end the cell magic - assert sp.push_accepts_more() is False # 2 - - -indentation_samples = [ - ('a = 1', 0), - ('for a in b:', 4), - ('def f():', 4), - ('def f(): #comment', 4), - ('a = ":#not a comment"', 0), - ('def f():\n a = 1', 4), - ('def f():\n return 1', 0), - ('for a in b:\n' - ' if a < 0:' - ' continue', 3), - ('a = {', 4), - ('a = {\n' - ' 1,', 5), - ('b = """123', 0), - ('', 0), - ('def f():\n pass', 0), - ('class Bar:\n def f():\n pass', 4), - ('class Bar:\n def f():\n raise', 4), -] - -def test_find_next_indent(): - for code, exp in indentation_samples: - res = isp.find_next_indent(code) - msg = "{!r} != {!r} (expected)\n Code: {!r}".format(res, exp, code) - assert res == exp, msg