diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 8a78e94..de941b1 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -339,6 +339,15 @@ class InteractiveShell(SingletonConfigurable): ()) @property + def input_transformers_cleanup(self): + return self.input_transformer_manager.cleanup_transforms + + input_transformers_post = List([], + help="A list of string input transformers, to be applied after IPython's " + "own input transformations." + ) + + @property def input_splitter(self): """Make this available for compatibility @@ -2717,21 +2726,10 @@ class InteractiveShell(SingletonConfigurable): preprocessing_exc_tuple = None try: # Static input transformations - cell = self.input_transformer_manager.transform_cell(raw_cell) - except SyntaxError: + cell = self.transform_cell(raw_cell) + except Exception: preprocessing_exc_tuple = sys.exc_info() cell = raw_cell # cell has to exist so it can be stored/logged - else: - if len(cell.splitlines()) == 1: - # Dynamic transformations - only applied for single line commands - with self.builtin_trap: - try: - # use prefilter_lines to handle trailing newlines - # restore trailing newline for ast.parse - cell = self.prefilter_manager.prefilter_lines(cell) + '\n' - except Exception: - # don't allow prefilter errors to crash IPython - preprocessing_exc_tuple = sys.exc_info() # Store raw and processed history if store_history: @@ -2802,6 +2800,24 @@ class InteractiveShell(SingletonConfigurable): self.execution_count += 1 return result + + def transform_cell(self, raw_cell): + # Static input transformations + cell = self.input_transformer_manager.transform_cell(raw_cell) + + if len(cell.splitlines()) == 1: + # Dynamic transformations - only applied for single line commands + with self.builtin_trap: + # use prefilter_lines to handle trailing newlines + # restore trailing newline for ast.parse + cell = self.prefilter_manager.prefilter_lines(cell) + '\n' + + lines = cell.splitlines(keepends=True) + for transform in self.input_transformers_post: + lines = transform(lines) + cell = ''.join(lines) + + return cell def transform_ast(self, node): """Apply the AST transformations from self.ast_transformers diff --git a/docs/source/config/inputtransforms.rst b/docs/source/config/inputtransforms.rst index 65ddc38..9e41707 100644 --- a/docs/source/config/inputtransforms.rst +++ b/docs/source/config/inputtransforms.rst @@ -24,34 +24,28 @@ end of this stage, it must be valid Python syntax. redesigned. Any third party code extending input transformation will need to be rewritten. The new API is, hopefully, simpler. -String based transformations are managed by -:class:`IPython.core.inputtransformer2.TransformerManager`, which is attached to -the :class:`~IPython.core.interactiveshell.InteractiveShell` instance as -``input_transformer_manager``. This passes the -data through a series of individual transformers. There are two kinds of -transformers stored in three groups: - -* ``cleanup_transforms`` and ``line_transforms`` are lists of functions. Each - function is called with a list of input lines (which include trailing - newlines), and they return a list in the same format. ``cleanup_transforms`` - are run first; they strip prompts and leading indentation from input. - The only default transform in ``line_transforms`` processes cell magics. -* ``token_transformers`` is a list of :class:`IPython.core.inputtransformer2.TokenTransformBase` - subclasses (not instances). They recognise special syntax like - ``%line magics`` and ``help?``, and transform them to Python syntax. The - interface for these is more complex; see below. +String based transformations are functions which accept a list of strings: +each string is a single line of the input cell, including its line ending. +The transformation function should return output in the same structure. + +These transformations are in two groups, accessible as attributes of +the :class:`~IPython.core.interactiveshell.InteractiveShell` instance. +Each group is a list of transformation functions. + +* ``input_transformers_cleanup`` run first on input, to do things like stripping + prompts and leading indents from copied code. It may not be possible at this + stage to parse the input as valid Python code. +* Then IPython runs its own transformations to handle its special syntax, like + ``%magics`` and ``!system`` commands. This part does not expose extension + points. +* ``input_transformers_post`` run as the last step, to do things like converting + float literals into decimal objects. These may attempt to parse the input as + Python code. These transformers may raise :exc:`SyntaxError` if the input code is invalid, but in most cases it is clearer to pass unrecognised code through unmodified and let Python's own parser decide whether it is valid. -.. versionchanged:: 2.0 - - Added the option to raise :exc:`SyntaxError`. - -Line based transformations --------------------------- - For example, imagine we want to obfuscate our code by reversing each line, so we'd write ``)5(f =+ a`` instead of ``a += f(5)``. Here's how we could swap it back the right way before IPython tries to run it:: @@ -66,86 +60,7 @@ back the right way before IPython tries to run it:: To start using this:: ip = get_ipython() - ip.input_transformer_manager.line_transforms.append(reverse_line_chars) - -Token based transformations ---------------------------- - -These recognise special syntax like ``%magics`` and ``help?``, and transform it -into valid Python code. Using tokens makes it easy to avoid transforming similar -patterns inside comments or strings. - -The API for a token-based transformation looks like this:: - -.. class:: MyTokenTransformer - - .. classmethod:: find(tokens_by_line) - - Takes a list of lists of :class:`tokenize.TokenInfo` objects. Each sublist - is the tokens from one Python line, which may span several physical lines, - because of line continuations, multiline strings or expressions. If it - finds a pattern to transform, it returns an instance of the class. - Otherwise, it returns None. - - .. attribute:: start_lineno - start_col - priority - - These attributes are used to select which transformation to run first. - ``start_lineno`` is 0-indexed (whereas the locations on - :class:`~tokenize.TokenInfo` use 1-indexed line numbers). If there are - multiple matches in the same location, the one with the smaller - ``priority`` number is used. - - .. method:: transform(lines) - - This should transform the individual recognised pattern that was - previously found. As with line-based transforms, it takes a list of - lines as strings, and returns a similar list. - -Because each transformation may affect the parsing of the code after it, -``TransformerManager`` takes a careful approach. It calls ``find()`` on all -available transformers. If any find a match, the transformation which matched -closest to the start is run. Then it tokenises the transformed code again, -and starts the process again. This continues until none of the transformers -return a match. So it's important that the transformation removes the pattern -which ``find()`` recognises, otherwise it will enter an infinite loop. - -For example, here's a transformer which will recognise ``¬`` as a prefix for a -new kind of special command:: - - import tokenize - from IPython.core.inputtransformer2 import TokenTransformBase - - class MySpecialCommand(TokenTransformBase): - @classmethod - def find(cls, tokens_by_line): - """Find the first escaped command (¬foo) in the cell. - """ - for line in tokens_by_line: - ix = 0 - # Find the first token that's not INDENT/DEDENT - while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}: - ix += 1 - if line[ix].string == '¬': - return cls(line[ix].start) - - def transform(self, lines): - indent = lines[self.start_line][:self.start_col] - content = lines[self.start_line][self.start_col+1:] - - lines_before = lines[:self.start_line] - call = "specialcommand(%r)" % content - new_line = indent + call + '\n' - lines_after = lines[self.start_line + 1:] - - return lines_before + [new_line] + lines_after - -And here's how you'd use it:: - - ip = get_ipython() - ip.input_transformer_manager.token_transformers.append(MySpecialCommand) - + ip.input_transformers_cleanup.append(reverse_line_chars) AST transformations ===================