diff --git a/IPython/core/compilerop.py b/IPython/core/compilerop.py index e92be98..585eba2 100644 --- a/IPython/core/compilerop.py +++ b/IPython/core/compilerop.py @@ -7,6 +7,7 @@ Authors ------- * Robert Kern * Fernando Perez +* Thomas Kluyver """ # Note: though it might be more natural to name this module 'compiler', that @@ -51,12 +52,12 @@ def code_name(code, number=0): # Classes and functions #----------------------------------------------------------------------------- -class CachingCompiler(object): +class CachingCompiler(codeop.Compile): """A compiler that caches code compiled from interactive statements. """ def __init__(self): - self._compiler = codeop.CommandCompiler() + codeop.Compile.__init__(self) # This is ugly, but it must be done this way to allow multiple # simultaneous ipython instances to coexist. Since Python itself @@ -81,35 +82,30 @@ class CachingCompiler(object): def compiler_flags(self): """Flags currently active in the compilation process. """ - return self._compiler.compiler.flags + return self.flags + + def cache(self, code, number=0): + """Make a name for a block of code, and cache the code. - def __call__(self, code, symbol, number=0): - """Compile some code while caching its contents such that the inspect - module can find it later. - Parameters ---------- code : str - Source code to be compiled, one or more lines. - - symbol : str - One of 'single', 'exec' or 'eval' (see the builtin ``compile`` - documentation for further details on these fields). - - number : int, optional - An integer argument identifying the code, useful for informational - purposes in tracebacks (typically it will be the IPython prompt - number). + The Python source code to cache. + number : int + A number which forms part of the code's name. Used for the execution + counter. + + Returns + ------- + The name of the cached code (as a string). Pass this as the filename + argument to compilation, so that tracebacks are correctly hooked up. """ name = code_name(code, number) - code_obj = self._compiler(code, name, symbol) entry = (len(code), time.time(), [line+'\n' for line in code.splitlines()], name) - # Cache the info both in the linecache (a global cache used internally - # by most of Python's inspect/traceback machinery), and in our cache linecache.cache[name] = entry linecache._ipython_cache[name] = entry - return code_obj + return name def check_cache(self, *args): """Call linecache.checkcache() safely protecting our cached values. diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py index 3b20545..83b35de 100644 --- a/IPython/core/inputsplitter.py +++ b/IPython/core/inputsplitter.py @@ -166,78 +166,6 @@ def get_input_encoding(): # Classes and functions for normal Python syntax handling #----------------------------------------------------------------------------- -# HACK! This implementation, written by Robert K a while ago using the -# compiler module, is more robust than the other one below, but it expects its -# input to be pure python (no ipython syntax). For now we're using it as a -# second-pass splitter after the first pass transforms the input to pure -# python. - -def split_blocks(python): - """ Split multiple lines of code into discrete commands that can be - executed singly. - - Parameters - ---------- - python : str - Pure, exec'able Python code. - - Returns - ------- - commands : list of str - Separate commands that can be exec'ed independently. - """ - # compiler.parse treats trailing spaces after a newline as a - # SyntaxError. This is different than codeop.CommandCompiler, which - # will compile the trailng spaces just fine. We simply strip any - # trailing whitespace off. Passing a string with trailing whitespace - # to exec will fail however. There seems to be some inconsistency in - # how trailing whitespace is handled, but this seems to work. - python_ori = python # save original in case we bail on error - python = python.strip() - - # The compiler module will parse the code into an abstract syntax tree. - # This has a bug with str("a\nb"), but not str("""a\nb""")!!! - try: - code_ast = ast.parse(python) - except: - return [python_ori] - - # Uncomment to help debug the ast tree - # for n in code_ast.body: - # print n.lineno,'->',n - - # Each separate command is available by iterating over ast.node. The - # lineno attribute is the line number (1-indexed) beginning the commands - # suite. - # lines ending with ";" yield a Discard Node that doesn't have a lineno - # attribute. These nodes can and should be discarded. But there are - # other situations that cause Discard nodes that shouldn't be discarded. - # We might eventually discover other cases where lineno is None and have - # to put in a more sophisticated test. - linenos = [x.lineno-1 for x in code_ast.body if x.lineno is not None] - - # When we finally get the slices, we will need to slice all the way to - # the end even though we don't have a line number for it. Fortunately, - # None does the job nicely. - linenos.append(None) - - # Same problem at the other end: sometimes the ast tree has its - # first complete statement not starting on line 0. In this case - # we might miss part of it. This fixes ticket 266993. Thanks Gael! - linenos[0] = 0 - - lines = python.splitlines() - - # Create a list of atomic commands. - cmds = [] - for i, j in zip(linenos[:-1], linenos[1:]): - cmd = lines[i:j] - if cmd: - cmds.append('\n'.join(cmd)+'\n') - - return cmds - - class InputSplitter(object): """An object that can split Python source input in executable blocks. @@ -445,96 +373,18 @@ class InputSplitter(object): if not self._full_dedent: return False else: - nblocks = len(split_blocks(''.join(self._buffer))) - if nblocks==1: + try: + code_ast = ast.parse(u''.join(self._buffer)) + except Exception: return False + else: + if len(code_ast.body) == 1: + return False # When input is complete, then termination is marked by an extra blank # line at the end. last_line = self.source.splitlines()[-1] return bool(last_line and not last_line.isspace()) - - def split_blocks(self, lines): - """Split a multiline string into multiple input blocks. - - Note: this method starts by performing a full reset(). - - Parameters - ---------- - lines : str - A possibly multiline string. - - Returns - ------- - blocks : list - A list of strings, each possibly multiline. Each string corresponds - to a single block that can be compiled in 'single' mode (unless it - has a syntax error).""" - - # This code is fairly delicate. If you make any changes here, make - # absolutely sure that you do run the full test suite and ALL tests - # pass. - - self.reset() - blocks = [] - - # Reversed copy so we can use pop() efficiently and consume the input - # as a stack - lines = lines.splitlines()[::-1] - # Outer loop over all input - while lines: - #print 'Current lines:', lines # dbg - # Inner loop to build each block - while True: - # Safety exit from inner loop - if not lines: - break - # Grab next line but don't push it yet - next_line = lines.pop() - # Blank/empty lines are pushed as-is - if not next_line or next_line.isspace(): - self.push(next_line) - continue - - # Check indentation changes caused by the *next* line - indent_spaces, _full_dedent = self._find_indent(next_line) - - # If the next line causes a dedent, it can be for two differnt - # reasons: either an explicit de-dent by the user or a - # return/raise/pass statement. These MUST be handled - # separately: - # - # 1. the first case is only detected when the actual explicit - # dedent happens, and that would be the *first* line of a *new* - # block. Thus, we must put the line back into the input buffer - # so that it starts a new block on the next pass. - # - # 2. the second case is detected in the line before the actual - # dedent happens, so , we consume the line and we can break out - # to start a new block. - - # Case 1, explicit dedent causes a break. - # Note: check that we weren't on the very last line, else we'll - # enter an infinite loop adding/removing the last line. - if _full_dedent and lines and not next_line.startswith(' '): - lines.append(next_line) - break - - # Otherwise any line is pushed - self.push(next_line) - - # Case 2, full dedent with full block ready: - if _full_dedent or \ - self.indent_spaces==0 and not self.push_accepts_more(): - break - # Form the new block with the current source input - blocks.append(self.source_reset()) - - #return blocks - # HACK!!! Now that our input is in blocks but guaranteed to be pure - # python syntax, feed it back a second time through the AST-based - # splitter, which is more accurate than ours. - return split_blocks(''.join(blocks)) #------------------------------------------------------------------------ # Private interface diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index d3c2c41..cd9595e 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -20,6 +20,7 @@ from __future__ import absolute_import import __builtin__ import __future__ import abc +import ast import atexit import codeop import inspect @@ -2098,115 +2099,95 @@ class InteractiveShell(Configurable, Magic): except: self.showtraceback() warn('Unknown failure executing file: <%s>' % fname) - + def run_cell(self, cell, store_history=True): - """Run the contents of an entire multiline 'cell' of code, and store it - in the history. - - The cell is split into separate blocks which can be executed - individually. Then, based on how many blocks there are, they are - executed as follows: - - - A single block: 'single' mode. If it is also a single line, dynamic - transformations, including automagic and macros, will be applied. - - If there's more than one block, it depends: - - - if the last one is no more than two lines long, run all but the last - in 'exec' mode and the very last one in 'single' mode. This makes it - easy to type simple expressions at the end to see computed values. - - otherwise (last one is also multiline), run all in 'exec' mode - - When code is executed in 'single' mode, :func:`sys.displayhook` fires, - results are displayed and output prompts are computed. In 'exec' mode, - no results are displayed unless :func:`print` is called explicitly; - this mode is more akin to running a script. - + """Run a complete IPython cell. + Parameters ---------- cell : str - A single or multiline string. + The code (including IPython code such as %magic functions) to run. + store_history : bool + If True, the raw and translated cell will be stored in IPython's + history. For user code calling back into IPython's machinery, this + should be set to False. """ - # Store the untransformed code raw_cell = cell - - # Code transformation and execution must take place with our - # modifications to builtins. with self.builtin_trap: + cell = self.prefilter_manager.prefilter_lines(cell) - # We need to break up the input into executable blocks that can - # be runin 'single' mode, to provide comfortable user behavior. - blocks = self.input_splitter.split_blocks(cell) - - if not blocks: # Blank cell - return - - # We only do dynamic transforms on a single line. But a macro - # can be expanded to several lines, so we need to split it - # into input blocks again. - if len(cell.splitlines()) <= 1: - cell = self.prefilter_manager.prefilter_line(blocks[0]) - blocks = self.input_splitter.split_blocks(cell) - - # Store the 'ipython' version of the cell as well, since - # that's what needs to go into the translated history and get - # executed (the original cell may contain non-python syntax). - cell = ''.join(blocks) - # Store raw and processed history if store_history: self.history_manager.store_inputs(self.execution_count, cell, raw_cell) self.logger.log(cell, raw_cell) - - # All user code execution should take place with our - # modified displayhook. + + cell_name = self.compile.cache(cell, self.execution_count) + with self.display_trap: - # Single-block input should behave like an interactive prompt - if len(blocks) == 1: - out = self.run_source(blocks[0]) - # Write output to the database. Does nothing unless - # history output logging is enabled. - if store_history: - self.history_manager.store_output(self.execution_count) - # Since we return here, we need to update the - # execution count - self.execution_count += 1 - return out - - # In multi-block input, if the last block is a simple (one-two - # lines) expression, run it in single mode so it produces output. - # Otherwise just run it all in 'exec' mode. This seems like a - # reasonable usability design. - last = blocks[-1] - last_nlines = len(last.splitlines()) + try: + code_ast = ast.parse(cell, filename=cell_name) + except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError): + # Case 1 + self.showsyntaxerror() + self.execution_count += 1 + return None + + interactivity = 'last' # Last node to be run interactive + if len(cell.splitlines()) == 1: + interactivity = 'all' # Single line; run fully interactive + + self.run_ast_nodes(code_ast.body, cell_name, interactivity) - if last_nlines < 2: - # Here we consider the cell split between 'body' and 'last', - # store all history and execute 'body', and if successful, then - # proceed to execute 'last'. - - # Get the main body to run as a cell - ipy_body = ''.join(blocks[:-1]) - retcode = self.run_source(ipy_body, symbol='exec', - post_execute=False) - if retcode==0: - # Last expression compiled as 'single' so it - # produces output - self.run_source(last) - else: - # Run the whole cell as one entity, storing both raw and - # processed input in history - self.run_source(cell, symbol='exec') - - # Write output to the database. Does nothing unless - # history output logging is enabled. if store_history: + # Write output to the database. Does nothing unless + # history output logging is enabled. self.history_manager.store_output(self.execution_count) # Each cell is a *single* input, regardless of how many lines it has self.execution_count += 1 - + + def run_ast_nodes(self, nodelist, cell_name, interactivity='last'): + """Run a sequence of AST nodes. The execution mode depends on the + interactivity parameter. + + Parameters + ---------- + nodelist : list + A sequence of AST nodes to run. + cell_name : str + Will be passed to the compiler as the filename of the cell. Typically + the value returned by ip.compile.cache(cell). + interactivity : str + 'all', 'last' or 'none', specifying which nodes should be run + interactively (displaying output from expressions). Other values for + this parameter will raise a ValueError. + """ + if not nodelist: + return + + if interactivity == 'none': + to_run_exec, to_run_interactive = nodelist, [] + elif interactivity == 'last': + to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:] + elif interactivity == 'all': + to_run_exec, to_run_interactive = [], nodelist + else: + raise ValueError("Interactivity was %r" % interactivity) + + exec_count = self.execution_count + if to_run_exec: + mod = ast.Module(to_run_exec) + self.code_to_run = code = self.compile(mod, cell_name, "exec") + if self.run_code(code) == 1: + return + + if to_run_interactive: + mod = ast.Interactive(to_run_interactive) + self.code_to_run = code = self.compile(mod, cell_name, "single") + return self.run_code(code) + + # PENDING REMOVAL: this method is slated for deletion, once our new # input logic has been 100% moved to frontends and is stable. def runlines(self, lines, clean=False): @@ -2296,7 +2277,8 @@ class InteractiveShell(Configurable, Magic): print 'encoding', self.stdin_encoding # dbg try: - code = self.compile(usource, symbol, self.execution_count) + code_name = self.compile.cache(usource, self.execution_count) + code = self.compile(usource, code_name, symbol) except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError): # Case 1 self.showsyntaxerror(filename) diff --git a/IPython/core/tests/test_compilerop.py b/IPython/core/tests/test_compilerop.py index 2fac2fc..e9989db 100644 --- a/IPython/core/tests/test_compilerop.py +++ b/IPython/core/tests/test_compilerop.py @@ -40,12 +40,12 @@ def test_code_name2(): nt.assert_true(name.startswith(' ncache) def setUp(): @@ -53,10 +53,10 @@ def setUp(): # as GTK, can change the default encoding, which can hide bugs.) nt.assert_equal(sys.getdefaultencoding(), "ascii") -def test_compiler_unicode(): +def test_cache_unicode(): cp = compilerop.CachingCompiler() ncache = len(linecache.cache) - cp(u"t = 'žćčšđ'", "single") + cp.cache(u"t = 'žćčšđ'") nt.assert_true(len(linecache.cache) > ncache) def test_compiler_check_cache(): @@ -64,7 +64,7 @@ def test_compiler_check_cache(): """ # Rather simple-minded tests that just exercise the API cp = compilerop.CachingCompiler() - cp('x=1', 'single', 99) + cp.cache('x=1', 99) # Ensure now that after clearing the cache, our entries survive cp.check_cache() for k in linecache.cache: diff --git a/IPython/core/tests/test_handlers.py b/IPython/core/tests/test_handlers.py index c8a9c94..483a8d7 100644 --- a/IPython/core/tests/test_handlers.py +++ b/IPython/core/tests/test_handlers.py @@ -43,9 +43,7 @@ def run(tests): for pre, post in tests: global num_tests num_tests += 1 - ip.runlines(pre) - ip.runlines('_i') # Not sure why I need this... - actual = ip.user_ns['_i'] + actual = ip.prefilter_manager.prefilter_lines(pre) if actual != None: actual = actual.rstrip('\n') if actual != post: diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py index 880a136..bfb5267 100644 --- a/IPython/core/tests/test_inputsplitter.py +++ b/IPython/core/tests/test_inputsplitter.py @@ -286,92 +286,6 @@ class InputSplitterTestCase(unittest.TestCase): isp.push('run foo') self.assertFalse(isp.push_accepts_more()) - def check_split(self, block_lines, compile=True): - blocks = assemble(block_lines) - lines = ''.join(blocks) - oblock = self.isp.split_blocks(lines) - self.assertEqual(oblock, blocks) - if compile: - for block in blocks: - self.isp._compile(block) - - def test_split(self): - # All blocks of input we want to test in a list. The format for each - # block is a list of lists, with each inner lists consisting of all the - # lines (as single-lines) that should make up a sub-block. - - # Note: do NOT put here sub-blocks that don't compile, as the - # check_split() routine makes a final verification pass to check that - # each sub_block, as returned by split_blocks(), does compile - # correctly. - all_blocks = [ [['x=1']], - - [['x=1'], - ['y=2']], - - [['x=1', - '# a comment'], - ['y=11']], - - [['if 1:', - ' x=1'], - ['y=3']], - - [['def f(x):', - ' return x'], - ['x=1']], - - [['def f(x):', - ' x+=1', - ' ', - ' return x'], - ['x=1']], - - [['def f(x):', - ' if x>0:', - ' y=1', - ' # a comment', - ' else:', - ' y=4', - ' ', - ' return y'], - ['x=1'], - ['if 1:', - ' y=11'] ], - - [['for i in range(10):' - ' x=i**2']], - - [['for i in range(10):' - ' x=i**2'], - ['z = 1']], - - [['"asdf"']], - - [['"asdf"'], - ['10'], - ], - - [['"""foo', - 'bar"""']], - ] - for block_lines in all_blocks: - self.check_split(block_lines) - - def test_split_syntax_errors(self): - # Block splitting with invalid syntax - all_blocks = [ [['a syntax error']], - - [['x=1', - 'another syntax error']], - - [['for i in range(10):' - ' yet another error']], - - ] - for block_lines in all_blocks: - self.check_split(block_lines, compile=False) - def test_unicode(self): self.isp.push(u"Pérez") self.isp.push(u'\xc3\xa9') diff --git a/IPython/core/tests/test_interactiveshell.py b/IPython/core/tests/test_interactiveshell.py index 2d533e4..8db596c 100644 --- a/IPython/core/tests/test_interactiveshell.py +++ b/IPython/core/tests/test_interactiveshell.py @@ -56,7 +56,6 @@ class InteractiveShellTestCase(unittest.TestCase): self.assertEquals(ip.user_ns['x'], 2) self.assertEquals(ip.user_ns['y'], 3) - @dec.skip_known_failure def test_multiline_string_cells(self): "Code sprinkled with multiline strings should execute (GH-306)" ip = get_ipython() diff --git a/IPython/lib/tests/test_irunner.py b/IPython/lib/tests/test_irunner.py index dd8cfd3..1ca1f7a 100755 --- a/IPython/lib/tests/test_irunner.py +++ b/IPython/lib/tests/test_irunner.py @@ -97,10 +97,10 @@ In [7]: autocall 0 Automatic calling is: OFF In [8]: cos pi - File "", line 1 + File "", line 1 cos pi ^ -SyntaxError: invalid syntax +SyntaxError: unexpected EOF while parsing In [9]: cos(pi)