diff --git a/IPython/core/compilerop.py b/IPython/core/compilerop.py
index e92be98..585eba2 100644
--- a/IPython/core/compilerop.py
+++ b/IPython/core/compilerop.py
@@ -7,6 +7,7 @@ Authors
-------
* Robert Kern
* Fernando Perez
+* Thomas Kluyver
"""
# Note: though it might be more natural to name this module 'compiler', that
@@ -51,12 +52,12 @@ def code_name(code, number=0):
# Classes and functions
#-----------------------------------------------------------------------------
-class CachingCompiler(object):
+class CachingCompiler(codeop.Compile):
"""A compiler that caches code compiled from interactive statements.
"""
def __init__(self):
- self._compiler = codeop.CommandCompiler()
+ codeop.Compile.__init__(self)
# This is ugly, but it must be done this way to allow multiple
# simultaneous ipython instances to coexist. Since Python itself
@@ -81,35 +82,30 @@ class CachingCompiler(object):
def compiler_flags(self):
"""Flags currently active in the compilation process.
"""
- return self._compiler.compiler.flags
+ return self.flags
+
+ def cache(self, code, number=0):
+ """Make a name for a block of code, and cache the code.
- def __call__(self, code, symbol, number=0):
- """Compile some code while caching its contents such that the inspect
- module can find it later.
-
Parameters
----------
code : str
- Source code to be compiled, one or more lines.
-
- symbol : str
- One of 'single', 'exec' or 'eval' (see the builtin ``compile``
- documentation for further details on these fields).
-
- number : int, optional
- An integer argument identifying the code, useful for informational
- purposes in tracebacks (typically it will be the IPython prompt
- number).
+ The Python source code to cache.
+ number : int
+ A number which forms part of the code's name. Used for the execution
+ counter.
+
+ Returns
+ -------
+ The name of the cached code (as a string). Pass this as the filename
+ argument to compilation, so that tracebacks are correctly hooked up.
"""
name = code_name(code, number)
- code_obj = self._compiler(code, name, symbol)
entry = (len(code), time.time(),
[line+'\n' for line in code.splitlines()], name)
- # Cache the info both in the linecache (a global cache used internally
- # by most of Python's inspect/traceback machinery), and in our cache
linecache.cache[name] = entry
linecache._ipython_cache[name] = entry
- return code_obj
+ return name
def check_cache(self, *args):
"""Call linecache.checkcache() safely protecting our cached values.
diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py
index 3b20545..83b35de 100644
--- a/IPython/core/inputsplitter.py
+++ b/IPython/core/inputsplitter.py
@@ -166,78 +166,6 @@ def get_input_encoding():
# Classes and functions for normal Python syntax handling
#-----------------------------------------------------------------------------
-# HACK! This implementation, written by Robert K a while ago using the
-# compiler module, is more robust than the other one below, but it expects its
-# input to be pure python (no ipython syntax). For now we're using it as a
-# second-pass splitter after the first pass transforms the input to pure
-# python.
-
-def split_blocks(python):
- """ Split multiple lines of code into discrete commands that can be
- executed singly.
-
- Parameters
- ----------
- python : str
- Pure, exec'able Python code.
-
- Returns
- -------
- commands : list of str
- Separate commands that can be exec'ed independently.
- """
- # compiler.parse treats trailing spaces after a newline as a
- # SyntaxError. This is different than codeop.CommandCompiler, which
- # will compile the trailng spaces just fine. We simply strip any
- # trailing whitespace off. Passing a string with trailing whitespace
- # to exec will fail however. There seems to be some inconsistency in
- # how trailing whitespace is handled, but this seems to work.
- python_ori = python # save original in case we bail on error
- python = python.strip()
-
- # The compiler module will parse the code into an abstract syntax tree.
- # This has a bug with str("a\nb"), but not str("""a\nb""")!!!
- try:
- code_ast = ast.parse(python)
- except:
- return [python_ori]
-
- # Uncomment to help debug the ast tree
- # for n in code_ast.body:
- # print n.lineno,'->',n
-
- # Each separate command is available by iterating over ast.node. The
- # lineno attribute is the line number (1-indexed) beginning the commands
- # suite.
- # lines ending with ";" yield a Discard Node that doesn't have a lineno
- # attribute. These nodes can and should be discarded. But there are
- # other situations that cause Discard nodes that shouldn't be discarded.
- # We might eventually discover other cases where lineno is None and have
- # to put in a more sophisticated test.
- linenos = [x.lineno-1 for x in code_ast.body if x.lineno is not None]
-
- # When we finally get the slices, we will need to slice all the way to
- # the end even though we don't have a line number for it. Fortunately,
- # None does the job nicely.
- linenos.append(None)
-
- # Same problem at the other end: sometimes the ast tree has its
- # first complete statement not starting on line 0. In this case
- # we might miss part of it. This fixes ticket 266993. Thanks Gael!
- linenos[0] = 0
-
- lines = python.splitlines()
-
- # Create a list of atomic commands.
- cmds = []
- for i, j in zip(linenos[:-1], linenos[1:]):
- cmd = lines[i:j]
- if cmd:
- cmds.append('\n'.join(cmd)+'\n')
-
- return cmds
-
-
class InputSplitter(object):
"""An object that can split Python source input in executable blocks.
@@ -445,96 +373,18 @@ class InputSplitter(object):
if not self._full_dedent:
return False
else:
- nblocks = len(split_blocks(''.join(self._buffer)))
- if nblocks==1:
+ try:
+ code_ast = ast.parse(u''.join(self._buffer))
+ except Exception:
return False
+ else:
+ if len(code_ast.body) == 1:
+ return False
# When input is complete, then termination is marked by an extra blank
# line at the end.
last_line = self.source.splitlines()[-1]
return bool(last_line and not last_line.isspace())
-
- def split_blocks(self, lines):
- """Split a multiline string into multiple input blocks.
-
- Note: this method starts by performing a full reset().
-
- Parameters
- ----------
- lines : str
- A possibly multiline string.
-
- Returns
- -------
- blocks : list
- A list of strings, each possibly multiline. Each string corresponds
- to a single block that can be compiled in 'single' mode (unless it
- has a syntax error)."""
-
- # This code is fairly delicate. If you make any changes here, make
- # absolutely sure that you do run the full test suite and ALL tests
- # pass.
-
- self.reset()
- blocks = []
-
- # Reversed copy so we can use pop() efficiently and consume the input
- # as a stack
- lines = lines.splitlines()[::-1]
- # Outer loop over all input
- while lines:
- #print 'Current lines:', lines # dbg
- # Inner loop to build each block
- while True:
- # Safety exit from inner loop
- if not lines:
- break
- # Grab next line but don't push it yet
- next_line = lines.pop()
- # Blank/empty lines are pushed as-is
- if not next_line or next_line.isspace():
- self.push(next_line)
- continue
-
- # Check indentation changes caused by the *next* line
- indent_spaces, _full_dedent = self._find_indent(next_line)
-
- # If the next line causes a dedent, it can be for two differnt
- # reasons: either an explicit de-dent by the user or a
- # return/raise/pass statement. These MUST be handled
- # separately:
- #
- # 1. the first case is only detected when the actual explicit
- # dedent happens, and that would be the *first* line of a *new*
- # block. Thus, we must put the line back into the input buffer
- # so that it starts a new block on the next pass.
- #
- # 2. the second case is detected in the line before the actual
- # dedent happens, so , we consume the line and we can break out
- # to start a new block.
-
- # Case 1, explicit dedent causes a break.
- # Note: check that we weren't on the very last line, else we'll
- # enter an infinite loop adding/removing the last line.
- if _full_dedent and lines and not next_line.startswith(' '):
- lines.append(next_line)
- break
-
- # Otherwise any line is pushed
- self.push(next_line)
-
- # Case 2, full dedent with full block ready:
- if _full_dedent or \
- self.indent_spaces==0 and not self.push_accepts_more():
- break
- # Form the new block with the current source input
- blocks.append(self.source_reset())
-
- #return blocks
- # HACK!!! Now that our input is in blocks but guaranteed to be pure
- # python syntax, feed it back a second time through the AST-based
- # splitter, which is more accurate than ours.
- return split_blocks(''.join(blocks))
#------------------------------------------------------------------------
# Private interface
diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py
index d3c2c41..cd9595e 100644
--- a/IPython/core/interactiveshell.py
+++ b/IPython/core/interactiveshell.py
@@ -20,6 +20,7 @@ from __future__ import absolute_import
import __builtin__
import __future__
import abc
+import ast
import atexit
import codeop
import inspect
@@ -2098,115 +2099,95 @@ class InteractiveShell(Configurable, Magic):
except:
self.showtraceback()
warn('Unknown failure executing file: <%s>' % fname)
-
+
def run_cell(self, cell, store_history=True):
- """Run the contents of an entire multiline 'cell' of code, and store it
- in the history.
-
- The cell is split into separate blocks which can be executed
- individually. Then, based on how many blocks there are, they are
- executed as follows:
-
- - A single block: 'single' mode. If it is also a single line, dynamic
- transformations, including automagic and macros, will be applied.
-
- If there's more than one block, it depends:
-
- - if the last one is no more than two lines long, run all but the last
- in 'exec' mode and the very last one in 'single' mode. This makes it
- easy to type simple expressions at the end to see computed values. -
- otherwise (last one is also multiline), run all in 'exec' mode
-
- When code is executed in 'single' mode, :func:`sys.displayhook` fires,
- results are displayed and output prompts are computed. In 'exec' mode,
- no results are displayed unless :func:`print` is called explicitly;
- this mode is more akin to running a script.
-
+ """Run a complete IPython cell.
+
Parameters
----------
cell : str
- A single or multiline string.
+ The code (including IPython code such as %magic functions) to run.
+ store_history : bool
+ If True, the raw and translated cell will be stored in IPython's
+ history. For user code calling back into IPython's machinery, this
+ should be set to False.
"""
- # Store the untransformed code
raw_cell = cell
-
- # Code transformation and execution must take place with our
- # modifications to builtins.
with self.builtin_trap:
+ cell = self.prefilter_manager.prefilter_lines(cell)
- # We need to break up the input into executable blocks that can
- # be runin 'single' mode, to provide comfortable user behavior.
- blocks = self.input_splitter.split_blocks(cell)
-
- if not blocks: # Blank cell
- return
-
- # We only do dynamic transforms on a single line. But a macro
- # can be expanded to several lines, so we need to split it
- # into input blocks again.
- if len(cell.splitlines()) <= 1:
- cell = self.prefilter_manager.prefilter_line(blocks[0])
- blocks = self.input_splitter.split_blocks(cell)
-
- # Store the 'ipython' version of the cell as well, since
- # that's what needs to go into the translated history and get
- # executed (the original cell may contain non-python syntax).
- cell = ''.join(blocks)
-
# Store raw and processed history
if store_history:
self.history_manager.store_inputs(self.execution_count,
cell, raw_cell)
self.logger.log(cell, raw_cell)
-
- # All user code execution should take place with our
- # modified displayhook.
+
+ cell_name = self.compile.cache(cell, self.execution_count)
+
with self.display_trap:
- # Single-block input should behave like an interactive prompt
- if len(blocks) == 1:
- out = self.run_source(blocks[0])
- # Write output to the database. Does nothing unless
- # history output logging is enabled.
- if store_history:
- self.history_manager.store_output(self.execution_count)
- # Since we return here, we need to update the
- # execution count
- self.execution_count += 1
- return out
-
- # In multi-block input, if the last block is a simple (one-two
- # lines) expression, run it in single mode so it produces output.
- # Otherwise just run it all in 'exec' mode. This seems like a
- # reasonable usability design.
- last = blocks[-1]
- last_nlines = len(last.splitlines())
+ try:
+ code_ast = ast.parse(cell, filename=cell_name)
+ except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError):
+ # Case 1
+ self.showsyntaxerror()
+ self.execution_count += 1
+ return None
+
+ interactivity = 'last' # Last node to be run interactive
+ if len(cell.splitlines()) == 1:
+ interactivity = 'all' # Single line; run fully interactive
+
+ self.run_ast_nodes(code_ast.body, cell_name, interactivity)
- if last_nlines < 2:
- # Here we consider the cell split between 'body' and 'last',
- # store all history and execute 'body', and if successful, then
- # proceed to execute 'last'.
-
- # Get the main body to run as a cell
- ipy_body = ''.join(blocks[:-1])
- retcode = self.run_source(ipy_body, symbol='exec',
- post_execute=False)
- if retcode==0:
- # Last expression compiled as 'single' so it
- # produces output
- self.run_source(last)
- else:
- # Run the whole cell as one entity, storing both raw and
- # processed input in history
- self.run_source(cell, symbol='exec')
-
- # Write output to the database. Does nothing unless
- # history output logging is enabled.
if store_history:
+ # Write output to the database. Does nothing unless
+ # history output logging is enabled.
self.history_manager.store_output(self.execution_count)
# Each cell is a *single* input, regardless of how many lines it has
self.execution_count += 1
-
+
+ def run_ast_nodes(self, nodelist, cell_name, interactivity='last'):
+ """Run a sequence of AST nodes. The execution mode depends on the
+ interactivity parameter.
+
+ Parameters
+ ----------
+ nodelist : list
+ A sequence of AST nodes to run.
+ cell_name : str
+ Will be passed to the compiler as the filename of the cell. Typically
+ the value returned by ip.compile.cache(cell).
+ interactivity : str
+ 'all', 'last' or 'none', specifying which nodes should be run
+ interactively (displaying output from expressions). Other values for
+ this parameter will raise a ValueError.
+ """
+ if not nodelist:
+ return
+
+ if interactivity == 'none':
+ to_run_exec, to_run_interactive = nodelist, []
+ elif interactivity == 'last':
+ to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
+ elif interactivity == 'all':
+ to_run_exec, to_run_interactive = [], nodelist
+ else:
+ raise ValueError("Interactivity was %r" % interactivity)
+
+ exec_count = self.execution_count
+ if to_run_exec:
+ mod = ast.Module(to_run_exec)
+ self.code_to_run = code = self.compile(mod, cell_name, "exec")
+ if self.run_code(code) == 1:
+ return
+
+ if to_run_interactive:
+ mod = ast.Interactive(to_run_interactive)
+ self.code_to_run = code = self.compile(mod, cell_name, "single")
+ return self.run_code(code)
+
+
# PENDING REMOVAL: this method is slated for deletion, once our new
# input logic has been 100% moved to frontends and is stable.
def runlines(self, lines, clean=False):
@@ -2296,7 +2277,8 @@ class InteractiveShell(Configurable, Magic):
print 'encoding', self.stdin_encoding # dbg
try:
- code = self.compile(usource, symbol, self.execution_count)
+ code_name = self.compile.cache(usource, self.execution_count)
+ code = self.compile(usource, code_name, symbol)
except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError):
# Case 1
self.showsyntaxerror(filename)
diff --git a/IPython/core/tests/test_compilerop.py b/IPython/core/tests/test_compilerop.py
index 2fac2fc..e9989db 100644
--- a/IPython/core/tests/test_compilerop.py
+++ b/IPython/core/tests/test_compilerop.py
@@ -40,12 +40,12 @@ def test_code_name2():
nt.assert_true(name.startswith('<ipython-input-9'))
-def test_compiler():
+def test_cache():
"""Test the compiler correctly compiles and caches inputs
"""
cp = compilerop.CachingCompiler()
ncache = len(linecache.cache)
- cp('x=1', 'single')
+ cp.cache('x=1')
nt.assert_true(len(linecache.cache) > ncache)
def setUp():
@@ -53,10 +53,10 @@ def setUp():
# as GTK, can change the default encoding, which can hide bugs.)
nt.assert_equal(sys.getdefaultencoding(), "ascii")
-def test_compiler_unicode():
+def test_cache_unicode():
cp = compilerop.CachingCompiler()
ncache = len(linecache.cache)
- cp(u"t = 'žćčšđ'", "single")
+ cp.cache(u"t = 'žćčšđ'")
nt.assert_true(len(linecache.cache) > ncache)
def test_compiler_check_cache():
@@ -64,7 +64,7 @@ def test_compiler_check_cache():
"""
# Rather simple-minded tests that just exercise the API
cp = compilerop.CachingCompiler()
- cp('x=1', 'single', 99)
+ cp.cache('x=1', 99)
# Ensure now that after clearing the cache, our entries survive
cp.check_cache()
for k in linecache.cache:
diff --git a/IPython/core/tests/test_handlers.py b/IPython/core/tests/test_handlers.py
index c8a9c94..483a8d7 100644
--- a/IPython/core/tests/test_handlers.py
+++ b/IPython/core/tests/test_handlers.py
@@ -43,9 +43,7 @@ def run(tests):
for pre, post in tests:
global num_tests
num_tests += 1
- ip.runlines(pre)
- ip.runlines('_i') # Not sure why I need this...
- actual = ip.user_ns['_i']
+ actual = ip.prefilter_manager.prefilter_lines(pre)
if actual != None:
actual = actual.rstrip('\n')
if actual != post:
diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py
index 880a136..bfb5267 100644
--- a/IPython/core/tests/test_inputsplitter.py
+++ b/IPython/core/tests/test_inputsplitter.py
@@ -286,92 +286,6 @@ class InputSplitterTestCase(unittest.TestCase):
isp.push('run foo')
self.assertFalse(isp.push_accepts_more())
- def check_split(self, block_lines, compile=True):
- blocks = assemble(block_lines)
- lines = ''.join(blocks)
- oblock = self.isp.split_blocks(lines)
- self.assertEqual(oblock, blocks)
- if compile:
- for block in blocks:
- self.isp._compile(block)
-
- def test_split(self):
- # All blocks of input we want to test in a list. The format for each
- # block is a list of lists, with each inner lists consisting of all the
- # lines (as single-lines) that should make up a sub-block.
-
- # Note: do NOT put here sub-blocks that don't compile, as the
- # check_split() routine makes a final verification pass to check that
- # each sub_block, as returned by split_blocks(), does compile
- # correctly.
- all_blocks = [ [['x=1']],
-
- [['x=1'],
- ['y=2']],
-
- [['x=1',
- '# a comment'],
- ['y=11']],
-
- [['if 1:',
- ' x=1'],
- ['y=3']],
-
- [['def f(x):',
- ' return x'],
- ['x=1']],
-
- [['def f(x):',
- ' x+=1',
- ' ',
- ' return x'],
- ['x=1']],
-
- [['def f(x):',
- ' if x>0:',
- ' y=1',
- ' # a comment',
- ' else:',
- ' y=4',
- ' ',
- ' return y'],
- ['x=1'],
- ['if 1:',
- ' y=11'] ],
-
- [['for i in range(10):'
- ' x=i**2']],
-
- [['for i in range(10):'
- ' x=i**2'],
- ['z = 1']],
-
- [['"asdf"']],
-
- [['"asdf"'],
- ['10'],
- ],
-
- [['"""foo',
- 'bar"""']],
- ]
- for block_lines in all_blocks:
- self.check_split(block_lines)
-
- def test_split_syntax_errors(self):
- # Block splitting with invalid syntax
- all_blocks = [ [['a syntax error']],
-
- [['x=1',
- 'another syntax error']],
-
- [['for i in range(10):'
- ' yet another error']],
-
- ]
- for block_lines in all_blocks:
- self.check_split(block_lines, compile=False)
-
def test_unicode(self):
self.isp.push(u"Pérez")
self.isp.push(u'\xc3\xa9')
diff --git a/IPython/core/tests/test_interactiveshell.py b/IPython/core/tests/test_interactiveshell.py
index 2d533e4..8db596c 100644
--- a/IPython/core/tests/test_interactiveshell.py
+++ b/IPython/core/tests/test_interactiveshell.py
@@ -56,7 +56,6 @@ class InteractiveShellTestCase(unittest.TestCase):
self.assertEquals(ip.user_ns['x'], 2)
self.assertEquals(ip.user_ns['y'], 3)
- @dec.skip_known_failure
def test_multiline_string_cells(self):
"Code sprinkled with multiline strings should execute (GH-306)"
ip = get_ipython()
diff --git a/IPython/lib/tests/test_irunner.py b/IPython/lib/tests/test_irunner.py
index dd8cfd3..1ca1f7a 100755
--- a/IPython/lib/tests/test_irunner.py
+++ b/IPython/lib/tests/test_irunner.py
@@ -97,10 +97,10 @@ In [7]: autocall 0
Automatic calling is: OFF
In [8]: cos pi
- File "<ipython-input-8-6bd7313dd9a9>", line 1
+ File "<ipython-input-8-586f1104ea44>", line 1
cos pi
^
-SyntaxError: invalid syntax
+SyntaxError: unexpected EOF while parsing
In [9]: cos(pi)