From 0a7f662cac8f303c25ffc8ed8fe625f349c682bd 2010-09-06 19:12:47 From: Fernando Perez Date: 2010-09-06 19:12:47 Subject: [PATCH] Add experimental support for cell-based execution. For now the implementation is a bit hackish, but it does already allow pasting very complex examples such as: http://matplotlib.sourceforge.net/examples/pylab_examples/demo_agg_filter.html which before couldn't be executed. We'll need to test it further. --- diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py index abb6e48..4805f5d 100644 --- a/IPython/core/inputsplitter.py +++ b/IPython/core/inputsplitter.py @@ -159,6 +159,88 @@ def get_input_encoding(): # Classes and functions for normal Python syntax handling #----------------------------------------------------------------------------- +# HACK! This implementation, written by Robert K a while ago using the +# compiler module, is more robust than the other one below, but it expects its +# input to be pure python (no ipython syntax). For now we're using it as a +# second-pass splitter after the first pass transforms the input to pure +# python. + +def split_blocks(python): + """ Split multiple lines of code into discrete commands that can be + executed singly. + + Parameters + ---------- + python : str + Pure, exec'able Python code. + + Returns + ------- + commands : list of str + Separate commands that can be exec'ed independently. + """ + + import compiler + + # compiler.parse treats trailing spaces after a newline as a + # SyntaxError. This is different than codeop.CommandCompiler, which + # will compile the trailng spaces just fine. We simply strip any + # trailing whitespace off. Passing a string with trailing whitespace + # to exec will fail however. There seems to be some inconsistency in + # how trailing whitespace is handled, but this seems to work. + python_ori = python # save original in case we bail on error + python = python.strip() + + # The compiler module does not like unicode. We need to convert + # it encode it: + if isinstance(python, unicode): + # Use the utf-8-sig BOM so the compiler detects this a UTF-8 + # encode string. + python = '\xef\xbb\xbf' + python.encode('utf-8') + + # The compiler module will parse the code into an abstract syntax tree. + # This has a bug with str("a\nb"), but not str("""a\nb""")!!! + try: + ast = compiler.parse(python) + except: + return [python_ori] + + # Uncomment to help debug the ast tree + # for n in ast.node: + # print n.lineno,'->',n + + # Each separate command is available by iterating over ast.node. The + # lineno attribute is the line number (1-indexed) beginning the commands + # suite. + # lines ending with ";" yield a Discard Node that doesn't have a lineno + # attribute. These nodes can and should be discarded. But there are + # other situations that cause Discard nodes that shouldn't be discarded. + # We might eventually discover other cases where lineno is None and have + # to put in a more sophisticated test. + linenos = [x.lineno-1 for x in ast.node if x.lineno is not None] + + # When we finally get the slices, we will need to slice all the way to + # the end even though we don't have a line number for it. Fortunately, + # None does the job nicely. + linenos.append(None) + + # Same problem at the other end: sometimes the ast tree has its + # first complete statement not starting on line 0. In this case + # we might miss part of it. This fixes ticket 266993. Thanks Gael! + linenos[0] = 0 + + lines = python.splitlines() + + # Create a list of atomic commands. + cmds = [] + for i, j in zip(linenos[:-1], linenos[1:]): + cmd = lines[i:j] + if cmd: + cmds.append('\n'.join(cmd)+'\n') + + return cmds + + class InputSplitter(object): """An object that can split Python source input in executable blocks. @@ -431,7 +513,11 @@ class InputSplitter(object): # Form the new block with the current source input blocks.append(self.source_reset()) - return blocks + #return blocks + # HACK!!! Now that our input is in blocks but guaranteed to be pure + # python syntax, feed it back a second time through the AST-based + # splitter, which is more accurate than ours. + return split_blocks(''.join(blocks)) #------------------------------------------------------------------------ # Private interface diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index eb3e455..ed72e35 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -46,6 +46,7 @@ from IPython.core.error import TryNext, UsageError from IPython.core.extensions import ExtensionManager from IPython.core.fakemodule import FakeModule, init_fakemod_dict from IPython.core.inputlist import InputList +from IPython.core.inputsplitter import IPythonInputSplitter from IPython.core.logger import Logger from IPython.core.magic import Magic from IPython.core.payload import PayloadManager @@ -154,6 +155,7 @@ class InteractiveShell(Configurable, Magic): exit_now = CBool(False) filename = Str("") ipython_dir= Unicode('', config=True) # Set to get_ipython_dir() in __init__ + input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter') logstart = CBool(False, config=True) logfile = Str('', config=True) logappend = Str('', config=True) @@ -212,7 +214,7 @@ class InteractiveShell(Configurable, Magic): def __init__(self, config=None, ipython_dir=None, user_ns=None, user_global_ns=None, - custom_exceptions=((),None)): + custom_exceptions=((), None)): # This is where traits with a config_key argument are updated # from the values on config. @@ -252,7 +254,7 @@ class InteractiveShell(Configurable, Magic): # pre_config_initialization self.init_shadow_hist() - # The next section should contain averything that was in ipmaker. + # The next section should contain everything that was in ipmaker. self.init_logstart() # The following was in post_config_initialization @@ -386,6 +388,10 @@ class InteractiveShell(Configurable, Magic): # Indentation management self.indent_current_nsp = 0 + # Input splitter, to split entire cells of input into either individual + # interactive statements or whole blocks. + self.input_splitter = IPythonInputSplitter() + def init_encoding(self): # Get system encoding at startup time. Certain terminals (like Emacs # under Win32 have it set to None, and we need to have a known valid @@ -2061,6 +2067,46 @@ class InteractiveShell(Configurable, Magic): self.showtraceback() warn('Unknown failure executing file: <%s>' % fname) + def run_cell(self, cell): + """Run the contents of an entire multiline 'cell' of code. + + The cell is split into separate blocks which can be executed + individually. Then, based on how many blocks there are, they are + executed as follows: + + - A single block: 'single' mode. + + If there's more than one block, it depends: + + - if the last one is a single line long, run all but the last in + 'exec' mode and the very last one in 'single' mode. This makes it + easy to type simple expressions at the end to see computed values. + - otherwise (last one is also multiline), run all in 'exec' mode + + When code is executed in 'single' mode, :func:`sys.displayhook` fires, + results are displayed and output prompts are computed. In 'exec' mode, + no results are displayed unless :func:`print` is called explicitly; + this mode is more akin to running a script. + + Parameters + ---------- + cell : str + A single or multiline string. + """ + blocks = self.input_splitter.split_blocks(cell) + if not blocks: + return + + if len(blocks) == 1: + self.runlines(blocks[0]) + + last = blocks[-1] + if len(last.splitlines()) < 2: + map(self.runcode, blocks[:-1]) + self.runlines(last) + else: + map(self.runcode, blocks) + def runlines(self, lines, clean=False): """Run a string of one or more lines of source. @@ -2166,7 +2212,7 @@ class InteractiveShell(Configurable, Magic): else: return None - def runcode(self,code_obj): + def runcode(self, code_obj): """Execute a code object. When an exception occurs, self.showtraceback() is called to display a diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py index d2cb1e7..a9cba51 100644 --- a/IPython/core/tests/test_inputsplitter.py +++ b/IPython/core/tests/test_inputsplitter.py @@ -278,8 +278,8 @@ class InputSplitterTestCase(unittest.TestCase): [['x=1'], ['y=2']], - [['x=1'], - ['# a comment'], + [['x=1', + '# a comment'], ['y=11']], [['if 1:', @@ -322,11 +322,11 @@ class InputSplitterTestCase(unittest.TestCase): # Block splitting with invalid syntax all_blocks = [ [['a syntax error']], - [['x=1'], - ['a syntax error']], + [['x=1', + 'another syntax error']], [['for i in range(10):' - ' an error']], + ' yet another error']], ] for block_lines in all_blocks: diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index be76f30..0d659f1 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -179,7 +179,12 @@ class Kernel(Configurable): else: # FIXME: runlines calls the exception handler itself. shell._reply_content = None - shell.runlines(code) + + # Experimental: cell mode! Test more before turning into + # default and removing the hacks around runlines. + shell.run_cell(code) + # For now leave this here until we're sure we can stop using it + #shell.runlines(code) except: status = u'error' # FIXME: this code right now isn't being used yet by default, diff --git a/IPython/zmq/zmqshell.py b/IPython/zmq/zmqshell.py index 73b9326..fde5194 100644 --- a/IPython/zmq/zmqshell.py +++ b/IPython/zmq/zmqshell.py @@ -26,12 +26,12 @@ from IPython.core.interactiveshell import ( ) from IPython.core.displayhook import DisplayHook from IPython.core.macro import Macro +from IPython.core.payloadpage import install_payload_page from IPython.utils.path import get_py_filename from IPython.utils.text import StringTypes from IPython.utils.traitlets import Instance, Type, Dict from IPython.utils.warn import warn from IPython.zmq.session import extract_header -from IPython.core.payloadpage import install_payload_page from session import Session #-----------------------------------------------------------------------------