Merge branch 'newkernel' into qtnewkernel

IPython/lib/guisupport.py

@@ -0,0 +1,140 b''
+#!/usr/bin/env python
+# coding: utf-8
+"""
+Support for creating GUI apps and starting event loops.
+
+IPython's GUI integration allows interative plotting and GUI usage in IPython
+session. IPython has two different types of GUI integration:
+
+1. The terminal based IPython supports GUI event loops through Python's
+   PyOS_InputHook. PyOS_InputHook is a hook that Python calls periodically
+   whenever raw_input is waiting for a user to type code. We implement GUI
+   support in the terminal by setting PyOS_InputHook to a function that 
+   iterates the event loop for a short while. It is important to note that
+   in this situation, the real GUI event loop is NOT run in the normal
+   manner, so you can't use the normal means to detect that it is running.
+2. In the two process IPython kernel/frontend, the GUI event loop is run in 
+   the kernel. In this case, the event loop is run in the normal manner by
+   calling the function or method of the GUI toolkit that starts the event
+   loop.
+
+In addition to starting the GUI event loops in one of these two ways, IPython
+will *always* create an appropriate GUI application object when GUi
+integration is enabled.
+
+If you want your GUI apps to run in IPython you need to do two things:
+
+1. Test to see if there is already an existing main application object. If
+   there is, you should use it. If there is not an existing application object
+   you should create one.
+2. Test to see if the GUI event loop is running. If it is, you should not
+   start it. If the event loop is not running you may start it.
+
+This module contains functions for each toolkit that perform these things
+in a consistent manner. Because of how PyOS_InputHook runs the event loop
+you cannot detect if the event loop is running using the traditional calls
+(such as ``wx.GetApp.IsMainLoopRunning()`` in wxPython). If PyOS_InputHook is
+set These methods will return a false negative. That is, they will say the
+event loop is not running, when is actually is. To work around this limitation
+we proposed the following informal protocol:
+
+* Whenever someone starts the event loop, they *must* set the ``_in_event_loop``
+  attribute of the main application object to ``True``. This should be done
+  regardless of how the event loop is actually run.
+* Whenever someone stops the event loop, they *must* set the ``_in_event_loop``
+  attribute of the main application object to ``False``.
+* If you want to see if the event loop is running, you *must* use ``hasattr``
+  to see if ``_in_event_loop`` attribute has been set. If it is set, you
+  *must* use its value. If it has not been set, you can query the toolkit
+  in the normal manner.
+
+The functions below implement this logic for each GUI toolkit. If you need
+to create custom application subclasses, you will likely have to modify this
+code for your own purposes. This code can be copied into your own project
+so you don't have to depend on IPython.
+
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2010  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# wx
+#-----------------------------------------------------------------------------
+
+def get_app_wx(*args, **kwargs):
+    """Create a new wx app or return an exiting one."""
+    import wx
+    app = wx.GetApp()
+    if app is None:
+        app = wx.PySimpleApp(*args, **kwargs)
+    return app
+
+def is_event_loop_running_wx(app=None):
+    """Is the wx event loop running."""
+    if app is None:
+        app = get_app_wx()
+    if hasattr(app, '_in_event_loop'):
+        return app._in_event_loop
+    else:
+        return app.IsMainLoopRunning()
+
+def start_event_loop_wx(app=None):
+    """Start the wx event loop in a consistent manner."""
+    if app is None:
+        app = get_app_wx()
+    if not is_event_loop_running_wx(app):
+        app._in_event_loop = True
+        app.MainLoop()
+        app._in_event_loop = False
+    else:
+        app._in_event_loop = True
+
+#-----------------------------------------------------------------------------
+# qt4
+#-----------------------------------------------------------------------------
+
+def get_app_qt4(*args, **kwargs):
+    """Create a new qt4 app or return an existing one."""
+    from PyQt4 import QtGui
+    app = QtGui.QApplication.instance()
+    if app is None:
+        app = QtGui.QApplication(*args, **kwargs)
+    return app
+
+def is_event_loop_running_qt4(app=None):
+    """Is the qt4 event loop running."""
+    if app is None:
+        app = get_app_qt4()
+    if hasattr(app, '_in_event_loop'):
+        return app._in_event_loop
+    else:
+        # Does qt4 provide a other way to detect this?
+        return False
+
+def start_event_loop_qt4(app=None):
+    """Start the qt4 event loop in a consistent manner."""
+    if app is None:
+        app = get_app_qt4()
+    if not is_event_loop_running_qt4(app):
+        app._in_event_loop = True
+        app.exec_()
+        app._in_event_loop = False
+    else:
+        app._in_event_loop = True
+
+#-----------------------------------------------------------------------------
+# Tk
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# gtk
+#-----------------------------------------------------------------------------

IPython/zmq/pylab/backend_payload_svg.py

@@ -1,3 +1,9 b''
+"""Produce SVG versions of active plots for display by the rich Qt frontend.
+"""
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
 # Standard library imports
 from cStringIO import StringIO
 
@@ -8,12 +14,14 b' from matplotlib._pylab_helpers import Gcf'
 # Local imports.
 from backend_payload import add_plot_payload
 
+#-----------------------------------------------------------------------------
+# Functions
+#-----------------------------------------------------------------------------
 
 def show():
     """ Deliver a SVG payload.
     """
-    figure_manager = Gcf.get_active()
-    if figure_manager is not None:
+    for figure_manager in Gcf.get_all_fig_managers():
         # Make the background transparent.
         # figure_manager.canvas.figure.patch.set_alpha(0.0)
         # Set the background to white instead so it looks good on black.
@@ -22,6 +30,7 b' def show():'
         data = svg_from_canvas(figure_manager.canvas)
         add_plot_payload('svg', data)
 
+
 def svg_from_canvas(canvas):
     """ Return a string containing the SVG representation of a FigureCanvasSvg.
     """

IPython/zmq/zmqshell.py

@@ -1,8 +1,29 b''
+"""A ZMQ-based subclass of InteractiveShell.
+
+This code is meant to ease the refactoring of the base InteractiveShell into
+something with a cleaner architecture for 2-process use, without actually
+breaking InteractiveShell itself.  So we're doing something a bit ugly, where
+we subclass and override what we want to fix.  Once this is working well, we
+can go back to the base class and refactor the code for a cleaner inheritance
+implementation that doesn't rely on so much monkeypatching.
+
+But this lets us maintain a fully working IPython as we develop the new
+machinery.  This should thus be thought of as scaffolding.
+"""
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from __future__ import print_function
+
+# Stdlib
 import inspect
+import os
 import re
 import sys
+
 from subprocess import Popen, PIPE
 
+# Our own
 from IPython.core.interactiveshell import (
     InteractiveShell, InteractiveShellABC
 )
@@ -16,9 +37,16 b' from IPython.zmq.session import extract_header'
 from IPython.core.payloadpage import install_payload_page
 from session import Session
 
+#-----------------------------------------------------------------------------
+# Globals and side-effects
+#-----------------------------------------------------------------------------
+
 # Install the payload version of page.
 install_payload_page()
 
+#-----------------------------------------------------------------------------
+# Functions and classes
+#-----------------------------------------------------------------------------
 
 class ZMQDisplayHook(DisplayHook):
 
@@ -56,16 +84,23 b' class ZMQInteractiveShell(InteractiveShell):'
     displayhook_class = Type(ZMQDisplayHook)
 
     def system(self, cmd):
-        cmd = self.var_expand(cmd, depth=2)
+        cmd = self.var_expand(cmd, depth=2).strip()
+        
+        # Runnning a bacgkrounded process from within the gui isn't supported
+        # because we do p.wait() at the end.  So instead of silently blocking
+        # we simply refuse to run in this mode, to avoid surprising the user.
+        if cmd.endswith('&'):
+            raise OSError("Background processes not supported.")
+        
         sys.stdout.flush()
         sys.stderr.flush()
         p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
         for line in p.stdout.read().split('\n'):
             if len(line) > 0:
-                print line
+                print(line)
         for line in p.stderr.read().split('\n'):
             if len(line) > 0:
-                print line
+                print(line, file=sys.stderr)
         p.wait()
 
     def init_io(self):
@@ -349,7 +384,11 b' class ZMQInteractiveShell(InteractiveShell):'
 
         if use_temp:
             filename = self.shell.mktempfile(data)
-            print 'IPython will make a temporary file named:',filename
+            print('IPython will make a temporary file named:', filename)
+
+        # Make sure we send to the client an absolute path, in case the working
+        # directory of client and kernel don't match
+        filename = os.path.abspath(filename)
 
         payload = {
             'source' : 'IPython.zmq.zmqshell.ZMQInteractiveShell.edit_magic',

docs/Makefile

@@ -101,4 +101,4 b' gitwash-update:'
 	cd source/development/gitwash && rename 's/.rst/.txt/' *.rst
 
 nightly: dist
-	rsync -avH --delete dist/ ipython:www/doc/nightly
  No newline at end of file
+	rsync -avH --delete dist/ ipython:www/doc/nightly

docs/source/development/messaging.txt

@@ -130,15 +130,9 b' Messages on the XREP/XREQ socket'
 Execute
 -------
 
-The execution request contains a single string, but this may be a multiline
-string.  The kernel is responsible for splitting this into possibly more than
-one block and deciding whether to compile these in 'single' or 'exec' mode.
-We're still sorting out this policy.  The current inputsplitter is capable of
-splitting the input for blocks that can all be run as 'single', but in the long
-run it may prove cleaner to only use 'single' mode for truly single-line
-inputs, and run all multiline input in 'exec' mode.  This would preserve the
-natural behavior of single-line inputs while allowing long cells to behave more
-likea a script.  This design will be refined as we complete the implementation.
+This message type is used by frontends to ask the kernel to execute code on
+behalf of the user, in a namespace reserved to the user's variables (and thus
+separate from the kernel's own internal code and variables).
 
 Message type: ``execute_request``::
 
@@ -148,17 +142,94 b' Message type: ``execute_request``::'
 
     # A boolean flag which, if True, signals the kernel to execute this
     # code as quietly as possible.  This means that the kernel will compile
-    # the code with 'exec' instead of 'single' (so sys.displayhook will not
-    # fire), and will *not*:
+    # the code witIPython/core/tests/h 'exec' instead of 'single' (so
+    # sys.displayhook will not fire), and will *not*:
     #   - broadcast exceptions on the PUB socket
     #   - do any logging
     #   - populate any history
+    #
     # The default is False.
     'silent' : bool,
+
+    # A list of variable names from the user's namespace to be retrieved.  What
+    # returns is a JSON string of the variable's repr(), not a python object.
+    'user_variables' : list,
+
+    # Similarly, a dict mapping names to expressions to be evaluated in the
+    # user's dict.
+    'user_expressions' : dict,
     }
 
-Upon execution, the kernel *always* sends a reply, with a status code
-indicating what happened and additional data depending on the outcome.
+The ``code`` field contains a single string, but this may be a multiline
+string.  The kernel is responsible for splitting this into possibly more than
+one block and deciding whether to compile these in 'single' or 'exec' mode.
+We're still sorting out this policy.  The current inputsplitter is capable of
+splitting the input for blocks that can all be run as 'single', but in the long
+run it may prove cleaner to only use 'single' mode for truly single-line
+inputs, and run all multiline input in 'exec' mode.  This would preserve the
+natural behavior of single-line inputs while allowing long cells to behave more
+likea a script.  This design will be refined as we complete the implementation.
+
+The ``user_`` fields deserve a detailed explanation.  In the past, IPython had
+the notion of a prompt string that allowed arbitrary code to be evaluated, and
+this was put to good use by many in creating prompts that displayed system
+status, path information, and even more esoteric uses like remote instrument
+status aqcuired over the network.  But now that IPython has a clean separation
+between the kernel and the clients, the notion of embedding 'prompt'
+maninpulations into the kernel itself feels awkward.  Prompts should be a
+frontend-side feature, and it should be even possible for different frontends
+to display different prompts while interacting with the same kernel.
+
+We have therefore abandoned the idea of a 'prompt string' to be evaluated by
+the kernel, and instead provide the ability to retrieve from the user's
+namespace information after the execution of the main ``code``, with two fields
+of the execution request:
+
+- ``user_variables``: If only variables from the user's namespace are needed, a
+  list of variable names can be passed and a dict with these names as keys and
+  their :func:`repr()` as values will be returned.
+
+- ``user_expressions``: For more complex expressions that require function
+  evaluations, a dict can be provided with string keys and arbitrary python
+  expressions as values.  The return message will contain also a dict with the
+  same keys and the :func:`repr()` of the evaluated expressions as value.
+
+With this information, frontends can display any status information they wish
+in the form that best suits each frontend (a status line, a popup, inline for a
+terminal, etc).
+
+.. Note::
+
+   In order to obtain the current execution counter for the purposes of
+   displaying input prompts, frontends simply make an execution request with an
+   empty code string and ``silent=True``.
+
+Execution semantics
+    Upon completion of the execution request, the kernel *always* sends a
+    reply, with a status code indicating what happened and additional data
+    depending on the outcome.
+
+    The ``code`` field is executed first, and then the ``user_variables`` and
+    ``user_expressions`` are computed.  This ensures that any error in the
+    latter don't harm the main code execution.
+
+    Any error in retrieving the ``user_variables`` or evaluating the
+    ``user_expressions`` will result in a simple error message in the return
+    fields of the form::
+
+       [ERROR] ExceptionType: Exception message
+
+    The user can simply send the same variable name or expression for
+    evaluation to see a regular traceback.
+
+Execution counter (old prompt number)
+    The kernel has a single, monotonically increasing counter of all execution
+    requests that are made with ``silent=False``.  This counter is used to
+    populate the ``In[n]``, ``Out[n]`` and ``_n`` variables, so clients will
+    likely want to display it in some form to the user, which will typically
+    (but not necessarily) be done in the prompts.  The value of this counter
+    will be returned as the ``execution_count`` field of all ``execute_reply```
+    messages.
 
 Message type: ``execute_reply``::
 
@@ -166,30 +237,25 b' Message type: ``execute_reply``::'
       # One of: 'ok' OR 'error' OR 'abort'
       'status' : str,
 
-      # This has the same structure as the output of a prompt request, but is
-      # for the client to set up the *next* prompt (with identical limitations
-      # to a prompt request)
-      'next_prompt' : {
-            'prompt_string' : str,
-            'prompt_number' : int,
-            'input_sep'     : str
-      },
-
-      # The prompt number of the actual execution for this code, which may be
-      # different from the one used when the code was typed, which was the
-      # 'next_prompt' field of the *previous* request.  They will differ in the
-      # case where there is more than one client talking simultaneously to a
-      # kernel, since the numbers can go out of sync.  GUI clients can use this
-      # to correct the previously written number in-place, terminal ones may
-      # re-print a corrected one if desired.
-      'prompt_number' : int,
+      # The global kernel counter that increases by one with each non-silent
+      # executed request.  This will typically be used by clients to display
+      # prompt numbers to the user.  If the request was a silent one, this will
+      # be the current value of the counter in the kernel.
+      'execution_count' : int,
+
+      # If the state_template was provided, this will contain the evaluated
+      # form of the template.
+      'state' : str,
     }
 
 When status is 'ok', the following extra fields are present::
 
     {
-      # The kernel will often transform the input provided to it.  This
-      # contains the transformed code, which is what was actually executed.
+      # The kernel will often transform the input provided to it.  If the
+      # '---->' transform had been applied, this is filled, otherwise it's the
+      # empty string.  So transformations like magics don't appear here, only
+      # autocall ones.
+    
       'transformed_code' : str,
 
       # The execution payload is a dict with string keys that may have been
@@ -235,31 +301,65 b" When status is 'error', the following extra fields are present::"
 When status is 'abort', there are for now no additional data fields.  This
 happens when the kernel was interrupted by a signal.
 
+Kernel attribute access
+-----------------------
 
-Prompt
-------
+While this protocol does not specify full RPC access to arbitrary methods of
+the kernel object, the kernel does allow read (and in some cases write) access
+to certain attributes.
 
-A simple request for a current prompt string.
+The policy for which attributes can be read is: any attribute of the kernel, or
+its sub-objects, that belongs to a :class:`Configurable` object and has been
+declared at the class-level with Traits validation, is in principle accessible
+as long as its name does not begin with a leading underscore.  The attribute
+itself will have metadata indicating whether it allows remote read and/or write
+access.  The message spec follows for attribute read and write requests.
 
-Message type: ``prompt_request``::
+Message type: ``getattr_request``::
 
-    content = {}
+    content = {
+        # The (possibly dotted) name of the attribute
+	'name' : str,
+    }
+
+When a ``getattr_request`` fails, there are two possible error types:
+
+- AttributeError: this type of error was raised when trying to access the
+  given name by the kernel itself.  This means that the attribute likely
+  doesn't exist.
+
+- AccessError: the attribute exists but its value is not readable remotely.
 
-In the reply, the prompt string comes back with the prompt number placeholder
-*unevaluated*.  The message format is:
-    
-Message type: ``prompt_reply``::
+
+Message type: ``getattr_reply``::
+
+    content = {
+        # One of ['ok', 'AttributeError', 'AccessError'].
+        'status' : str,
+	# If status is 'ok', a JSON object.
+	'value' : object,
+    }
+
+Message type: ``setattr_request``::
 
     content = {
-      'prompt_string' : str,
-      'prompt_number' : int,
-      'input_sep'     : str
+        # The (possibly dotted) name of the attribute
+	'name' : str,
+	
+	# A JSON-encoded object, that will be validated by the Traits
+	# information in the kernel
+	'value' : object,
     }
 
-Clients can produce a prompt with ``prompt_string.format(prompt_number)``, but
-they should be aware that the actual prompt number for that input could change
-later, in the case where multiple clients are interacting with a single
-kernel. 
+When a ``setattr_request`` fails, there are also two possible error types with
+similar meanings  as those of the ``getattr_request`` case, but for writing.
+    
+Message type: ``setattr_reply``::
+
+    content = {
+        # One of ['ok', 'AttributeError', 'AccessError'].
+        'status' : str,
+    }
 
 
 Object information
@@ -276,12 +376,12 b' Message type: ``object_info_request``::'
 
     content = {
         # The (possibly dotted) name of the object to be searched in all
-    # relevant namespaces
-    'name' : str,
+	# relevant namespaces
+        'name' : str,
 
-    # The level of detail desired.  The default (0) is equivalent to typing
-    # 'x?' at the prompt, 1 is equivalent to 'x??'.
-    'detail_level' : int,
+    	# The level of detail desired.  The default (0) is equivalent to typing
+	# 'x?' at the prompt, 1 is equivalent to 'x??'.
+	'detail_level' : int,
     }
 
 The returned information will be a dictionary with keys very similar to the
@@ -315,9 +415,29 b' Message type: ``object_info_reply``::'
     'file' : str,
 
     # For pure Python callable objects, we can reconstruct the object
-    # definition line which provides its call signature
+    # definition line which provides its call signature.  For convenience this
+    # is returned as a single 'definition' field, but below the raw parts that
+    # compose it are also returned as the argspec field.
     'definition' : str,
 
+    # The individual parts that together form the definition string.  Clients
+    # with rich display capabilities may use this to provide a richer and more
+    # precise representation of the definition line (e.g. by highlighting
+    # arguments based on the user's cursor position).  For non-callable
+    # objects, this field is empty.
+    'argspec' : { # The names of all the arguments
+                  args : list,
+		  # The name of the varargs (*args), if any
+                  varargs : str,
+		  # The name of the varkw (**kw), if any
+		  varkw : str,
+		  # The values (as strings) of all default arguments.  Note
+		  # that these must be matched *in reverse* with the 'args'
+		  # list above, since the first positional args have no default
+		  # value at all.
+		  func_defaults : list,
+		  },
+
     # For instances, provide the constructor signature (the definition of
     # the __init__ method):
     'init_definition' : str,
@@ -406,6 +526,7 b' Message type: ``history_reply``::'
       # respectively.
       'history' : dict,
     }
+    
 Messages on the PUB/SUB socket
 ==============================
 
@@ -458,10 +579,10 b' Message type: ``pyout``::'
         # The data is typically the repr() of the object.
     'data' : str,
     
-    # The prompt number for this execution is also provided so that clients
-    # can display it, since IPython automatically creates variables called
-    # _N (for prompt N).
-    'prompt_number' : int,
+    # The counter for this execution is also provided so that clients can
+    # display it, since IPython automatically creates variables called _N (for
+    # prompt N).
+    'execution_count' : int,
     }
     
 Python errors