Merge pull request #5188 from takluyver/callbacks

New events system

IPython/core/events.py

@@ -0,0 +1,139 b''
+"""Infrastructure for registering and firing callbacks on application events.
+
+Unlike :mod:`IPython.core.hooks`, which lets end users set single functions to
+be called at specific times, or a collection of alternative methods to try,
+callbacks are designed to be used by extension authors. A number of callbacks
+can be registered for the same event without needing to be aware of one another.
+
+The functions defined in this module are no-ops indicating the names of available
+events and the arguments which will be passed to them.
+
+.. note::
+
+   This API is experimental in IPython 2.0, and may be revised in future versions.
+"""
+from __future__ import print_function
+
+class EventManager(object):
+    """Manage a collection of events and a sequence of callbacks for each.
+    
+    This is attached to :class:`~IPython.core.interactiveshell.InteractiveShell`
+    instances as an ``events`` attribute.
+    
+    .. note::
+
+       This API is experimental in IPython 2.0, and may be revised in future versions.
+    """
+    def __init__(self, shell, available_events):
+        """Initialise the :class:`CallbackManager`.
+        
+        Parameters
+        ----------
+        shell
+          The :class:`~IPython.core.interactiveshell.InteractiveShell` instance
+        available_callbacks
+          An iterable of names for callback events.
+        """
+        self.shell = shell
+        self.callbacks = {n:[] for n in available_events}
+    
+    def register(self, event, function):
+        """Register a new event callback
+        
+        Parameters
+        ----------
+        event : str
+          The event for which to register this callback.
+        function : callable
+          A function to be called on the given event. It should take the same
+          parameters as the appropriate callback prototype.
+        
+        Raises
+        ------
+        TypeError
+          If ``function`` is not callable.
+        KeyError
+          If ``event`` is not one of the known events.
+        """
+        if not callable(function):
+            raise TypeError('Need a callable, got %r' % function)
+        self.callbacks[event].append(function)
+    
+    def unregister(self, event, function):
+        """Remove a callback from the given event."""
+        self.callbacks[event].remove(function)
+    
+    def reset(self, event):
+        """Clear all callbacks for the given event."""
+        self.callbacks[event] = []
+    
+    def reset_all(self):
+        """Clear all callbacks for all events."""
+        self.callbacks = {n:[] for n in self.callbacks}
+    
+    def trigger(self, event, *args, **kwargs):
+        """Call callbacks for ``event``.
+        
+        Any additional arguments are passed to all callbacks registered for this
+        event. Exceptions raised by callbacks are caught, and a message printed.
+        """
+        for func in self.callbacks[event]:
+            try:
+                func(*args, **kwargs)
+            except Exception:
+                print("Error in callback {} (for {}):".format(func, event))
+                self.shell.showtraceback()
+
+# event_name -> prototype mapping
+available_events = {}
+
+def _define_event(callback_proto):
+    available_events[callback_proto.__name__] = callback_proto
+    return callback_proto
+
+# ------------------------------------------------------------------------------
+# Callback prototypes
+#
+# No-op functions which describe the names of available events and the
+# signatures of callbacks for those events.
+# ------------------------------------------------------------------------------
+
+@_define_event
+def pre_execute():
+    """Fires before code is executed in response to user/frontend action.
+    
+    This includes comm and widget messages and silent execution, as well as user
+    code cells."""
+    pass
+
+@_define_event
+def pre_run_cell():
+    """Fires before user-entered code runs."""
+    pass
+
+@_define_event
+def post_execute():
+    """Fires after code is executed in response to user/frontend action.
+    
+    This includes comm and widget messages and silent execution, as well as user
+    code cells."""
+    pass
+
+@_define_event
+def post_run_cell():
+    """Fires after user-entered code runs."""
+    pass
+
+@_define_event
+def shell_initialized(ip):
+    """Fires after initialisation of :class:`~IPython.core.interactiveshell.InteractiveShell`.
+    
+    This is before extensions and startup scripts are loaded, so it can only be
+    set by subclassing.
+    
+    Parameters
+    ----------
+    ip : :class:`~IPython.core.interactiveshell.InteractiveShell`
+      The newly initialised shell.
+    """
+    pass

IPython/core/tests/test_events.py

@@ -0,0 +1,46 b''
+import unittest
+try:  # Python 3.3 +
+    from unittest.mock import Mock
+except ImportError:
+    from mock import Mock
+
+from IPython.core import events
+import IPython.testing.tools as tt
+
+def ping_received():
+    pass
+
+class CallbackTests(unittest.TestCase):
+    def setUp(self):
+        self.em = events.EventManager(get_ipython(), {'ping_received': ping_received})
+    
+    def test_register_unregister(self):
+        cb = Mock()
+
+        self.em.register('ping_received', cb)        
+        self.em.trigger('ping_received')
+        self.assertEqual(cb.call_count, 1)
+        
+        self.em.unregister('ping_received', cb)
+        self.em.trigger('ping_received')
+        self.assertEqual(cb.call_count, 1)
+    
+    def test_reset(self):
+        cb = Mock()
+        self.em.register('ping_received', cb)
+        self.em.reset('ping_received')
+        self.em.trigger('ping_received')
+        assert not cb.called
+    
+    def test_reset_all(self):
+        cb = Mock()
+        self.em.register('ping_received', cb)
+        self.em.reset_all()
+        self.em.trigger('ping_received')
+        assert not cb.called
+    
+    def test_cb_error(self):
+        cb = Mock(side_effect=ValueError)
+        self.em.register('ping_received', cb)
+        with tt.AssertPrints("Error in callback"):
+            self.em.trigger('ping_received')
  No newline at end of file

docs/source/config/callbacks.rst

@@ -0,0 +1,42 b''
+=====================
+Registering callbacks
+=====================
+
+Extension code can register callbacks functions which will be called on specific
+events within the IPython code. You can see the current list of available
+callbacks, and the parameters that will be passed with each, in the callback
+prototype functions defined in :mod:`IPython.core.callbacks`.
+
+To register callbacks, use :meth:`IPython.core.events.EventManager.register`.
+For example::
+
+    class VarWatcher(object):
+        def __init__(self, ip):
+            self.shell = ip
+            self.last_x = None
+        
+        def pre_execute(self):
+            self.last_x = self.shell.user_ns.get('x', None)
+        
+        def post_execute(self):
+            if self.shell.user_ns.get('x', None) != self.last_x:
+                print("x changed!")
+
+    def load_ipython_extension(ip):
+        vw = VarWatcher(ip)
+        ip.events.register('pre_execute', vw.pre_execute)
+        ip.events.register('post_execute', vw.post_execute)
+
+.. note::
+
+   This API is experimental in IPython 2.0, and may be revised in future versions.
+
+.. seealso::
+
+   Module :mod:`IPython.core.hooks`
+     The older 'hooks' system allows end users to customise some parts of
+     IPython's behaviour.
+   
+   :doc:`inputtransforms`
+     By registering input transformers that don't change code, you can monitor
+     what is being executed.

docs/source/whatsnew/pr/callbacks.rst

@@ -0,0 +1,1 b''
+* A new callback system has been introduced. For details, see :doc:`/config/callbacks`.

.travis.yml

@@ -7,7 +7,7 b' before_install:'
     # workaround for https://github.com/travis-ci/travis-cookbooks/issues/155
     - sudo rm -rf /dev/shm && sudo ln -s /run/shm /dev/shm
     - easy_install -q pyzmq
-    - pip install jinja2 sphinx pygments tornado requests
+    - pip install jinja2 sphinx pygments tornado requests mock
     # Pierre Carrier's PPA for PhantomJS and CasperJS
     - sudo add-apt-repository -y ppa:pcarrier/ppa
     - sudo apt-get update

IPython/core/hooks.py

@@ -49,6 +49,11 b" __all__ = ['editor', 'fix_error_editor', 'synchronize_with_editor',"
            'show_in_pager','pre_prompt_hook',
            'pre_run_code_hook', 'clipboard_get']
 
+deprecated = {'pre_run_code_hook': "a callback for the 'pre_execute' or 'pre_run_cell' event",
+              'late_startup_hook': "a callback for the 'shell_initialized' event",
+              'shutdown_hook': "the atexit module",
+             }
+
 def editor(self, filename, linenum=None, wait=True):
     """Open the default editor at the given filename and linenumber.
 

IPython/core/interactiveshell.py

@@ -41,6 +41,7 b' from IPython.core import ultratb'
 from IPython.core.alias import AliasManager, AliasError
 from IPython.core.autocall import ExitAutocall
 from IPython.core.builtin_trap import BuiltinTrap
+from IPython.core.events import EventManager, available_events
 from IPython.core.compilerop import CachingCompiler, check_linecache_ipython
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.displayhook import DisplayHook
@@ -467,6 +468,7 b' class InteractiveShell(SingletonConfigurable):'
 
         self.init_syntax_highlighting()
         self.init_hooks()
+        self.init_events()
         self.init_pushd_popd_magic()
         # self.init_traceback_handlers use to be here, but we moved it below
         # because it and init_io have to come after init_readline.
@@ -510,6 +512,7 b' class InteractiveShell(SingletonConfigurable):'
         self.init_payload()
         self.init_comms()
         self.hooks.late_startup_hook()
+        self.events.trigger('shell_initialized', self)
         atexit.register(self.atexit_operations)
 
     def get_ipython(self):
@@ -785,9 +788,10 b' class InteractiveShell(SingletonConfigurable):'
         for hook_name in hooks.__all__:
             # default hooks have priority 100, i.e. low; user hooks should have
             # 0-100 priority
-            self.set_hook(hook_name,getattr(hooks,hook_name), 100)
+            self.set_hook(hook_name,getattr(hooks,hook_name), 100, _warn_deprecated=False)
 
-    def set_hook(self,name,hook, priority = 50, str_key = None, re_key = None):
+    def set_hook(self,name,hook, priority=50, str_key=None, re_key=None,
+                 _warn_deprecated=True):
         """set_hook(name,hook) -> sets an internal IPython hook.
 
         IPython exposes some of its internal API as user-modifiable hooks.  By
@@ -816,6 +820,11 b' class InteractiveShell(SingletonConfigurable):'
         if name not in IPython.core.hooks.__all__:
             print("Warning! Hook '%s' is not one of %s" % \
                   (name, IPython.core.hooks.__all__ ))
+
+        if _warn_deprecated and (name in IPython.core.hooks.deprecated):
+            alternative = IPython.core.hooks.deprecated[name]
+            warn("Hook {} is deprecated. Use {} instead.".format(name, alternative))
+
         if not dp:
             dp = IPython.core.hooks.CommandChainDispatcher()
 
@@ -827,12 +836,21 b' class InteractiveShell(SingletonConfigurable):'
 
         setattr(self.hooks,name, dp)
 
+    #-------------------------------------------------------------------------
+    # Things related to events
+    #-------------------------------------------------------------------------
+
+    def init_events(self):
+        self.events = EventManager(self, available_events)
+
     def register_post_execute(self, func):
-        """Register a function for calling after code execution.
+        """DEPRECATED: Use ip.events.register('post_run_cell', func)
+        
+        Register a function for calling after code execution.
         """
-        if not callable(func):
-            raise ValueError('argument %s must be callable' % func)
-        self._post_execute[func] = True
+        warn("ip.register_post_execute is deprecated, use "
+             "ip.events.register('post_run_cell', func) instead.")
+        self.events.register('post_run_cell', func)
     
     #-------------------------------------------------------------------------
     # Things related to the "main" module
@@ -2649,6 +2667,10 b' class InteractiveShell(SingletonConfigurable):'
         if silent:
             store_history = False
 
+        self.events.trigger('pre_execute')
+        if not silent:
+            self.events.trigger('pre_run_cell')
+
         # If any of our input transformation (input_transformer_manager or
         # prefilter_manager) raises an exception, we store it in this variable
         # so that we can display the error after logging the input and storing
@@ -2717,28 +2739,10 b' class InteractiveShell(SingletonConfigurable):'
                 interactivity = "none" if silent else self.ast_node_interactivity
                 self.run_ast_nodes(code_ast.body, cell_name,
                                    interactivity=interactivity, compiler=compiler)
-
-                # Execute any registered post-execution functions.
-                # unless we are silent
-                post_exec = [] if silent else iteritems(self._post_execute)
-
-                for func, status in post_exec:
-                    if self.disable_failing_post_execute and not status:
-                        continue
-                    try:
-                        func()
-                    except KeyboardInterrupt:
-                        print("\nKeyboardInterrupt", file=io.stderr)
-                    except Exception:
-                        # register as failing:
-                        self._post_execute[func] = False
-                        self.showtraceback()
-                        print('\n'.join([
-                            "post-execution function %r produced an error." % func,
-                            "If this problem persists, you can disable failing post-exec functions with:",
-                            "",
-                            "    get_ipython().disable_failing_post_execute = True"
-                        ]), file=io.stderr)
+                
+                self.events.trigger('post_execute')
+                if not silent:
+                    self.events.trigger('post_run_cell')
 
         if store_history:
             # Write output to the database. Does nothing unless

IPython/core/pylabtools.py

@@ -353,7 +353,7 b' def configure_inline_support(shell, backend):'
 
     if backend == backends['inline']:
         from IPython.kernel.zmq.pylab.backend_inline import flush_figures
-        shell.register_post_execute(flush_figures)
+        shell.events.register('post_execute', flush_figures)
 
         # Save rcParams that will be overwrittern
         shell._saved_rcParams = dict()
@@ -363,8 +363,10 b' def configure_inline_support(shell, backend):'
         pyplot.rcParams.update(cfg.rc)
     else:
         from IPython.kernel.zmq.pylab.backend_inline import flush_figures
-        if flush_figures in shell._post_execute:
-            shell._post_execute.pop(flush_figures)
+        try:
+            shell.events.unregister('post_execute', flush_figures)
+        except ValueError:
+            pass
         if hasattr(shell, '_saved_rcParams'):
             pyplot.rcParams.update(shell._saved_rcParams)
             del shell._saved_rcParams

IPython/core/tests/test_interactiveshell.py

@@ -27,6 +27,10 b' import shutil'
 import sys
 import tempfile
 import unittest
+try:
+    from unittest import mock
+except ImportError:
+    import mock
 from os.path import join
 
 # third-party
@@ -277,21 +281,32 b' class InteractiveShellTestCase(unittest.TestCase):'
         # ZeroDivisionError
         self.assertEqual(ip.var_expand(u"{1/0}"), u"{1/0}")
     
-    def test_silent_nopostexec(self):
-        """run_cell(silent=True) doesn't invoke post-exec funcs"""
-        d = dict(called=False)
-        def set_called():
-            d['called'] = True
+    def test_silent_postexec(self):
+        """run_cell(silent=True) doesn't invoke pre/post_run_cell callbacks"""
+        pre_explicit = mock.Mock()
+        pre_always = mock.Mock()
+        post_explicit = mock.Mock()
+        post_always = mock.Mock()
         
-        ip.register_post_execute(set_called)
-        ip.run_cell("1", silent=True)
-        self.assertFalse(d['called'])
-        # double-check that non-silent exec did what we expected
-        # silent to avoid
-        ip.run_cell("1")
-        self.assertTrue(d['called'])
-        # remove post-exec
-        ip._post_execute.pop(set_called)
+        ip.events.register('pre_run_cell', pre_explicit)
+        ip.events.register('pre_execute', pre_always)
+        ip.events.register('post_run_cell', post_explicit)
+        ip.events.register('post_execute', post_always)
+        
+        try:
+            ip.run_cell("1", silent=True)
+            assert pre_always.called
+            assert not pre_explicit.called
+            assert post_always.called
+            assert not post_explicit.called
+            # double-check that non-silent exec did what we expected
+            # silent to avoid
+            ip.run_cell("1")
+            assert pre_explicit.called
+            assert post_explicit.called
+        finally:
+            # remove post-exec
+            ip.events.reset_all()
     
     def test_silent_noadvance(self):
         """run_cell(silent=True) doesn't advance execution_count"""

IPython/extensions/autoreload.py

@@ -392,7 +392,6 b' def superreload(module, reload=reload, old_objects={}):'
 # IPython connectivity
 #------------------------------------------------------------------------------
 
-from IPython.core.hooks import TryNext
 from IPython.core.magic import Magics, magics_class, line_magic
 
 @magics_class
@@ -491,17 +490,16 b' class AutoreloadMagics(Magics):'
             # Inject module to user namespace
             self.shell.push({top_name: top_module})
 
-    def pre_run_code_hook(self, ip):
-        if not self._reloader.enabled:
-            raise TryNext
-        try:
-            self._reloader.check()
-        except:
-            pass
+    def pre_run_cell(self):
+        if self._reloader.enabled:
+            try:
+                self._reloader.check()
+            except:
+                pass
 
 
 def load_ipython_extension(ip):
     """Load the extension in IPython."""
     auto_reload = AutoreloadMagics(ip)
     ip.register_magics(auto_reload)
-    ip.set_hook('pre_run_code_hook', auto_reload.pre_run_code_hook)
+    ip.events.register('pre_run_cell', auto_reload.pre_run_cell)

IPython/extensions/tests/test_autoreload.py

@@ -23,7 +23,7 b' import nose.tools as nt'
 import IPython.testing.tools as tt
 
 from IPython.extensions.autoreload import AutoreloadMagics
-from IPython.core.hooks import TryNext
+from IPython.core.events import EventManager, pre_run_cell
 from IPython.utils.py3compat import PY3
 
 if PY3:
@@ -41,15 +41,14 b' class FakeShell(object):'
 
     def __init__(self):
         self.ns = {}
+        self.events = EventManager(self, {'pre_run_cell', pre_run_cell})
         self.auto_magics = AutoreloadMagics(shell=self)
+        self.events.register('pre_run_cell', self.auto_magics.pre_run_cell)
 
     register_magics = set_hook = noop
 
     def run_code(self, code):
-        try:
-            self.auto_magics.pre_run_code_hook(self)
-        except TryNext:
-            pass
+        self.events.trigger('pre_run_cell')
         exec(code, self.ns)
 
     def push(self, items):

IPython/kernel/comm/comm.py

@@ -134,7 +134,9 b' class Comm(LoggingConfigurable):'
         """Handle a comm_msg message"""
         self.log.debug("handle_msg[%s](%s)", self.comm_id, msg)
         if self._msg_callback:
+            self.shell.events.trigger('pre_execute')
             self._msg_callback(msg)
+            self.shell.events.trigger('post_execute')
 
 
 __all__ = ['Comm']

IPython/utils/signatures.py

@@ -1,7 +1,7 b''
-"""Function signature objects for callables
+"""Function signature objects for callables.
 
 Back port of Python 3.3's function signature tools from the inspect module,
-modified to be compatible with Python 2.6, 2.7 and 3.2+.
+modified to be compatible with Python 2.7 and 3.2+.
 """
 
 #-----------------------------------------------------------------------------
@@ -346,20 +346,20 b' class Parameter(object):'
 
 
 class BoundArguments(object):
-    '''Result of `Signature.bind` call.  Holds the mapping of arguments
+    '''Result of :meth:`Signature.bind` call.  Holds the mapping of arguments
     to the function's parameters.
 
     Has the following public attributes:
 
-    * arguments : OrderedDict
-        An ordered mutable mapping of parameters' names to arguments' values.
-        Does not contain arguments' default values.
-    * signature : Signature
-        The Signature object that created this instance.
-    * args : tuple
-        Tuple of positional arguments values.
-    * kwargs : dict
-        Dict of keyword arguments values.
+    arguments : :class:`collections.OrderedDict`
+      An ordered mutable mapping of parameters' names to arguments' values.
+      Does not contain arguments' default values.
+    signature : :class:`Signature`
+      The Signature object that created this instance.
+    args : tuple
+      Tuple of positional arguments values.
+    kwargs : dict
+      Dict of keyword arguments values.
     '''
 
     def __init__(self, signature, arguments):
@@ -447,22 +447,16 b' class Signature(object):'
     It stores a Parameter object for each parameter accepted by the
     function, as well as information specific to the function itself.
 
-    A Signature object has the following public attributes and methods:
-
-    * parameters : OrderedDict
-        An ordered mapping of parameters' names to the corresponding
-        Parameter objects (keyword-only arguments are in the same order
-        as listed in `code.co_varnames`).
-    * return_annotation : object
-        The annotation for the return type of the function if specified.
-        If the function has no annotation for its return type, this
-        attribute is not set.
-    * bind(*args, **kwargs) -> BoundArguments
-        Creates a mapping from positional and keyword arguments to
-        parameters.
-    * bind_partial(*args, **kwargs) -> BoundArguments
-        Creates a partial mapping from positional and keyword arguments
-        to parameters (simulating 'functools.partial' behavior.)
+    A Signature object has the following public attributes:
+
+    parameters : :class:`collections.OrderedDict`
+      An ordered mapping of parameters' names to the corresponding
+      Parameter objects (keyword-only arguments are in the same order
+      as listed in `code.co_varnames`).
+    return_annotation
+      The annotation for the return type of the function if specified.
+      If the function has no annotation for its return type, this
+      attribute is not set.
     '''
 
     __slots__ = ('_return_annotation', '_parameters')
@@ -775,16 +769,16 b' class Signature(object):'
         return self._bound_arguments_cls(self, arguments)
 
     def bind(self, *args, **kwargs):
-        '''Get a BoundArguments object, that maps the passed `args`
-        and `kwargs` to the function's signature.  Raises `TypeError`
+        '''Get a :class:`BoundArguments` object, that maps the passed `args`
+        and `kwargs` to the function's signature.  Raises :exc:`TypeError`
         if the passed arguments can not be bound.
         '''
         return self._bind(args, kwargs)
 
     def bind_partial(self, *args, **kwargs):
-        '''Get a BoundArguments object, that partially maps the
+        '''Get a :class:`BoundArguments` object, that partially maps the
         passed `args` and `kwargs` to the function's signature.
-        Raises `TypeError` if the passed arguments can not be bound.
+        Raises :exc:`TypeError` if the passed arguments can not be bound.
         '''
         return self._bind(args, kwargs, partial=True)
 

docs/source/config/index.rst

@@ -28,3 +28,4 b' Extending and integrating with IPython'
    extensions/index
    integrating
    inputtransforms
+   callbacks

setup.py

@@ -273,10 +273,14 b' extras_require = dict('
     notebook = ['tornado>=3.1', 'pyzmq>=2.1.11', 'jinja2'],
     nbconvert = ['pygments', 'jinja2', 'Sphinx>=0.3']
 )
+if sys.version_info < (3, 3):
+    extras_require['test'].append('mock')
+
 everything = set()
 for deps in extras_require.values():
     everything.update(deps)
 extras_require['all'] = everything
+
 install_requires = []
 if sys.platform == 'darwin':
     if any(arg.startswith('bdist') for arg in sys.argv) or not setupext.check_for_readline():

setupbase.py

@@ -639,10 +639,11 b' def get_bdist_wheel():'
                     if found:
                         lis.pop(idx)
                 
-                for pkg in ("gnureadline", "pyreadline"):
+                for pkg in ("gnureadline", "pyreadline", "mock"):
                     _remove_startswith(requires, pkg)
                 requires.append("gnureadline; sys.platform == 'darwin' and platform.python_implementation == 'CPython'")
                 requires.append("pyreadline (>=2.0); sys.platform == 'win32' and platform.python_implementation == 'CPython'")
+                requires.append("mock; extra == 'test' and python_version < '3.3'")
                 for r in requires:
                     pkg_info['Requires-Dist'] = r
                 write_pkg_info(metadata_path, pkg_info)

tox.ini

@@ -9,6 +9,7 b' envlist = py27, py33'
 [testenv]
 deps = 
   nose
+  mock
   tornado
   jinja2
   sphinx