Fix references to xrange

IPython/kernel/channels.py

@@ -1,649 +1,649 b''
 """Base classes to manage a Client's interaction with a running kernel
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2013  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import absolute_import
 
 # Standard library imports
 import atexit
 import errno
 from threading import Thread
 import time
 
 import zmq
 # import ZMQError in top-level namespace, to avoid ugly attribute-error messages
 # during garbage collection of threads at exit:
 from zmq import ZMQError
 from zmq.eventloop import ioloop, zmqstream
 
 # Local imports
 from .channelsabc import (
     ShellChannelABC, IOPubChannelABC,
     HBChannelABC, StdInChannelABC,
 )
 from IPython.utils.py3compat import string_types
 
 #-----------------------------------------------------------------------------
 # Constants and exceptions
 #-----------------------------------------------------------------------------
 
 class InvalidPortNumber(Exception):
     pass
 
 #-----------------------------------------------------------------------------
 # Utility functions
 #-----------------------------------------------------------------------------
 
 # some utilities to validate message structure, these might get moved elsewhere
 # if they prove to have more generic utility
 
 def validate_string_list(lst):
     """Validate that the input is a list of strings.
 
     Raises ValueError if not."""
     if not isinstance(lst, list):
         raise ValueError('input %r must be a list' % lst)
     for x in lst:
         if not isinstance(x, string_types):
             raise ValueError('element %r in list must be a string' % x)
 
 
 def validate_string_dict(dct):
     """Validate that the input is a dict with string keys and values.
 
     Raises ValueError if not."""
     for k,v in dct.iteritems():
         if not isinstance(k, string_types):
             raise ValueError('key %r in dict must be a string' % k)
         if not isinstance(v, string_types):
             raise ValueError('value %r in dict must be a string' % v)
 
 
 #-----------------------------------------------------------------------------
 # ZMQ Socket Channel classes
 #-----------------------------------------------------------------------------
 
 class ZMQSocketChannel(Thread):
     """The base class for the channels that use ZMQ sockets."""
     context = None
     session = None
     socket = None
     ioloop = None
     stream = None
     _address = None
     _exiting = False
     proxy_methods = []
 
     def __init__(self, context, session, address):
         """Create a channel.
 
         Parameters
         ----------
         context : :class:`zmq.Context`
             The ZMQ context to use.
         session : :class:`session.Session`
             The session to use.
         address : zmq url
             Standard (ip, port) tuple that the kernel is listening on.
         """
         super(ZMQSocketChannel, self).__init__()
         self.daemon = True
 
         self.context = context
         self.session = session
         if isinstance(address, tuple):
             if address[1] == 0:
                 message = 'The port number for a channel cannot be 0.'
                 raise InvalidPortNumber(message)
             address = "tcp://%s:%i" % address
         self._address = address
         atexit.register(self._notice_exit)
 
     def _notice_exit(self):
         self._exiting = True
 
     def _run_loop(self):
         """Run my loop, ignoring EINTR events in the poller"""
         while True:
             try:
                 self.ioloop.start()
             except ZMQError as e:
                 if e.errno == errno.EINTR:
                     continue
                 else:
                     raise
             except Exception:
                 if self._exiting:
                     break
                 else:
                     raise
             else:
                 break
 
     def stop(self):
         """Stop the channel's event loop and join its thread.
 
         This calls :method:`Thread.join` and returns when the thread
         terminates. :class:`RuntimeError` will be raised if
         :method:`self.start` is called again.
         """
         self.join()
 
     @property
     def address(self):
         """Get the channel's address as a zmq url string.
 
         These URLS have the form: 'tcp://127.0.0.1:5555'.
         """
         return self._address
 
     def _queue_send(self, msg):
         """Queue a message to be sent from the IOLoop's thread.
 
         Parameters
         ----------
         msg : message to send
 
         This is threadsafe, as it uses IOLoop.add_callback to give the loop's
         thread control of the action.
         """
         def thread_send():
             self.session.send(self.stream, msg)
         self.ioloop.add_callback(thread_send)
 
     def _handle_recv(self, msg):
         """Callback for stream.on_recv.
 
         Unpacks message, and calls handlers with it.
         """
         ident,smsg = self.session.feed_identities(msg)
         self.call_handlers(self.session.unserialize(smsg))
 
 
 
 class ShellChannel(ZMQSocketChannel):
     """The shell channel for issuing request/replies to the kernel."""
 
     command_queue = None
     # flag for whether execute requests should be allowed to call raw_input:
     allow_stdin = True
     proxy_methods = [
         'execute',
         'complete',
         'object_info',
         'history',
         'kernel_info',
         'shutdown',
     ]
 
     def __init__(self, context, session, address):
         super(ShellChannel, self).__init__(context, session, address)
         self.ioloop = ioloop.IOLoop()
 
     def run(self):
         """The thread's main activity.  Call start() instead."""
         self.socket = self.context.socket(zmq.DEALER)
         self.socket.setsockopt(zmq.IDENTITY, self.session.bsession)
         self.socket.connect(self.address)
         self.stream = zmqstream.ZMQStream(self.socket, self.ioloop)
         self.stream.on_recv(self._handle_recv)
         self._run_loop()
         try:
             self.socket.close()
         except:
             pass
 
     def stop(self):
         """Stop the channel's event loop and join its thread."""
         self.ioloop.stop()
         super(ShellChannel, self).stop()
 
     def call_handlers(self, msg):
         """This method is called in the ioloop thread when a message arrives.
 
         Subclasses should override this method to handle incoming messages.
         It is important to remember that this method is called in the thread
         so that some logic must be done to ensure that the application level
         handlers are called in the application thread.
         """
         raise NotImplementedError('call_handlers must be defined in a subclass.')
 
     def execute(self, code, silent=False, store_history=True,
                 user_variables=None, user_expressions=None, allow_stdin=None):
         """Execute code in the kernel.
 
         Parameters
         ----------
         code : str
             A string of Python code.
 
         silent : bool, optional (default False)
             If set, the kernel will execute the code as quietly possible, and
             will force store_history to be False.
 
         store_history : bool, optional (default True)
             If set, the kernel will store command history.  This is forced
             to be False if silent is True.
 
         user_variables : list, optional
             A list of variable names to pull from the user's namespace.  They
             will come back as a dict with these names as keys and their
             :func:`repr` as values.
 
         user_expressions : dict, optional
             A dict mapping names to expressions to be evaluated in the user's
             dict. The expression values are returned as strings formatted using
             :func:`repr`.
 
         allow_stdin : bool, optional (default self.allow_stdin)
             Flag for whether the kernel can send stdin requests to frontends.
 
             Some frontends (e.g. the Notebook) do not support stdin requests.
             If raw_input is called from code executed from such a frontend, a
             StdinNotImplementedError will be raised.
 
         Returns
         -------
         The msg_id of the message sent.
         """
         if user_variables is None:
             user_variables = []
         if user_expressions is None:
             user_expressions = {}
         if allow_stdin is None:
             allow_stdin = self.allow_stdin
 
 
         # Don't waste network traffic if inputs are invalid
         if not isinstance(code, string_types):
             raise ValueError('code %r must be a string' % code)
         validate_string_list(user_variables)
         validate_string_dict(user_expressions)
 
         # Create class for content/msg creation. Related to, but possibly
         # not in Session.
         content = dict(code=code, silent=silent, store_history=store_history,
                        user_variables=user_variables,
                        user_expressions=user_expressions,
                        allow_stdin=allow_stdin,
                        )
         msg = self.session.msg('execute_request', content)
         self._queue_send(msg)
         return msg['header']['msg_id']
 
     def complete(self, text, line, cursor_pos, block=None):
         """Tab complete text in the kernel's namespace.
 
         Parameters
         ----------
         text : str
             The text to complete.
         line : str
             The full line of text that is the surrounding context for the
             text to complete.
         cursor_pos : int
             The position of the cursor in the line where the completion was
             requested.
         block : str, optional
             The full block of code in which the completion is being requested.
 
         Returns
         -------
         The msg_id of the message sent.
         """
         content = dict(text=text, line=line, block=block, cursor_pos=cursor_pos)
         msg = self.session.msg('complete_request', content)
         self._queue_send(msg)
         return msg['header']['msg_id']
 
     def object_info(self, oname, detail_level=0):
         """Get metadata information about an object in the kernel's namespace.
 
         Parameters
         ----------
         oname : str
             A string specifying the object name.
         detail_level : int, optional
             The level of detail for the introspection (0-2)
 
         Returns
         -------
         The msg_id of the message sent.
         """
         content = dict(oname=oname, detail_level=detail_level)
         msg = self.session.msg('object_info_request', content)
         self._queue_send(msg)
         return msg['header']['msg_id']
 
     def history(self, raw=True, output=False, hist_access_type='range', **kwargs):
         """Get entries from the kernel's history list.
 
         Parameters
         ----------
         raw : bool
             If True, return the raw input.
         output : bool
             If True, then return the output as well.
         hist_access_type : str
             'range' (fill in session, start and stop params), 'tail' (fill in n)
              or 'search' (fill in pattern param).
 
         session : int
             For a range request, the session from which to get lines. Session
             numbers are positive integers; negative ones count back from the
             current session.
         start : int
             The first line number of a history range.
         stop : int
             The final (excluded) line number of a history range.
 
         n : int
             The number of lines of history to get for a tail request.
 
         pattern : str
             The glob-syntax pattern for a search request.
 
         Returns
         -------
         The msg_id of the message sent.
         """
         content = dict(raw=raw, output=output, hist_access_type=hist_access_type,
                                                                     **kwargs)
         msg = self.session.msg('history_request', content)
         self._queue_send(msg)
         return msg['header']['msg_id']
 
     def kernel_info(self):
         """Request kernel info."""
         msg = self.session.msg('kernel_info_request')
         self._queue_send(msg)
         return msg['header']['msg_id']
 
     def shutdown(self, restart=False):
         """Request an immediate kernel shutdown.
 
         Upon receipt of the (empty) reply, client code can safely assume that
         the kernel has shut down and it's safe to forcefully terminate it if
         it's still alive.
 
         The kernel will send the reply via a function registered with Python's
         atexit module, ensuring it's truly done as the kernel is done with all
         normal operation.
         """
         # Send quit message to kernel. Once we implement kernel-side setattr,
         # this should probably be done that way, but for now this will do.
         msg = self.session.msg('shutdown_request', {'restart':restart})
         self._queue_send(msg)
         return msg['header']['msg_id']
 
 
 
 class IOPubChannel(ZMQSocketChannel):
     """The iopub channel which listens for messages that the kernel publishes.
 
     This channel is where all output is published to frontends.
     """
 
     def __init__(self, context, session, address):
         super(IOPubChannel, self).__init__(context, session, address)
         self.ioloop = ioloop.IOLoop()
 
     def run(self):
         """The thread's main activity.  Call start() instead."""
         self.socket = self.context.socket(zmq.SUB)
         self.socket.setsockopt(zmq.SUBSCRIBE,b'')
         self.socket.setsockopt(zmq.IDENTITY, self.session.bsession)
         self.socket.connect(self.address)
         self.stream = zmqstream.ZMQStream(self.socket, self.ioloop)
         self.stream.on_recv(self._handle_recv)
         self._run_loop()
         try:
             self.socket.close()
         except:
             pass
 
     def stop(self):
         """Stop the channel's event loop and join its thread."""
         self.ioloop.stop()
         super(IOPubChannel, self).stop()
 
     def call_handlers(self, msg):
         """This method is called in the ioloop thread when a message arrives.
 
         Subclasses should override this method to handle incoming messages.
         It is important to remember that this method is called in the thread
         so that some logic must be done to ensure that the application leve
         handlers are called in the application thread.
         """
         raise NotImplementedError('call_handlers must be defined in a subclass.')
 
     def flush(self, timeout=1.0):
         """Immediately processes all pending messages on the iopub channel.
 
         Callers should use this method to ensure that :method:`call_handlers`
         has been called for all messages that have been received on the
         0MQ SUB socket of this channel.
 
         This method is thread safe.
 
         Parameters
         ----------
         timeout : float, optional
             The maximum amount of time to spend flushing, in seconds. The
             default is one second.
         """
         # We do the IOLoop callback process twice to ensure that the IOLoop
         # gets to perform at least one full poll.
         stop_time = time.time() + timeout
-        for i in xrange(2):
+        for i in range(2):
             self._flushed = False
             self.ioloop.add_callback(self._flush)
             while not self._flushed and time.time() < stop_time:
                 time.sleep(0.01)
 
     def _flush(self):
         """Callback for :method:`self.flush`."""
         self.stream.flush()
         self._flushed = True
 
 
 class StdInChannel(ZMQSocketChannel):
     """The stdin channel to handle raw_input requests that the kernel makes."""
 
     msg_queue = None
     proxy_methods = ['input']
 
     def __init__(self, context, session, address):
         super(StdInChannel, self).__init__(context, session, address)
         self.ioloop = ioloop.IOLoop()
 
     def run(self):
         """The thread's main activity.  Call start() instead."""
         self.socket = self.context.socket(zmq.DEALER)
         self.socket.setsockopt(zmq.IDENTITY, self.session.bsession)
         self.socket.connect(self.address)
         self.stream = zmqstream.ZMQStream(self.socket, self.ioloop)
         self.stream.on_recv(self._handle_recv)
         self._run_loop()
         try:
             self.socket.close()
         except:
             pass
 
     def stop(self):
         """Stop the channel's event loop and join its thread."""
         self.ioloop.stop()
         super(StdInChannel, self).stop()
 
     def call_handlers(self, msg):
         """This method is called in the ioloop thread when a message arrives.
 
         Subclasses should override this method to handle incoming messages.
         It is important to remember that this method is called in the thread
         so that some logic must be done to ensure that the application leve
         handlers are called in the application thread.
         """
         raise NotImplementedError('call_handlers must be defined in a subclass.')
 
     def input(self, string):
         """Send a string of raw input to the kernel."""
         content = dict(value=string)
         msg = self.session.msg('input_reply', content)
         self._queue_send(msg)
 
 
 class HBChannel(ZMQSocketChannel):
     """The heartbeat channel which monitors the kernel heartbeat.
 
     Note that the heartbeat channel is paused by default. As long as you start
     this channel, the kernel manager will ensure that it is paused and un-paused
     as appropriate.
     """
 
     time_to_dead = 3.0
     socket = None
     poller = None
     _running = None
     _pause = None
     _beating = None
 
     def __init__(self, context, session, address):
         super(HBChannel, self).__init__(context, session, address)
         self._running = False
         self._pause =True
         self.poller = zmq.Poller()
 
     def _create_socket(self):
         if self.socket is not None:
             # close previous socket, before opening a new one
             self.poller.unregister(self.socket)
             self.socket.close()
         self.socket = self.context.socket(zmq.REQ)
         self.socket.setsockopt(zmq.LINGER, 0)
         self.socket.connect(self.address)
 
         self.poller.register(self.socket, zmq.POLLIN)
 
     def _poll(self, start_time):
         """poll for heartbeat replies until we reach self.time_to_dead.
 
         Ignores interrupts, and returns the result of poll(), which
         will be an empty list if no messages arrived before the timeout,
         or the event tuple if there is a message to receive.
         """
 
         until_dead = self.time_to_dead - (time.time() - start_time)
         # ensure poll at least once
         until_dead = max(until_dead, 1e-3)
         events = []
         while True:
             try:
                 events = self.poller.poll(1000 * until_dead)
             except ZMQError as e:
                 if e.errno == errno.EINTR:
                     # ignore interrupts during heartbeat
                     # this may never actually happen
                     until_dead = self.time_to_dead - (time.time() - start_time)
                     until_dead = max(until_dead, 1e-3)
                     pass
                 else:
                     raise
             except Exception:
                 if self._exiting:
                     break
                 else:
                     raise
             else:
                 break
         return events
 
     def run(self):
         """The thread's main activity.  Call start() instead."""
         self._create_socket()
         self._running = True
         self._beating = True
 
         while self._running:
             if self._pause:
                 # just sleep, and skip the rest of the loop
                 time.sleep(self.time_to_dead)
                 continue
 
             since_last_heartbeat = 0.0
             # io.rprint('Ping from HB channel') # dbg
             # no need to catch EFSM here, because the previous event was
             # either a recv or connect, which cannot be followed by EFSM
             self.socket.send(b'ping')
             request_time = time.time()
             ready = self._poll(request_time)
             if ready:
                 self._beating = True
                 # the poll above guarantees we have something to recv
                 self.socket.recv()
                 # sleep the remainder of the cycle
                 remainder = self.time_to_dead - (time.time() - request_time)
                 if remainder > 0:
                     time.sleep(remainder)
                 continue
             else:
                 # nothing was received within the time limit, signal heart failure
                 self._beating = False
                 since_last_heartbeat = time.time() - request_time
                 self.call_handlers(since_last_heartbeat)
                 # and close/reopen the socket, because the REQ/REP cycle has been broken
                 self._create_socket()
                 continue
         try:
             self.socket.close()
         except:
             pass
 
     def pause(self):
         """Pause the heartbeat."""
         self._pause = True
 
     def unpause(self):
         """Unpause the heartbeat."""
         self._pause = False
 
     def is_beating(self):
         """Is the heartbeat running and responsive (and not paused)."""
         if self.is_alive() and not self._pause and self._beating:
             return True
         else:
             return False
 
     def stop(self):
         """Stop the channel's event loop and join its thread."""
         self._running = False
         super(HBChannel, self).stop()
 
     def call_handlers(self, since_last_heartbeat):
         """This method is called in the ioloop thread when a message arrives.
 
         Subclasses should override this method to handle incoming messages.
         It is important to remember that this method is called in the thread
         so that some logic must be done to ensure that the application level
         handlers are called in the application thread.
         """
         raise NotImplementedError('call_handlers must be defined in a subclass.')
 
 
 #---------------------------------------------------------------------#-----------------------------------------------------------------------------
 # ABC Registration
 #-----------------------------------------------------------------------------
 
 ShellChannelABC.register(ShellChannel)
 IOPubChannelABC.register(IOPubChannel)
 HBChannelABC.register(HBChannel)
 StdInChannelABC.register(StdInChannel)

IPython/lib/deepreload.py

@@ -1,339 +1,339 b''
 # -*- coding: utf-8 -*-
 """
 A module to change reload() so that it acts recursively.
 To enable it type::
 
     import __builtin__, deepreload
     __builtin__.reload = deepreload.reload
 
 You can then disable it with::
 
     __builtin__.reload = deepreload.original_reload
 
 Alternatively, you can add a dreload builtin alongside normal reload with::
 
     __builtin__.dreload = deepreload.reload
 
 This code is almost entirely based on knee.py, which is a Python
 re-implementation of hierarchical module import.
 """
 from __future__ import print_function
 #*****************************************************************************
 #       Copyright (C) 2001 Nathaniel Gray <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from contextlib import contextmanager
 import imp
 import sys
 
 from types import ModuleType
 from warnings import warn
 
 from IPython.utils.py3compat import builtin_mod, builtin_mod_name
 
 original_import = builtin_mod.__import__
 
 @contextmanager
 def replace_import_hook(new_import):
     saved_import = builtin_mod.__import__
     builtin_mod.__import__ = new_import
     try:
         yield
     finally:
         builtin_mod.__import__ = saved_import
 
 def get_parent(globals, level):
     """
     parent, name = get_parent(globals, level)
 
     Return the package that an import is being performed in.  If globals comes
     from the module foo.bar.bat (not itself a package), this returns the
     sys.modules entry for foo.bar.  If globals is from a package's __init__.py,
     the package's entry in sys.modules is returned.
 
     If globals doesn't come from a package or a module in a package, or a
     corresponding entry is not found in sys.modules, None is returned.
     """
     orig_level = level
 
     if not level or not isinstance(globals, dict):
         return None, ''
 
     pkgname = globals.get('__package__', None)
 
     if pkgname is not None:
         # __package__ is set, so use it
         if not hasattr(pkgname, 'rindex'):
             raise ValueError('__package__ set to non-string')
         if len(pkgname) == 0:
             if level > 0:
                 raise ValueError('Attempted relative import in non-package')
             return None, ''
         name = pkgname
     else:
         # __package__ not set, so figure it out and set it
         if '__name__' not in globals:
             return None, ''
         modname = globals['__name__']
 
         if '__path__' in globals:
             # __path__ is set, so modname is already the package name
             globals['__package__'] = name = modname
         else:
             # Normal module, so work out the package name if any
             lastdot = modname.rfind('.')
             if lastdot < 0 and level > 0:
                 raise ValueError("Attempted relative import in non-package")
             if lastdot < 0:
                 globals['__package__'] = None
                 return None, ''
             globals['__package__'] = name = modname[:lastdot]
 
     dot = len(name)
-    for x in xrange(level, 1, -1):
+    for x in range(level, 1, -1):
         try:
             dot = name.rindex('.', 0, dot)
         except ValueError:
             raise ValueError("attempted relative import beyond top-level "
                              "package")
     name = name[:dot]
 
     try:
         parent = sys.modules[name]
     except:
         if orig_level < 1:
             warn("Parent module '%.200s' not found while handling absolute "
                  "import" % name)
             parent = None
         else:
             raise SystemError("Parent module '%.200s' not loaded, cannot "
                               "perform relative import" % name)
 
     # We expect, but can't guarantee, if parent != None, that:
     # - parent.__name__ == name
     # - parent.__dict__ is globals
     # If this is violated...  Who cares?
     return parent, name
 
 def load_next(mod, altmod, name, buf):
     """
     mod, name, buf = load_next(mod, altmod, name, buf)
 
     altmod is either None or same as mod
     """
 
     if len(name) == 0:
         # completely empty module name should only happen in
         # 'from . import' (or '__import__("")')
         return mod, None, buf
 
     dot = name.find('.')
     if dot == 0:
         raise ValueError('Empty module name')
 
     if dot < 0:
         subname = name
         next = None
     else:
         subname = name[:dot]
         next = name[dot+1:]
 
     if buf != '':
         buf += '.'
     buf += subname
 
     result = import_submodule(mod, subname, buf)
     if result is None and mod != altmod:
         result = import_submodule(altmod, subname, subname)
         if result is not None:
             buf = subname
 
     if result is None:
         raise ImportError("No module named %.200s" % name)
 
     return result, next, buf
 
 # Need to keep track of what we've already reloaded to prevent cyclic evil
 found_now = {}
 
 def import_submodule(mod, subname, fullname):
     """m = import_submodule(mod, subname, fullname)"""
     # Require:
     # if mod == None: subname == fullname
     # else: mod.__name__ + "." + subname == fullname
 
     global found_now
     if fullname in found_now and fullname in sys.modules:
         m = sys.modules[fullname]
     else:
         print('Reloading', fullname)
         found_now[fullname] = 1
         oldm = sys.modules.get(fullname, None)
 
         if mod is None:
             path = None
         elif hasattr(mod, '__path__'):
             path = mod.__path__
         else:
             return None
 
         try:
             # This appears to be necessary on Python 3, because imp.find_module()
             # tries to import standard libraries (like io) itself, and we don't
             # want them to be processed by our deep_import_hook.
             with replace_import_hook(original_import):
                 fp, filename, stuff = imp.find_module(subname, path)
         except ImportError:
             return None
 
         try:
             m = imp.load_module(fullname, fp, filename, stuff)
         except:
             # load_module probably removed name from modules because of
             # the error.  Put back the original module object.
             if oldm:
                 sys.modules[fullname] = oldm
             raise
         finally:
             if fp: fp.close()
 
         add_submodule(mod, m, fullname, subname)
 
     return m
 
 def add_submodule(mod, submod, fullname, subname):
     """mod.{subname} = submod"""
     if mod is None:
         return #Nothing to do here.
 
     if submod is None:
         submod = sys.modules[fullname]
 
     setattr(mod, subname, submod)
 
     return
 
 def ensure_fromlist(mod, fromlist, buf, recursive):
     """Handle 'from module import a, b, c' imports."""
     if not hasattr(mod, '__path__'):
         return
     for item in fromlist:
         if not hasattr(item, 'rindex'):
             raise TypeError("Item in ``from list'' not a string")
         if item == '*':
             if recursive:
                 continue # avoid endless recursion
             try:
                 all = mod.__all__
             except AttributeError:
                 pass
             else:
                 ret = ensure_fromlist(mod, all, buf, 1)
                 if not ret:
                     return 0
         elif not hasattr(mod, item):
             import_submodule(mod, item, buf + '.' + item)
 
 def deep_import_hook(name, globals=None, locals=None, fromlist=None, level=-1):
     """Replacement for __import__()"""
     parent, buf = get_parent(globals, level)
 
     head, name, buf = load_next(parent, None if level < 0 else parent, name, buf)
 
     tail = head
     while name:
         tail, name, buf = load_next(tail, tail, name, buf)
 
     # If tail is None, both get_parent and load_next found
     # an empty module name: someone called __import__("") or
     # doctored faulty bytecode
     if tail is None:
         raise ValueError('Empty module name')
 
     if not fromlist:
         return head
 
     ensure_fromlist(tail, fromlist, buf, 0)
     return tail
 
 modules_reloading = {}
 
 def deep_reload_hook(m):
     """Replacement for reload()."""
     if not isinstance(m, ModuleType):
         raise TypeError("reload() argument must be module")
 
     name = m.__name__
 
     if name not in sys.modules:
         raise ImportError("reload(): module %.200s not in sys.modules" % name)
 
     global modules_reloading
     try:
         return modules_reloading[name]
     except:
         modules_reloading[name] = m
 
     dot = name.rfind('.')
     if dot < 0:
         subname = name
         path = None
     else:
         try:
             parent = sys.modules[name[:dot]]
         except KeyError:
             modules_reloading.clear()
             raise ImportError("reload(): parent %.200s not in sys.modules" % name[:dot])
         subname = name[dot+1:]
         path = getattr(parent, "__path__", None)
 
     try:
         # This appears to be necessary on Python 3, because imp.find_module()
         # tries to import standard libraries (like io) itself, and we don't
         # want them to be processed by our deep_import_hook.
         with replace_import_hook(original_import):
             fp, filename, stuff  = imp.find_module(subname, path)
     finally:
         modules_reloading.clear()
 
     try:
         newm = imp.load_module(name, fp, filename, stuff)
     except:
          # load_module probably removed name from modules because of
          # the error.  Put back the original module object.
         sys.modules[name] = m
         raise
     finally:
         if fp: fp.close()
 
     modules_reloading.clear()
     return newm
 
 # Save the original hooks
 try:
     original_reload = builtin_mod.reload
 except AttributeError:
     original_reload = imp.reload    # Python 3
 
 # Replacement for reload()
 def reload(module, exclude=['sys', 'os.path', builtin_mod_name, '__main__']):
     """Recursively reload all modules used in the given module.  Optionally
     takes a list of modules to exclude from reloading.  The default exclude
     list contains sys, __main__, and __builtin__, to prevent, e.g., resetting
     display, exception, and io hooks.
     """
     global found_now
     for i in exclude:
         found_now[i] = 1
     try:
         with replace_import_hook(deep_import_hook):
             return deep_reload_hook(module)
     finally:
         found_now = {}
 
 # Uncomment the following to automatically activate deep reloading whenever
 # this module is imported
 #builtin_mod.reload = reload

IPython/lib/pretty.py

@@ -1,790 +1,790 b''
 # -*- coding: utf-8 -*-
 """
 Python advanced pretty printer.  This pretty printer is intended to
 replace the old `pprint` python module which does not allow developers
 to provide their own pretty print callbacks.
 
 This module is based on ruby's `prettyprint.rb` library by `Tanaka Akira`.
 
 
 Example Usage
 -------------
 
 To directly print the representation of an object use `pprint`::
 
     from pretty import pprint
     pprint(complex_object)
 
 To get a string of the output use `pretty`::
 
     from pretty import pretty
     string = pretty(complex_object)
 
 
 Extending
 ---------
 
 The pretty library allows developers to add pretty printing rules for their
 own objects.  This process is straightforward.  All you have to do is to
 add a `_repr_pretty_` method to your object and call the methods on the
 pretty printer passed::
 
     class MyObject(object):
 
         def _repr_pretty_(self, p, cycle):
             ...
 
 Depending on the python version you want to support you have two
 possibilities.  The following list shows the python 2.5 version and the
 compatibility one.
 
 
 Here the example implementation of a `_repr_pretty_` method for a list
 subclass for python 2.5 and higher (python 2.5 requires the with statement
 __future__ import)::
 
     class MyList(list):
 
         def _repr_pretty_(self, p, cycle):
             if cycle:
                 p.text('MyList(...)')
             else:
                 with p.group(8, 'MyList([', '])'):
                     for idx, item in enumerate(self):
                         if idx:
                             p.text(',')
                             p.breakable()
                         p.pretty(item)
 
 The `cycle` parameter is `True` if pretty detected a cycle.  You *have* to
 react to that or the result is an infinite loop.  `p.text()` just adds
 non breaking text to the output, `p.breakable()` either adds a whitespace
 or breaks here.  If you pass it an argument it's used instead of the
 default space.  `p.pretty` prettyprints another object using the pretty print
 method.
 
 The first parameter to the `group` function specifies the extra indentation
 of the next line.  In this example the next item will either be not
 breaked (if the items are short enough) or aligned with the right edge of
 the opening bracked of `MyList`.
 
 If you want to support python 2.4 and lower you can use this code::
 
     class MyList(list):
 
         def _repr_pretty_(self, p, cycle):
             if cycle:
                 p.text('MyList(...)')
             else:
                 p.begin_group(8, 'MyList([')
                 for idx, item in enumerate(self):
                     if idx:
                         p.text(',')
                         p.breakable()
                     p.pretty(item)
                 p.end_group(8, '])')
 
 If you just want to indent something you can use the group function
 without open / close parameters.  Under python 2.5 you can also use this
 code::
 
     with p.indent(2):
         ...
 
 Or under python2.4 you might want to modify ``p.indentation`` by hand but
 this is rather ugly.
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.lib.pretty
    :parts: 3
 
 :copyright: 2007 by Armin Ronacher.
             Portions (c) 2009 by Robert Kern.
 :license: BSD License.
 """
 from __future__ import print_function
 from contextlib import contextmanager
 import sys
 import types
 import re
 import datetime
 from io import StringIO
 from collections import deque
 
 
 __all__ = ['pretty', 'pprint', 'PrettyPrinter', 'RepresentationPrinter',
     'for_type', 'for_type_by_name']
 
 
 _re_pattern_type = type(re.compile(''))
 
 
 def pretty(obj, verbose=False, max_width=79, newline='\n'):
     """
     Pretty print the object's representation.
     """
     stream = StringIO()
     printer = RepresentationPrinter(stream, verbose, max_width, newline)
     printer.pretty(obj)
     printer.flush()
     return stream.getvalue()
 
 
 def pprint(obj, verbose=False, max_width=79, newline='\n'):
     """
     Like `pretty` but print to stdout.
     """
     printer = RepresentationPrinter(sys.stdout, verbose, max_width, newline)
     printer.pretty(obj)
     printer.flush()
     sys.stdout.write(newline)
     sys.stdout.flush()
 
 class _PrettyPrinterBase(object):
 
     @contextmanager
     def indent(self, indent):
         """with statement support for indenting/dedenting."""
         self.indentation += indent
         try:
             yield
         finally:
             self.indentation -= indent
 
     @contextmanager
     def group(self, indent=0, open='', close=''):
         """like begin_group / end_group but for the with statement."""
         self.begin_group(indent, open)
         try:
             yield
         finally:
             self.end_group(indent, close)
 
 class PrettyPrinter(_PrettyPrinterBase):
     """
     Baseclass for the `RepresentationPrinter` prettyprinter that is used to
     generate pretty reprs of objects.  Contrary to the `RepresentationPrinter`
     this printer knows nothing about the default pprinters or the `_repr_pretty_`
     callback method.
     """
 
     def __init__(self, output, max_width=79, newline='\n'):
         self.output = output
         self.max_width = max_width
         self.newline = newline
         self.output_width = 0
         self.buffer_width = 0
         self.buffer = deque()
 
         root_group = Group(0)
         self.group_stack = [root_group]
         self.group_queue = GroupQueue(root_group)
         self.indentation = 0
 
     def _break_outer_groups(self):
         while self.max_width < self.output_width + self.buffer_width:
             group = self.group_queue.deq()
             if not group:
                 return
             while group.breakables:
                 x = self.buffer.popleft()
                 self.output_width = x.output(self.output, self.output_width)
                 self.buffer_width -= x.width
             while self.buffer and isinstance(self.buffer[0], Text):
                 x = self.buffer.popleft()
                 self.output_width = x.output(self.output, self.output_width)
                 self.buffer_width -= x.width
 
     def text(self, obj):
         """Add literal text to the output."""
         width = len(obj)
         if self.buffer:
             text = self.buffer[-1]
             if not isinstance(text, Text):
                 text = Text()
                 self.buffer.append(text)
             text.add(obj, width)
             self.buffer_width += width
             self._break_outer_groups()
         else:
             self.output.write(obj)
             self.output_width += width
 
     def breakable(self, sep=' '):
         """
         Add a breakable separator to the output.  This does not mean that it
         will automatically break here.  If no breaking on this position takes
         place the `sep` is inserted which default to one space.
         """
         width = len(sep)
         group = self.group_stack[-1]
         if group.want_break:
             self.flush()
             self.output.write(self.newline)
             self.output.write(' ' * self.indentation)
             self.output_width = self.indentation
             self.buffer_width = 0
         else:
             self.buffer.append(Breakable(sep, width, self))
             self.buffer_width += width
             self._break_outer_groups()
             
     def break_(self):
         """
         Explicitly insert a newline into the output, maintaining correct indentation.
         """
         self.flush()
         self.output.write(self.newline)
         self.output.write(' ' * self.indentation)
         self.output_width = self.indentation
         self.buffer_width = 0
         
 
     def begin_group(self, indent=0, open=''):
         """
         Begin a group.  If you want support for python < 2.5 which doesn't has
         the with statement this is the preferred way:
 
             p.begin_group(1, '{')
             ...
             p.end_group(1, '}')
 
         The python 2.5 expression would be this:
 
             with p.group(1, '{', '}'):
                 ...
 
         The first parameter specifies the indentation for the next line (usually
         the width of the opening text), the second the opening text.  All
         parameters are optional.
         """
         if open:
             self.text(open)
         group = Group(self.group_stack[-1].depth + 1)
         self.group_stack.append(group)
         self.group_queue.enq(group)
         self.indentation += indent
 
     def end_group(self, dedent=0, close=''):
         """End a group. See `begin_group` for more details."""
         self.indentation -= dedent
         group = self.group_stack.pop()
         if not group.breakables:
             self.group_queue.remove(group)
         if close:
             self.text(close)
 
     def flush(self):
         """Flush data that is left in the buffer."""
         for data in self.buffer:
             self.output_width += data.output(self.output, self.output_width)
         self.buffer.clear()
         self.buffer_width = 0
 
 
 def _get_mro(obj_class):
     """ Get a reasonable method resolution order of a class and its superclasses
     for both old-style and new-style classes.
     """
     if not hasattr(obj_class, '__mro__'):
         # Old-style class. Mix in object to make a fake new-style class.
         try:
             obj_class = type(obj_class.__name__, (obj_class, object), {})
         except TypeError:
             # Old-style extension type that does not descend from object.
             # FIXME: try to construct a more thorough MRO.
             mro = [obj_class]
         else:
             mro = obj_class.__mro__[1:-1]
     else:
         mro = obj_class.__mro__
     return mro
 
 
 class RepresentationPrinter(PrettyPrinter):
     """
     Special pretty printer that has a `pretty` method that calls the pretty
     printer for a python object.
 
     This class stores processing data on `self` so you must *never* use
     this class in a threaded environment.  Always lock it or reinstanciate
     it.
 
     Instances also have a verbose flag callbacks can access to control their
     output.  For example the default instance repr prints all attributes and
     methods that are not prefixed by an underscore if the printer is in
     verbose mode.
     """
 
     def __init__(self, output, verbose=False, max_width=79, newline='\n',
         singleton_pprinters=None, type_pprinters=None, deferred_pprinters=None):
 
         PrettyPrinter.__init__(self, output, max_width, newline)
         self.verbose = verbose
         self.stack = []
         if singleton_pprinters is None:
             singleton_pprinters = _singleton_pprinters.copy()
         self.singleton_pprinters = singleton_pprinters
         if type_pprinters is None:
             type_pprinters = _type_pprinters.copy()
         self.type_pprinters = type_pprinters
         if deferred_pprinters is None:
             deferred_pprinters = _deferred_type_pprinters.copy()
         self.deferred_pprinters = deferred_pprinters
 
     def pretty(self, obj):
         """Pretty print the given object."""
         obj_id = id(obj)
         cycle = obj_id in self.stack
         self.stack.append(obj_id)
         self.begin_group()
         try:
             obj_class = getattr(obj, '__class__', None) or type(obj)
             # First try to find registered singleton printers for the type.
             try:
                 printer = self.singleton_pprinters[obj_id]
             except (TypeError, KeyError):
                 pass
             else:
                 return printer(obj, self, cycle)
             # Next walk the mro and check for either:
             #   1) a registered printer
             #   2) a _repr_pretty_ method
             for cls in _get_mro(obj_class):
                 if cls in self.type_pprinters:
                     # printer registered in self.type_pprinters
                     return self.type_pprinters[cls](obj, self, cycle)
                 else:
                     # deferred printer
                     printer = self._in_deferred_types(cls)
                     if printer is not None:
                         return printer(obj, self, cycle)
                     else:
                         # Finally look for special method names.
                         # Some objects automatically create any requested
                         # attribute. Try to ignore most of them by checking for
                         # callability.
                         if '_repr_pretty_' in cls.__dict__:
                             meth = cls._repr_pretty_
                             if callable(meth):
                                 return meth(obj, self, cycle)
             return _default_pprint(obj, self, cycle)
         finally:
             self.end_group()
             self.stack.pop()
 
     def _in_deferred_types(self, cls):
         """
         Check if the given class is specified in the deferred type registry.
 
         Returns the printer from the registry if it exists, and None if the
         class is not in the registry. Successful matches will be moved to the
         regular type registry for future use.
         """
         mod = getattr(cls, '__module__', None)
         name = getattr(cls, '__name__', None)
         key = (mod, name)
         printer = None
         if key in self.deferred_pprinters:
             # Move the printer over to the regular registry.
             printer = self.deferred_pprinters.pop(key)
             self.type_pprinters[cls] = printer
         return printer
 
 
 class Printable(object):
 
     def output(self, stream, output_width):
         return output_width
 
 
 class Text(Printable):
 
     def __init__(self):
         self.objs = []
         self.width = 0
 
     def output(self, stream, output_width):
         for obj in self.objs:
             stream.write(obj)
         return output_width + self.width
 
     def add(self, obj, width):
         self.objs.append(obj)
         self.width += width
 
 
 class Breakable(Printable):
 
     def __init__(self, seq, width, pretty):
         self.obj = seq
         self.width = width
         self.pretty = pretty
         self.indentation = pretty.indentation
         self.group = pretty.group_stack[-1]
         self.group.breakables.append(self)
 
     def output(self, stream, output_width):
         self.group.breakables.popleft()
         if self.group.want_break:
             stream.write(self.pretty.newline)
             stream.write(' ' * self.indentation)
             return self.indentation
         if not self.group.breakables:
             self.pretty.group_queue.remove(self.group)
         stream.write(self.obj)
         return output_width + self.width
 
 
 class Group(Printable):
 
     def __init__(self, depth):
         self.depth = depth
         self.breakables = deque()
         self.want_break = False
 
 
 class GroupQueue(object):
 
     def __init__(self, *groups):
         self.queue = []
         for group in groups:
             self.enq(group)
 
     def enq(self, group):
         depth = group.depth
         while depth > len(self.queue) - 1:
             self.queue.append([])
         self.queue[depth].append(group)
 
     def deq(self):
         for stack in self.queue:
             for idx, group in enumerate(reversed(stack)):
                 if group.breakables:
                     del stack[idx]
                     group.want_break = True
                     return group
             for group in stack:
                 group.want_break = True
             del stack[:]
 
     def remove(self, group):
         try:
             self.queue[group.depth].remove(group)
         except ValueError:
             pass
 
 try:
     _baseclass_reprs = (object.__repr__, types.InstanceType.__repr__)
 except AttributeError:  # Python 3
     _baseclass_reprs = (object.__repr__,)
 
 
 def _default_pprint(obj, p, cycle):
     """
     The default print function.  Used if an object does not provide one and
     it's none of the builtin objects.
     """
     klass = getattr(obj, '__class__', None) or type(obj)
     if getattr(klass, '__repr__', None) not in _baseclass_reprs:
         # A user-provided repr. Find newlines and replace them with p.break_()
         output = repr(obj)
         for idx,output_line in enumerate(output.splitlines()):
             if idx:
                 p.break_()
             p.text(output_line)
         return
     p.begin_group(1, '<')
     p.pretty(klass)
     p.text(' at 0x%x' % id(obj))
     if cycle:
         p.text(' ...')
     elif p.verbose:
         first = True
         for key in dir(obj):
             if not key.startswith('_'):
                 try:
                     value = getattr(obj, key)
                 except AttributeError:
                     continue
                 if isinstance(value, types.MethodType):
                     continue
                 if not first:
                     p.text(',')
                 p.breakable()
                 p.text(key)
                 p.text('=')
                 step = len(key) + 1
                 p.indentation += step
                 p.pretty(value)
                 p.indentation -= step
                 first = False
     p.end_group(1, '>')
 
 
 def _seq_pprinter_factory(start, end, basetype):
     """
     Factory that returns a pprint function useful for sequences.  Used by
     the default pprint for tuples, dicts, and lists.
     """
     def inner(obj, p, cycle):
         typ = type(obj)
         if basetype is not None and typ is not basetype and typ.__repr__ != basetype.__repr__:
             # If the subclass provides its own repr, use it instead.
             return p.text(typ.__repr__(obj))
 
         if cycle:
             return p.text(start + '...' + end)
         step = len(start)
         p.begin_group(step, start)
         for idx, x in enumerate(obj):
             if idx:
                 p.text(',')
                 p.breakable()
             p.pretty(x)
         if len(obj) == 1 and type(obj) is tuple:
             # Special case for 1-item tuples.
             p.text(',')
         p.end_group(step, end)
     return inner
 
 
 def _set_pprinter_factory(start, end, basetype):
     """
     Factory that returns a pprint function useful for sets and frozensets.
     """
     def inner(obj, p, cycle):
         typ = type(obj)
         if basetype is not None and typ is not basetype and typ.__repr__ != basetype.__repr__:
             # If the subclass provides its own repr, use it instead.
             return p.text(typ.__repr__(obj))
 
         if cycle:
             return p.text(start + '...' + end)
         if len(obj) == 0:
             # Special case.
             p.text(basetype.__name__ + '()')
         else:
             step = len(start)
             p.begin_group(step, start)
             # Like dictionary keys, we will try to sort the items.
             items = list(obj)
             try:
                 items.sort()
             except Exception:
                 # Sometimes the items don't sort.
                 pass
             for idx, x in enumerate(items):
                 if idx:
                     p.text(',')
                     p.breakable()
                 p.pretty(x)
             p.end_group(step, end)
     return inner
 
 
 def _dict_pprinter_factory(start, end, basetype=None):
     """
     Factory that returns a pprint function used by the default pprint of
     dicts and dict proxies.
     """
     def inner(obj, p, cycle):
         typ = type(obj)
         if basetype is not None and typ is not basetype and typ.__repr__ != basetype.__repr__:
             # If the subclass provides its own repr, use it instead.
             return p.text(typ.__repr__(obj))
 
         if cycle:
             return p.text('{...}')
         p.begin_group(1, start)
         keys = obj.keys()
         try:
             keys.sort()
         except Exception as e:
             # Sometimes the keys don't sort.
             pass
         for idx, key in enumerate(keys):
             if idx:
                 p.text(',')
                 p.breakable()
             p.pretty(key)
             p.text(': ')
             p.pretty(obj[key])
         p.end_group(1, end)
     return inner
 
 
 def _super_pprint(obj, p, cycle):
     """The pprint for the super type."""
     p.begin_group(8, '<super: ')
     p.pretty(obj.__self_class__)
     p.text(',')
     p.breakable()
     p.pretty(obj.__self__)
     p.end_group(8, '>')
 
 
 def _re_pattern_pprint(obj, p, cycle):
     """The pprint function for regular expression patterns."""
     p.text('re.compile(')
     pattern = repr(obj.pattern)
     if pattern[:1] in 'uU':
         pattern = pattern[1:]
         prefix = 'ur'
     else:
         prefix = 'r'
     pattern = prefix + pattern.replace('\\\\', '\\')
     p.text(pattern)
     if obj.flags:
         p.text(',')
         p.breakable()
         done_one = False
         for flag in ('TEMPLATE', 'IGNORECASE', 'LOCALE', 'MULTILINE', 'DOTALL',
             'UNICODE', 'VERBOSE', 'DEBUG'):
             if obj.flags & getattr(re, flag):
                 if done_one:
                     p.text('|')
                 p.text('re.' + flag)
                 done_one = True
     p.text(')')
 
 
 def _type_pprint(obj, p, cycle):
     """The pprint for classes and types."""
     mod = getattr(obj, '__module__', None)
     if mod is None:
         # Heap allocated types might not have the module attribute,
         # and others may set it to None.
         return p.text(obj.__name__)
 
     if mod in ('__builtin__', 'builtins', 'exceptions'):
         name = obj.__name__
     else:
         name = mod + '.' + obj.__name__
     p.text(name)
 
 
 def _repr_pprint(obj, p, cycle):
     """A pprint that just redirects to the normal repr function."""
     p.text(repr(obj))
 
 
 def _function_pprint(obj, p, cycle):
     """Base pprint for all functions and builtin functions."""
     if obj.__module__ in ('__builtin__', 'builtins', 'exceptions') or not obj.__module__:
         name = obj.__name__
     else:
         name = obj.__module__ + '.' + obj.__name__
     p.text('<function %s>' % name)
 
 
 def _exception_pprint(obj, p, cycle):
     """Base pprint for all exceptions."""
     if obj.__class__.__module__ in ('exceptions', 'builtins'):
         name = obj.__class__.__name__
     else:
         name = '%s.%s' % (
             obj.__class__.__module__,
             obj.__class__.__name__
         )
     step = len(name) + 1
     p.begin_group(step, name + '(')
     for idx, arg in enumerate(getattr(obj, 'args', ())):
         if idx:
             p.text(',')
             p.breakable()
         p.pretty(arg)
     p.end_group(step, ')')
 
 
 #: the exception base
 try:
     _exception_base = BaseException
 except NameError:
     _exception_base = Exception
 
 
 #: printers for builtin types
 _type_pprinters = {
     int:                        _repr_pprint,
-    long:                       _repr_pprint,
     float:                      _repr_pprint,
     str:                        _repr_pprint,
-    unicode:                    _repr_pprint,
     tuple:                      _seq_pprinter_factory('(', ')', tuple),
     list:                       _seq_pprinter_factory('[', ']', list),
     dict:                       _dict_pprinter_factory('{', '}', dict),
     
     set:                        _set_pprinter_factory('{', '}', set),
     frozenset:                  _set_pprinter_factory('frozenset({', '})', frozenset),
     super:                      _super_pprint,
     _re_pattern_type:           _re_pattern_pprint,
     type:                       _type_pprint,
     types.FunctionType:         _function_pprint,
     types.BuiltinFunctionType:  _function_pprint,
     types.SliceType:            _repr_pprint,
     types.MethodType:           _repr_pprint,
     
     datetime.datetime:          _repr_pprint,
     datetime.timedelta:         _repr_pprint,
     _exception_base:            _exception_pprint
 }
 
 try:
     _type_pprinters[types.DictProxyType] = _dict_pprinter_factory('<dictproxy {', '}>')
     _type_pprinters[types.ClassType] = _type_pprint
-except AttributeError: # Python 3
-    pass
+    _type_pprinters[long] = _repr_pprint
+    _type_pprinters[unicode] = _repr_pprint
+except (AttributeError, NameError): # Python 3
+    _type_pprinters[bytes] = _repr_pprint
     
 try:
     _type_pprinters[xrange] = _repr_pprint
 except NameError:
     _type_pprinters[range] = _repr_pprint
 
 #: printers for types specified by name
 _deferred_type_pprinters = {
 }
 
 def for_type(typ, func):
     """
     Add a pretty printer for a given type.
     """
     oldfunc = _type_pprinters.get(typ, None)
     if func is not None:
         # To support easy restoration of old pprinters, we need to ignore Nones.
         _type_pprinters[typ] = func
     return oldfunc
 
 def for_type_by_name(type_module, type_name, func):
     """
     Add a pretty printer for a type specified by the module and name of a type
     rather than the type object itself.
     """
     key = (type_module, type_name)
     oldfunc = _deferred_type_pprinters.get(key, None)
     if func is not None:
         # To support easy restoration of old pprinters, we need to ignore Nones.
         _deferred_type_pprinters[key] = func
     return oldfunc
 
 
 #: printers for the default singletons
 _singleton_pprinters = dict.fromkeys(map(id, [None, True, False, Ellipsis,
                                       NotImplemented]), _repr_pprint)
 
 
 if __name__ == '__main__':
     from random import randrange
     class Foo(object):
         def __init__(self):
             self.foo = 1
             self.bar = re.compile(r'\s+')
             self.blub = dict.fromkeys(range(30), randrange(1, 40))
             self.hehe = 23424.234234
             self.list = ["blub", "blah", self]
 
         def get_foo(self):
             print("foo")
 
     pprint(Foo(), verbose=True)

IPython/parallel/client/client.py

@@ -1,1859 +1,1854 b''
 """A semi-synchronous Client for the ZMQ cluster
 
 Authors:
 
 * MinRK
 """
 from __future__ import print_function
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import json
 import sys
 from threading import Thread, Event
 import time
 import warnings
 from datetime import datetime
 from getpass import getpass
 from pprint import pprint
 
 pjoin = os.path.join
 
 import zmq
 # from zmq.eventloop import ioloop, zmqstream
 
 from IPython.config.configurable import MultipleInstanceError
 from IPython.core.application import BaseIPythonApplication
 from IPython.core.profiledir import ProfileDir, ProfileDirError
 
 from IPython.utils.capture import RichOutput
 from IPython.utils.coloransi import TermColors
 from IPython.utils.jsonutil import rekey
 from IPython.utils.localinterfaces import localhost, is_local_ip
 from IPython.utils.path import get_ipython_dir
-from IPython.utils.py3compat import cast_bytes, string_types
+from IPython.utils.py3compat import cast_bytes, string_types, xrange
 from IPython.utils.traitlets import (HasTraits, Integer, Instance, Unicode,
                                     Dict, List, Bool, Set, Any)
 from IPython.external.decorator import decorator
 from IPython.external.ssh import tunnel
 
 from IPython.parallel import Reference
 from IPython.parallel import error
 from IPython.parallel import util
 
 from IPython.kernel.zmq.session import Session, Message
 from IPython.kernel.zmq import serialize
 
 from .asyncresult import AsyncResult, AsyncHubResult
 from .view import DirectView, LoadBalancedView
 
-if sys.version_info[0] >= 3:
-    # xrange is used in a couple 'isinstance' tests in py2
-    # should be just 'range' in 3k
-    xrange = range
-
 #--------------------------------------------------------------------------
 # Decorators for Client methods
 #--------------------------------------------------------------------------
 
 @decorator
 def spin_first(f, self, *args, **kwargs):
     """Call spin() to sync state prior to calling the method."""
     self.spin()
     return f(self, *args, **kwargs)
 
 
 #--------------------------------------------------------------------------
 # Classes
 #--------------------------------------------------------------------------
 
 
 class ExecuteReply(RichOutput):
     """wrapper for finished Execute results"""
     def __init__(self, msg_id, content, metadata):
         self.msg_id = msg_id
         self._content = content
         self.execution_count = content['execution_count']
         self.metadata = metadata
     
     # RichOutput overrides
     
     @property
     def source(self):
         pyout = self.metadata['pyout']
         if pyout:
             return pyout.get('source', '')
     
     @property
     def data(self):
         pyout = self.metadata['pyout']
         if pyout:
             return pyout.get('data', {})
     
     @property
     def _metadata(self):
         pyout = self.metadata['pyout']
         if pyout:
             return pyout.get('metadata', {})
     
     def display(self):
         from IPython.display import publish_display_data
         publish_display_data(self.source, self.data, self.metadata)
     
     def _repr_mime_(self, mime):
         if mime not in self.data:
             return
         data = self.data[mime]
         if mime in self._metadata:
             return data, self._metadata[mime]
         else:
             return data
     
     def __getitem__(self, key):
         return self.metadata[key]
     
     def __getattr__(self, key):
         if key not in self.metadata:
             raise AttributeError(key)
         return self.metadata[key]
     
     def __repr__(self):
         pyout = self.metadata['pyout'] or {'data':{}}
         text_out = pyout['data'].get('text/plain', '')
         if len(text_out) > 32:
             text_out = text_out[:29] + '...'
         
         return "<ExecuteReply[%i]: %s>" % (self.execution_count, text_out)
     
     def _repr_pretty_(self, p, cycle):
         pyout = self.metadata['pyout'] or {'data':{}}
         text_out = pyout['data'].get('text/plain', '')
         
         if not text_out:
             return
         
         try:
             ip = get_ipython()
         except NameError:
             colors = "NoColor"
         else:
             colors = ip.colors
         
         if colors == "NoColor":
             out = normal = ""
         else:
             out = TermColors.Red
             normal = TermColors.Normal
         
         if '\n' in text_out and not text_out.startswith('\n'):
             # add newline for multiline reprs
             text_out = '\n' + text_out
         
         p.text(
             out + u'Out[%i:%i]: ' % (
                 self.metadata['engine_id'], self.execution_count
             ) + normal + text_out
         )
 
 
 class Metadata(dict):
     """Subclass of dict for initializing metadata values.
 
     Attribute access works on keys.
 
     These objects have a strict set of keys - errors will raise if you try
     to add new keys.
     """
     def __init__(self, *args, **kwargs):
         dict.__init__(self)
         md = {'msg_id' : None,
               'submitted' : None,
               'started' : None,
               'completed' : None,
               'received' : None,
               'engine_uuid' : None,
               'engine_id' : None,
               'follow' : None,
               'after' : None,
               'status' : None,
 
               'pyin' : None,
               'pyout' : None,
               'pyerr' : None,
               'stdout' : '',
               'stderr' : '',
               'outputs' : [],
               'data': {},
               'outputs_ready' : False,
             }
         self.update(md)
         self.update(dict(*args, **kwargs))
 
     def __getattr__(self, key):
         """getattr aliased to getitem"""
         if key in self.iterkeys():
             return self[key]
         else:
             raise AttributeError(key)
 
     def __setattr__(self, key, value):
         """setattr aliased to setitem, with strict"""
         if key in self.iterkeys():
             self[key] = value
         else:
             raise AttributeError(key)
 
     def __setitem__(self, key, value):
         """strict static key enforcement"""
         if key in self.iterkeys():
             dict.__setitem__(self, key, value)
         else:
             raise KeyError(key)
 
 
 class Client(HasTraits):
     """A semi-synchronous client to the IPython ZMQ cluster
 
     Parameters
     ----------
 
     url_file : str/unicode; path to ipcontroller-client.json
         This JSON file should contain all the information needed to connect to a cluster,
         and is likely the only argument needed.
         Connection information for the Hub's registration.  If a json connector
         file is given, then likely no further configuration is necessary.
         [Default: use profile]
     profile : bytes
         The name of the Cluster profile to be used to find connector information.
         If run from an IPython application, the default profile will be the same
         as the running application, otherwise it will be 'default'.
     cluster_id : str
         String id to added to runtime files, to prevent name collisions when using
         multiple clusters with a single profile simultaneously.
         When set, will look for files named like: 'ipcontroller-<cluster_id>-client.json'
         Since this is text inserted into filenames, typical recommendations apply:
         Simple character strings are ideal, and spaces are not recommended (but
         should generally work)
     context : zmq.Context
         Pass an existing zmq.Context instance, otherwise the client will create its own.
     debug : bool
         flag for lots of message printing for debug purposes
     timeout : int/float
         time (in seconds) to wait for connection replies from the Hub
         [Default: 10]
 
     #-------------- session related args ----------------
 
     config : Config object
         If specified, this will be relayed to the Session for configuration
     username : str
         set username for the session object
 
     #-------------- ssh related args ----------------
     # These are args for configuring the ssh tunnel to be used
     # credentials are used to forward connections over ssh to the Controller
     # Note that the ip given in `addr` needs to be relative to sshserver
     # The most basic case is to leave addr as pointing to localhost (127.0.0.1),
     # and set sshserver as the same machine the Controller is on. However,
     # the only requirement is that sshserver is able to see the Controller
     # (i.e. is within the same trusted network).
 
     sshserver : str
         A string of the form passed to ssh, i.e. 'server.tld' or '[e-mail]:port'
         If keyfile or password is specified, and this is not, it will default to
         the ip given in addr.
     sshkey : str; path to ssh private key file
         This specifies a key to be used in ssh login, default None.
         Regular default ssh keys will be used without specifying this argument.
     password : str
         Your ssh password to sshserver. Note that if this is left None,
         you will be prompted for it if passwordless key based login is unavailable.
     paramiko : bool
         flag for whether to use paramiko instead of shell ssh for tunneling.
         [default: True on win32, False else]
 
 
     Attributes
     ----------
 
     ids : list of int engine IDs
         requesting the ids attribute always synchronizes
         the registration state. To request ids without synchronization,
         use semi-private _ids attributes.
 
     history : list of msg_ids
         a list of msg_ids, keeping track of all the execution
         messages you have submitted in order.
 
     outstanding : set of msg_ids
         a set of msg_ids that have been submitted, but whose
         results have not yet been received.
 
     results : dict
         a dict of all our results, keyed by msg_id
 
     block : bool
         determines default behavior when block not specified
         in execution methods
 
     Methods
     -------
 
     spin
         flushes incoming results and registration state changes
         control methods spin, and requesting `ids` also ensures up to date
 
     wait
         wait on one or more msg_ids
 
     execution methods
         apply
         legacy: execute, run
 
     data movement
         push, pull, scatter, gather
 
     query methods
         queue_status, get_result, purge, result_status
 
     control methods
         abort, shutdown
 
     """
 
 
     block = Bool(False)
     outstanding = Set()
     results = Instance('collections.defaultdict', (dict,))
     metadata = Instance('collections.defaultdict', (Metadata,))
     history = List()
     debug = Bool(False)
     _spin_thread = Any()
     _stop_spinning = Any()
 
     profile=Unicode()
     def _profile_default(self):
         if BaseIPythonApplication.initialized():
             # an IPython app *might* be running, try to get its profile
             try:
                 return BaseIPythonApplication.instance().profile
             except (AttributeError, MultipleInstanceError):
                 # could be a *different* subclass of config.Application,
                 # which would raise one of these two errors.
                 return u'default'
         else:
             return u'default'
 
 
     _outstanding_dict = Instance('collections.defaultdict', (set,))
     _ids = List()
     _connected=Bool(False)
     _ssh=Bool(False)
     _context = Instance('zmq.Context')
     _config = Dict()
     _engines=Instance(util.ReverseDict, (), {})
     # _hub_socket=Instance('zmq.Socket')
     _query_socket=Instance('zmq.Socket')
     _control_socket=Instance('zmq.Socket')
     _iopub_socket=Instance('zmq.Socket')
     _notification_socket=Instance('zmq.Socket')
     _mux_socket=Instance('zmq.Socket')
     _task_socket=Instance('zmq.Socket')
     _task_scheme=Unicode()
     _closed = False
     _ignored_control_replies=Integer(0)
     _ignored_hub_replies=Integer(0)
 
     def __new__(self, *args, **kw):
         # don't raise on positional args
         return HasTraits.__new__(self, **kw)
 
     def __init__(self, url_file=None, profile=None, profile_dir=None, ipython_dir=None,
             context=None, debug=False,
             sshserver=None, sshkey=None, password=None, paramiko=None,
             timeout=10, cluster_id=None, **extra_args
             ):
         if profile:
             super(Client, self).__init__(debug=debug, profile=profile)
         else:
             super(Client, self).__init__(debug=debug)
         if context is None:
             context = zmq.Context.instance()
         self._context = context
         self._stop_spinning = Event()
         
         if 'url_or_file' in extra_args:
             url_file = extra_args['url_or_file']
             warnings.warn("url_or_file arg no longer supported, use url_file", DeprecationWarning)
         
         if url_file and util.is_url(url_file):
             raise ValueError("single urls cannot be specified, url-files must be used.")
 
         self._setup_profile_dir(self.profile, profile_dir, ipython_dir)
         
         if self._cd is not None:
             if url_file is None:
                 if not cluster_id:
                     client_json = 'ipcontroller-client.json'
                 else:
                     client_json = 'ipcontroller-%s-client.json' % cluster_id
                 url_file = pjoin(self._cd.security_dir, client_json)
         if url_file is None:
             raise ValueError(
                 "I can't find enough information to connect to a hub!"
                 " Please specify at least one of url_file or profile."
             )
         
         with open(url_file) as f:
             cfg = json.load(f)
         
         self._task_scheme = cfg['task_scheme']
 
         # sync defaults from args, json:
         if sshserver:
             cfg['ssh'] = sshserver
 
         location = cfg.setdefault('location', None)
         
         proto,addr = cfg['interface'].split('://')
         addr = util.disambiguate_ip_address(addr, location)
         cfg['interface'] = "%s://%s" % (proto, addr)
         
         # turn interface,port into full urls:
         for key in ('control', 'task', 'mux', 'iopub', 'notification', 'registration'):
             cfg[key] = cfg['interface'] + ':%i' % cfg[key]
         
         url = cfg['registration']
         
         if location is not None and addr == localhost():
             # location specified, and connection is expected to be local
             if not is_local_ip(location) and not sshserver:
                 # load ssh from JSON *only* if the controller is not on
                 # this machine
                 sshserver=cfg['ssh']
             if not is_local_ip(location) and not sshserver:
                 # warn if no ssh specified, but SSH is probably needed
                 # This is only a warning, because the most likely cause
                 # is a local Controller on a laptop whose IP is dynamic
                 warnings.warn("""
             Controller appears to be listening on localhost, but not on this machine.
             If this is true, you should specify Client(...,sshserver='you@%s')
             or instruct your controller to listen on an external IP."""%location,
                 RuntimeWarning)
         elif not sshserver:
             # otherwise sync with cfg
             sshserver = cfg['ssh']
 
         self._config = cfg
 
         self._ssh = bool(sshserver or sshkey or password)
         if self._ssh and sshserver is None:
             # default to ssh via localhost
             sshserver = addr
         if self._ssh and password is None:
             if tunnel.try_passwordless_ssh(sshserver, sshkey, paramiko):
                 password=False
             else:
                 password = getpass("SSH Password for %s: "%sshserver)
         ssh_kwargs = dict(keyfile=sshkey, password=password, paramiko=paramiko)
 
         # configure and construct the session
         try:
             extra_args['packer'] = cfg['pack']
             extra_args['unpacker'] = cfg['unpack']
             extra_args['key'] = cast_bytes(cfg['key'])
             extra_args['signature_scheme'] = cfg['signature_scheme']
         except KeyError as exc:
             msg = '\n'.join([
                 "Connection file is invalid (missing '{}'), possibly from an old version of IPython.",
                 "If you are reusing connection files, remove them and start ipcontroller again."
             ])
             raise ValueError(msg.format(exc.message))
         
         self.session = Session(**extra_args)
 
         self._query_socket = self._context.socket(zmq.DEALER)
 
         if self._ssh:
             tunnel.tunnel_connection(self._query_socket, cfg['registration'], sshserver, **ssh_kwargs)
         else:
             self._query_socket.connect(cfg['registration'])
 
         self.session.debug = self.debug
 
         self._notification_handlers = {'registration_notification' : self._register_engine,
                                     'unregistration_notification' : self._unregister_engine,
                                     'shutdown_notification' : lambda msg: self.close(),
                                     }
         self._queue_handlers = {'execute_reply' : self._handle_execute_reply,
                                 'apply_reply' : self._handle_apply_reply}
         
         try:
             self._connect(sshserver, ssh_kwargs, timeout)
         except:
             self.close(linger=0)
             raise
         
         # last step: setup magics, if we are in IPython:
         
         try:
             ip = get_ipython()
         except NameError:
             return
         else:
             if 'px' not in ip.magics_manager.magics:
                 # in IPython but we are the first Client.
                 # activate a default view for parallel magics.
                 self.activate()
 
     def __del__(self):
         """cleanup sockets, but _not_ context."""
         self.close()
 
     def _setup_profile_dir(self, profile, profile_dir, ipython_dir):
         if ipython_dir is None:
             ipython_dir = get_ipython_dir()
         if profile_dir is not None:
             try:
                 self._cd = ProfileDir.find_profile_dir(profile_dir)
                 return
             except ProfileDirError:
                 pass
         elif profile is not None:
             try:
                 self._cd = ProfileDir.find_profile_dir_by_name(
                     ipython_dir, profile)
                 return
             except ProfileDirError:
                 pass
         self._cd = None
 
     def _update_engines(self, engines):
         """Update our engines dict and _ids from a dict of the form: {id:uuid}."""
         for k,v in engines.iteritems():
             eid = int(k)
             if eid not in self._engines:
                 self._ids.append(eid)
             self._engines[eid] = v
         self._ids = sorted(self._ids)
         if sorted(self._engines.keys()) != range(len(self._engines)) and \
                         self._task_scheme == 'pure' and self._task_socket:
             self._stop_scheduling_tasks()
 
     def _stop_scheduling_tasks(self):
         """Stop scheduling tasks because an engine has been unregistered
         from a pure ZMQ scheduler.
         """
         self._task_socket.close()
         self._task_socket = None
         msg = "An engine has been unregistered, and we are using pure " +\
               "ZMQ task scheduling.  Task farming will be disabled."
         if self.outstanding:
             msg += " If you were running tasks when this happened, " +\
                    "some `outstanding` msg_ids may never resolve."
         warnings.warn(msg, RuntimeWarning)
 
     def _build_targets(self, targets):
         """Turn valid target IDs or 'all' into two lists:
         (int_ids, uuids).
         """
         if not self._ids:
             # flush notification socket if no engines yet, just in case
             if not self.ids:
                 raise error.NoEnginesRegistered("Can't build targets without any engines")
 
         if targets is None:
             targets = self._ids
         elif isinstance(targets, string_types):
             if targets.lower() == 'all':
                 targets = self._ids
             else:
                 raise TypeError("%r not valid str target, must be 'all'"%(targets))
         elif isinstance(targets, int):
             if targets < 0:
                 targets = self.ids[targets]
             if targets not in self._ids:
                 raise IndexError("No such engine: %i"%targets)
             targets = [targets]
 
         if isinstance(targets, slice):
             indices = range(len(self._ids))[targets]
             ids = self.ids
             targets = [ ids[i] for i in indices ]
 
         if not isinstance(targets, (tuple, list, xrange)):
             raise TypeError("targets by int/slice/collection of ints only, not %s"%(type(targets)))
 
         return [cast_bytes(self._engines[t]) for t in targets], list(targets)
 
     def _connect(self, sshserver, ssh_kwargs, timeout):
         """setup all our socket connections to the cluster. This is called from
         __init__."""
 
         # Maybe allow reconnecting?
         if self._connected:
             return
         self._connected=True
 
         def connect_socket(s, url):
             if self._ssh:
                 return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs)
             else:
                 return s.connect(url)
 
         self.session.send(self._query_socket, 'connection_request')
         # use Poller because zmq.select has wrong units in pyzmq 2.1.7
         poller = zmq.Poller()
         poller.register(self._query_socket, zmq.POLLIN)
         # poll expects milliseconds, timeout is seconds
         evts = poller.poll(timeout*1000)
         if not evts:
             raise error.TimeoutError("Hub connection request timed out")
         idents,msg = self.session.recv(self._query_socket,mode=0)
         if self.debug:
             pprint(msg)
         content = msg['content']
         # self._config['registration'] = dict(content)
         cfg = self._config
         if content['status'] == 'ok':
             self._mux_socket = self._context.socket(zmq.DEALER)
             connect_socket(self._mux_socket, cfg['mux'])
 
             self._task_socket = self._context.socket(zmq.DEALER)
             connect_socket(self._task_socket, cfg['task'])
 
             self._notification_socket = self._context.socket(zmq.SUB)
             self._notification_socket.setsockopt(zmq.SUBSCRIBE, b'')
             connect_socket(self._notification_socket, cfg['notification'])
 
             self._control_socket = self._context.socket(zmq.DEALER)
             connect_socket(self._control_socket, cfg['control'])
 
             self._iopub_socket = self._context.socket(zmq.SUB)
             self._iopub_socket.setsockopt(zmq.SUBSCRIBE, b'')
             connect_socket(self._iopub_socket, cfg['iopub'])
 
             self._update_engines(dict(content['engines']))
         else:
             self._connected = False
             raise Exception("Failed to connect!")
 
     #--------------------------------------------------------------------------
     # handlers and callbacks for incoming messages
     #--------------------------------------------------------------------------
 
     def _unwrap_exception(self, content):
         """unwrap exception, and remap engine_id to int."""
         e = error.unwrap_exception(content)
         # print e.traceback
         if e.engine_info:
             e_uuid = e.engine_info['engine_uuid']
             eid = self._engines[e_uuid]
             e.engine_info['engine_id'] = eid
         return e
 
     def _extract_metadata(self, msg):
         header = msg['header']
         parent = msg['parent_header']
         msg_meta = msg['metadata']
         content = msg['content']
         md = {'msg_id' : parent['msg_id'],
               'received' : datetime.now(),
               'engine_uuid' : msg_meta.get('engine', None),
               'follow' : msg_meta.get('follow', []),
               'after' : msg_meta.get('after', []),
               'status' : content['status'],
             }
 
         if md['engine_uuid'] is not None:
             md['engine_id'] = self._engines.get(md['engine_uuid'], None)
 
         if 'date' in parent:
             md['submitted'] = parent['date']
         if 'started' in msg_meta:
             md['started'] = msg_meta['started']
         if 'date' in header:
             md['completed'] = header['date']
         return md
 
     def _register_engine(self, msg):
         """Register a new engine, and update our connection info."""
         content = msg['content']
         eid = content['id']
         d = {eid : content['uuid']}
         self._update_engines(d)
 
     def _unregister_engine(self, msg):
         """Unregister an engine that has died."""
         content = msg['content']
         eid = int(content['id'])
         if eid in self._ids:
             self._ids.remove(eid)
             uuid = self._engines.pop(eid)
 
             self._handle_stranded_msgs(eid, uuid)
 
         if self._task_socket and self._task_scheme == 'pure':
             self._stop_scheduling_tasks()
 
     def _handle_stranded_msgs(self, eid, uuid):
         """Handle messages known to be on an engine when the engine unregisters.
 
         It is possible that this will fire prematurely - that is, an engine will
         go down after completing a result, and the client will be notified
         of the unregistration and later receive the successful result.
         """
 
         outstanding = self._outstanding_dict[uuid]
 
         for msg_id in list(outstanding):
             if msg_id in self.results:
                 # we already
                 continue
             try:
                 raise error.EngineError("Engine %r died while running task %r"%(eid, msg_id))
             except:
                 content = error.wrap_exception()
             # build a fake message:
             msg = self.session.msg('apply_reply', content=content)
             msg['parent_header']['msg_id'] = msg_id
             msg['metadata']['engine'] = uuid
             self._handle_apply_reply(msg)
 
     def _handle_execute_reply(self, msg):
         """Save the reply to an execute_request into our results.
 
         execute messages are never actually used. apply is used instead.
         """
 
         parent = msg['parent_header']
         msg_id = parent['msg_id']
         if msg_id not in self.outstanding:
             if msg_id in self.history:
                 print(("got stale result: %s"%msg_id))
             else:
                 print(("got unknown result: %s"%msg_id))
         else:
             self.outstanding.remove(msg_id)
 
         content = msg['content']
         header = msg['header']
 
         # construct metadata:
         md = self.metadata[msg_id]
         md.update(self._extract_metadata(msg))
         # is this redundant?
         self.metadata[msg_id] = md
         
         e_outstanding = self._outstanding_dict[md['engine_uuid']]
         if msg_id in e_outstanding:
             e_outstanding.remove(msg_id)
 
         # construct result:
         if content['status'] == 'ok':
             self.results[msg_id] = ExecuteReply(msg_id, content, md)
         elif content['status'] == 'aborted':
             self.results[msg_id] = error.TaskAborted(msg_id)
         elif content['status'] == 'resubmitted':
             # TODO: handle resubmission
             pass
         else:
             self.results[msg_id] = self._unwrap_exception(content)
 
     def _handle_apply_reply(self, msg):
         """Save the reply to an apply_request into our results."""
         parent = msg['parent_header']
         msg_id = parent['msg_id']
         if msg_id not in self.outstanding:
             if msg_id in self.history:
                 print(("got stale result: %s"%msg_id))
                 print(self.results[msg_id])
                 print(msg)
             else:
                 print(("got unknown result: %s"%msg_id))
         else:
             self.outstanding.remove(msg_id)
         content = msg['content']
         header = msg['header']
 
         # construct metadata:
         md = self.metadata[msg_id]
         md.update(self._extract_metadata(msg))
         # is this redundant?
         self.metadata[msg_id] = md
 
         e_outstanding = self._outstanding_dict[md['engine_uuid']]
         if msg_id in e_outstanding:
             e_outstanding.remove(msg_id)
 
         # construct result:
         if content['status'] == 'ok':
             self.results[msg_id] = serialize.unserialize_object(msg['buffers'])[0]
         elif content['status'] == 'aborted':
             self.results[msg_id] = error.TaskAborted(msg_id)
         elif content['status'] == 'resubmitted':
             # TODO: handle resubmission
             pass
         else:
             self.results[msg_id] = self._unwrap_exception(content)
 
     def _flush_notifications(self):
         """Flush notifications of engine registrations waiting
         in ZMQ queue."""
         idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
             msg_type = msg['header']['msg_type']
             handler = self._notification_handlers.get(msg_type, None)
             if handler is None:
                 raise Exception("Unhandled message type: %s" % msg_type)
             else:
                 handler(msg)
             idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
 
     def _flush_results(self, sock):
         """Flush task or queue results waiting in ZMQ queue."""
         idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
             msg_type = msg['header']['msg_type']
             handler = self._queue_handlers.get(msg_type, None)
             if handler is None:
                 raise Exception("Unhandled message type: %s" % msg_type)
             else:
                 handler(msg)
             idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
 
     def _flush_control(self, sock):
         """Flush replies from the control channel waiting
         in the ZMQ queue.
 
         Currently: ignore them."""
         if self._ignored_control_replies <= 0:
             return
         idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             self._ignored_control_replies -= 1
             if self.debug:
                 pprint(msg)
             idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
 
     def _flush_ignored_control(self):
         """flush ignored control replies"""
         while self._ignored_control_replies > 0:
             self.session.recv(self._control_socket)
             self._ignored_control_replies -= 1
 
     def _flush_ignored_hub_replies(self):
         ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
         while msg is not None:
             ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
 
     def _flush_iopub(self, sock):
         """Flush replies from the iopub channel waiting
         in the ZMQ queue.
         """
         idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
             parent = msg['parent_header']
             # ignore IOPub messages with no parent.
             # Caused by print statements or warnings from before the first execution.
             if not parent:
                 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
                 continue
             msg_id = parent['msg_id']
             content = msg['content']
             header = msg['header']
             msg_type = msg['header']['msg_type']
 
             # init metadata:
             md = self.metadata[msg_id]
 
             if msg_type == 'stream':
                 name = content['name']
                 s = md[name] or ''
                 md[name] = s + content['data']
             elif msg_type == 'pyerr':
                 md.update({'pyerr' : self._unwrap_exception(content)})
             elif msg_type == 'pyin':
                 md.update({'pyin' : content['code']})
             elif msg_type == 'display_data':
                 md['outputs'].append(content)
             elif msg_type == 'pyout':
                 md['pyout'] = content
             elif msg_type == 'data_message':
                 data, remainder = serialize.unserialize_object(msg['buffers'])
                 md['data'].update(data)
             elif msg_type == 'status':
                 # idle message comes after all outputs
                 if content['execution_state'] == 'idle':
                     md['outputs_ready'] = True
             else:
                 # unhandled msg_type (status, etc.)
                 pass
 
             # reduntant?
             self.metadata[msg_id] = md
 
             idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
 
     #--------------------------------------------------------------------------
     # len, getitem
     #--------------------------------------------------------------------------
 
     def __len__(self):
         """len(client) returns # of engines."""
         return len(self.ids)
 
     def __getitem__(self, key):
         """index access returns DirectView multiplexer objects
 
         Must be int, slice, or list/tuple/xrange of ints"""
         if not isinstance(key, (int, slice, tuple, list, xrange)):
             raise TypeError("key by int/slice/iterable of ints only, not %s"%(type(key)))
         else:
             return self.direct_view(key)
 
     #--------------------------------------------------------------------------
     # Begin public methods
     #--------------------------------------------------------------------------
 
     @property
     def ids(self):
         """Always up-to-date ids property."""
         self._flush_notifications()
         # always copy:
         return list(self._ids)
 
     def activate(self, targets='all', suffix=''):
         """Create a DirectView and register it with IPython magics
         
         Defines the magics `%px, %autopx, %pxresult, %%px`
         
         Parameters
         ----------
         
         targets: int, list of ints, or 'all'
             The engines on which the view's magics will run
         suffix: str [default: '']
             The suffix, if any, for the magics.  This allows you to have
             multiple views associated with parallel magics at the same time.
             
             e.g. ``rc.activate(targets=0, suffix='0')`` will give you
             the magics ``%px0``, ``%pxresult0``, etc. for running magics just
             on engine 0.
         """
         view = self.direct_view(targets)
         view.block = True
         view.activate(suffix)
         return view
 
     def close(self, linger=None):
         """Close my zmq Sockets
         
         If `linger`, set the zmq LINGER socket option,
         which allows discarding of messages.
         """
         if self._closed:
             return
         self.stop_spin_thread()
         snames = [ trait for trait in self.trait_names() if trait.endswith("socket") ]
         for name in snames:
             socket = getattr(self, name)
             if socket is not None and not socket.closed:
                 if linger is not None:
                     socket.close(linger=linger)
                 else:
                     socket.close()
         self._closed = True
 
     def _spin_every(self, interval=1):
         """target func for use in spin_thread"""
         while True:
             if self._stop_spinning.is_set():
                 return
             time.sleep(interval)
             self.spin()
 
     def spin_thread(self, interval=1):
         """call Client.spin() in a background thread on some regular interval
         
         This helps ensure that messages don't pile up too much in the zmq queue
         while you are working on other things, or just leaving an idle terminal.
         
         It also helps limit potential padding of the `received` timestamp
         on AsyncResult objects, used for timings.
         
         Parameters
         ----------
         
         interval : float, optional
             The interval on which to spin the client in the background thread
             (simply passed to time.sleep).
         
         Notes
         -----
         
         For precision timing, you may want to use this method to put a bound
         on the jitter (in seconds) in `received` timestamps used
         in AsyncResult.wall_time.
         
         """
         if self._spin_thread is not None:
             self.stop_spin_thread()
         self._stop_spinning.clear()
         self._spin_thread = Thread(target=self._spin_every, args=(interval,))
         self._spin_thread.daemon = True
         self._spin_thread.start()
     
     def stop_spin_thread(self):
         """stop background spin_thread, if any"""
         if self._spin_thread is not None:
             self._stop_spinning.set()
             self._spin_thread.join()
             self._spin_thread = None
 
     def spin(self):
         """Flush any registration notifications and execution results
         waiting in the ZMQ queue.
         """
         if self._notification_socket:
             self._flush_notifications()
         if self._iopub_socket:
             self._flush_iopub(self._iopub_socket)
         if self._mux_socket:
             self._flush_results(self._mux_socket)
         if self._task_socket:
             self._flush_results(self._task_socket)
         if self._control_socket:
             self._flush_control(self._control_socket)
         if self._query_socket:
             self._flush_ignored_hub_replies()
 
     def wait(self, jobs=None, timeout=-1):
         """waits on one or more `jobs`, for up to `timeout` seconds.
 
         Parameters
         ----------
 
         jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
                 ints are indices to self.history
                 strs are msg_ids
                 default: wait on all outstanding messages
         timeout : float
                 a time in seconds, after which to give up.
                 default is -1, which means no timeout
 
         Returns
         -------
 
         True : when all msg_ids are done
         False : timeout reached, some msg_ids still outstanding
         """
         tic = time.time()
         if jobs is None:
             theids = self.outstanding
         else:
             if isinstance(jobs, string_types + (int, AsyncResult)):
                 jobs = [jobs]
             theids = set()
             for job in jobs:
                 if isinstance(job, int):
                     # index access
                     job = self.history[job]
                 elif isinstance(job, AsyncResult):
                     map(theids.add, job.msg_ids)
                     continue
                 theids.add(job)
         if not theids.intersection(self.outstanding):
             return True
         self.spin()
         while theids.intersection(self.outstanding):
             if timeout >= 0 and ( time.time()-tic ) > timeout:
                 break
             time.sleep(1e-3)
             self.spin()
         return len(theids.intersection(self.outstanding)) == 0
 
     #--------------------------------------------------------------------------
     # Control methods
     #--------------------------------------------------------------------------
 
     @spin_first
     def clear(self, targets=None, block=None):
         """Clear the namespace in target(s)."""
         block = self.block if block is None else block
         targets = self._build_targets(targets)[0]
         for t in targets:
             self.session.send(self._control_socket, 'clear_request', content={}, ident=t)
         error = False
         if block:
             self._flush_ignored_control()
             for i in range(len(targets)):
                 idents,msg = self.session.recv(self._control_socket,0)
                 if self.debug:
                     pprint(msg)
                 if msg['content']['status'] != 'ok':
                     error = self._unwrap_exception(msg['content'])
         else:
             self._ignored_control_replies += len(targets)
         if error:
             raise error
 
 
     @spin_first
     def abort(self, jobs=None, targets=None, block=None):
         """Abort specific jobs from the execution queues of target(s).
 
         This is a mechanism to prevent jobs that have already been submitted
         from executing.
 
         Parameters
         ----------
 
         jobs : msg_id, list of msg_ids, or AsyncResult
             The jobs to be aborted
             
             If unspecified/None: abort all outstanding jobs.
 
         """
         block = self.block if block is None else block
         jobs = jobs if jobs is not None else list(self.outstanding)
         targets = self._build_targets(targets)[0]
         
         msg_ids = []
         if isinstance(jobs, string_types + (AsyncResult,)):
             jobs = [jobs]
         bad_ids = filter(lambda obj: not isinstance(obj, string_types + (AsyncResult,)), jobs)
         if bad_ids:
             raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
         for j in jobs:
             if isinstance(j, AsyncResult):
                 msg_ids.extend(j.msg_ids)
             else:
                 msg_ids.append(j)
         content = dict(msg_ids=msg_ids)
         for t in targets:
             self.session.send(self._control_socket, 'abort_request',
                     content=content, ident=t)
         error = False
         if block:
             self._flush_ignored_control()
             for i in range(len(targets)):
                 idents,msg = self.session.recv(self._control_socket,0)
                 if self.debug:
                     pprint(msg)
                 if msg['content']['status'] != 'ok':
                     error = self._unwrap_exception(msg['content'])
         else:
             self._ignored_control_replies += len(targets)
         if error:
             raise error
 
     @spin_first
     def shutdown(self, targets='all', restart=False, hub=False, block=None):
         """Terminates one or more engine processes, optionally including the hub.
         
         Parameters
         ----------
         
         targets: list of ints or 'all' [default: all]
             Which engines to shutdown.
         hub: bool [default: False]
             Whether to include the Hub.  hub=True implies targets='all'.
         block: bool [default: self.block]
             Whether to wait for clean shutdown replies or not.
         restart: bool [default: False]
             NOT IMPLEMENTED
             whether to restart engines after shutting them down.
         """
         from IPython.parallel.error import NoEnginesRegistered
         if restart:
             raise NotImplementedError("Engine restart is not yet implemented")
         
         block = self.block if block is None else block
         if hub:
             targets = 'all'
         try:
             targets = self._build_targets(targets)[0]
         except NoEnginesRegistered:
             targets = []
         for t in targets:
             self.session.send(self._control_socket, 'shutdown_request',
                         content={'restart':restart},ident=t)
         error = False
         if block or hub:
             self._flush_ignored_control()
             for i in range(len(targets)):
                 idents,msg = self.session.recv(self._control_socket, 0)
                 if self.debug:
                     pprint(msg)
                 if msg['content']['status'] != 'ok':
                     error = self._unwrap_exception(msg['content'])
         else:
             self._ignored_control_replies += len(targets)
 
         if hub:
             time.sleep(0.25)
             self.session.send(self._query_socket, 'shutdown_request')
             idents,msg = self.session.recv(self._query_socket, 0)
             if self.debug:
                 pprint(msg)
             if msg['content']['status'] != 'ok':
                 error = self._unwrap_exception(msg['content'])
 
         if error:
             raise error
 
     #--------------------------------------------------------------------------
     # Execution related methods
     #--------------------------------------------------------------------------
 
     def _maybe_raise(self, result):
         """wrapper for maybe raising an exception if apply failed."""
         if isinstance(result, error.RemoteError):
             raise result
 
         return result
 
     def send_apply_request(self, socket, f, args=None, kwargs=None, metadata=None, track=False,
                             ident=None):
         """construct and send an apply message via a socket.
 
         This is the principal method with which all engine execution is performed by views.
         """
 
         if self._closed:
             raise RuntimeError("Client cannot be used after its sockets have been closed")
         
         # defaults:
         args = args if args is not None else []
         kwargs = kwargs if kwargs is not None else {}
         metadata = metadata if metadata is not None else {}
 
         # validate arguments
         if not callable(f) and not isinstance(f, Reference):
             raise TypeError("f must be callable, not %s"%type(f))
         if not isinstance(args, (tuple, list)):
             raise TypeError("args must be tuple or list, not %s"%type(args))
         if not isinstance(kwargs, dict):
             raise TypeError("kwargs must be dict, not %s"%type(kwargs))
         if not isinstance(metadata, dict):
             raise TypeError("metadata must be dict, not %s"%type(metadata))
 
         bufs = serialize.pack_apply_message(f, args, kwargs,
             buffer_threshold=self.session.buffer_threshold,
             item_threshold=self.session.item_threshold,
         )
 
         msg = self.session.send(socket, "apply_request", buffers=bufs, ident=ident,
                             metadata=metadata, track=track)
 
         msg_id = msg['header']['msg_id']
         self.outstanding.add(msg_id)
         if ident:
             # possibly routed to a specific engine
             if isinstance(ident, list):
                 ident = ident[-1]
             if ident in self._engines.values():
                 # save for later, in case of engine death
                 self._outstanding_dict[ident].add(msg_id)
         self.history.append(msg_id)
         self.metadata[msg_id]['submitted'] = datetime.now()
 
         return msg
 
     def send_execute_request(self, socket, code, silent=True, metadata=None, ident=None):
         """construct and send an execute request via a socket.
 
         """
 
         if self._closed:
             raise RuntimeError("Client cannot be used after its sockets have been closed")
         
         # defaults:
         metadata = metadata if metadata is not None else {}
 
         # validate arguments
         if not isinstance(code, string_types):
             raise TypeError("code must be text, not %s" % type(code))
         if not isinstance(metadata, dict):
             raise TypeError("metadata must be dict, not %s" % type(metadata))
         
         content = dict(code=code, silent=bool(silent), user_variables=[], user_expressions={})
 
 
         msg = self.session.send(socket, "execute_request", content=content, ident=ident,
                             metadata=metadata)
 
         msg_id = msg['header']['msg_id']
         self.outstanding.add(msg_id)
         if ident:
             # possibly routed to a specific engine
             if isinstance(ident, list):
                 ident = ident[-1]
             if ident in self._engines.values():
                 # save for later, in case of engine death
                 self._outstanding_dict[ident].add(msg_id)
         self.history.append(msg_id)
         self.metadata[msg_id]['submitted'] = datetime.now()
 
         return msg
 
     #--------------------------------------------------------------------------
     # construct a View object
     #--------------------------------------------------------------------------
 
     def load_balanced_view(self, targets=None):
         """construct a DirectView object.
 
         If no arguments are specified, create a LoadBalancedView
         using all engines.
 
         Parameters
         ----------
 
         targets: list,slice,int,etc. [default: use all engines]
             The subset of engines across which to load-balance
         """
         if targets == 'all':
             targets = None
         if targets is not None:
             targets = self._build_targets(targets)[1]
         return LoadBalancedView(client=self, socket=self._task_socket, targets=targets)
 
     def direct_view(self, targets='all'):
         """construct a DirectView object.
 
         If no targets are specified, create a DirectView using all engines.
         
         rc.direct_view('all') is distinguished from rc[:] in that 'all' will
         evaluate the target engines at each execution, whereas rc[:] will connect to
         all *current* engines, and that list will not change.
         
         That is, 'all' will always use all engines, whereas rc[:] will not use
         engines added after the DirectView is constructed.
 
         Parameters
         ----------
 
         targets: list,slice,int,etc. [default: use all engines]
             The engines to use for the View
         """
         single = isinstance(targets, int)
         # allow 'all' to be lazily evaluated at each execution
         if targets != 'all':
             targets = self._build_targets(targets)[1]
         if single:
             targets = targets[0]
         return DirectView(client=self, socket=self._mux_socket, targets=targets)
 
     #--------------------------------------------------------------------------
     # Query methods
     #--------------------------------------------------------------------------
 
     @spin_first
     def get_result(self, indices_or_msg_ids=None, block=None):
         """Retrieve a result by msg_id or history index, wrapped in an AsyncResult object.
 
         If the client already has the results, no request to the Hub will be made.
 
         This is a convenient way to construct AsyncResult objects, which are wrappers
         that include metadata about execution, and allow for awaiting results that
         were not submitted by this Client.
 
         It can also be a convenient way to retrieve the metadata associated with
         blocking execution, since it always retrieves
 
         Examples
         --------
         ::
 
             In [10]: r = client.apply()
 
         Parameters
         ----------
 
         indices_or_msg_ids : integer history index, str msg_id, or list of either
             The indices or msg_ids of indices to be retrieved
 
         block : bool
             Whether to wait for the result to be done
 
         Returns
         -------
 
         AsyncResult
             A single AsyncResult object will always be returned.
 
         AsyncHubResult
             A subclass of AsyncResult that retrieves results from the Hub
 
         """
         block = self.block if block is None else block
         if indices_or_msg_ids is None:
             indices_or_msg_ids = -1
         
         single_result = False
         if not isinstance(indices_or_msg_ids, (list,tuple)):
             indices_or_msg_ids = [indices_or_msg_ids]
             single_result = True
 
         theids = []
         for id in indices_or_msg_ids:
             if isinstance(id, int):
                 id = self.history[id]
             if not isinstance(id, string_types):
                 raise TypeError("indices must be str or int, not %r"%id)
             theids.append(id)
 
         local_ids = filter(lambda msg_id: msg_id in self.outstanding or msg_id in self.results, theids)
         remote_ids = filter(lambda msg_id: msg_id not in local_ids, theids)
         
         # given single msg_id initially, get_result shot get the result itself,
         # not a length-one list
         if single_result:
             theids = theids[0]
 
         if remote_ids:
             ar = AsyncHubResult(self, msg_ids=theids)
         else:
             ar = AsyncResult(self, msg_ids=theids)
 
         if block:
             ar.wait()
 
         return ar
 
     @spin_first
     def resubmit(self, indices_or_msg_ids=None, metadata=None, block=None):
         """Resubmit one or more tasks.
 
         in-flight tasks may not be resubmitted.
 
         Parameters
         ----------
 
         indices_or_msg_ids : integer history index, str msg_id, or list of either
             The indices or msg_ids of indices to be retrieved
 
         block : bool
             Whether to wait for the result to be done
 
         Returns
         -------
 
         AsyncHubResult
             A subclass of AsyncResult that retrieves results from the Hub
 
         """
         block = self.block if block is None else block
         if indices_or_msg_ids is None:
             indices_or_msg_ids = -1
 
         if not isinstance(indices_or_msg_ids, (list,tuple)):
             indices_or_msg_ids = [indices_or_msg_ids]
 
         theids = []
         for id in indices_or_msg_ids:
             if isinstance(id, int):
                 id = self.history[id]
             if not isinstance(id, string_types):
                 raise TypeError("indices must be str or int, not %r"%id)
             theids.append(id)
 
         content = dict(msg_ids = theids)
 
         self.session.send(self._query_socket, 'resubmit_request', content)
 
         zmq.select([self._query_socket], [], [])
         idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
         if self.debug:
             pprint(msg)
         content = msg['content']
         if content['status'] != 'ok':
             raise self._unwrap_exception(content)
         mapping = content['resubmitted']
         new_ids = [ mapping[msg_id] for msg_id in theids ]
 
         ar = AsyncHubResult(self, msg_ids=new_ids)
 
         if block:
             ar.wait()
 
         return ar
 
     @spin_first
     def result_status(self, msg_ids, status_only=True):
         """Check on the status of the result(s) of the apply request with `msg_ids`.
 
         If status_only is False, then the actual results will be retrieved, else
         only the status of the results will be checked.
 
         Parameters
         ----------
 
         msg_ids : list of msg_ids
             if int:
                 Passed as index to self.history for convenience.
         status_only : bool (default: True)
             if False:
                 Retrieve the actual results of completed tasks.
 
         Returns
         -------
 
         results : dict
             There will always be the keys 'pending' and 'completed', which will
             be lists of msg_ids that are incomplete or complete. If `status_only`
             is False, then completed results will be keyed by their `msg_id`.
         """
         if not isinstance(msg_ids, (list,tuple)):
             msg_ids = [msg_ids]
 
         theids = []
         for msg_id in msg_ids:
             if isinstance(msg_id, int):
                 msg_id = self.history[msg_id]
             if not isinstance(msg_id, string_types):
                 raise TypeError("msg_ids must be str, not %r"%msg_id)
             theids.append(msg_id)
 
         completed = []
         local_results = {}
 
         # comment this block out to temporarily disable local shortcut:
         for msg_id in theids:
             if msg_id in self.results:
                 completed.append(msg_id)
                 local_results[msg_id] = self.results[msg_id]
                 theids.remove(msg_id)
 
         if theids: # some not locally cached
             content = dict(msg_ids=theids, status_only=status_only)
             msg = self.session.send(self._query_socket, "result_request", content=content)
             zmq.select([self._query_socket], [], [])
             idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
             if self.debug:
                 pprint(msg)
             content = msg['content']
             if content['status'] != 'ok':
                 raise self._unwrap_exception(content)
             buffers = msg['buffers']
         else:
             content = dict(completed=[],pending=[])
 
         content['completed'].extend(completed)
 
         if status_only:
             return content
 
         failures = []
         # load cached results into result:
         content.update(local_results)
 
         # update cache with results:
         for msg_id in sorted(theids):
             if msg_id in content['completed']:
                 rec = content[msg_id]
                 parent = rec['header']
                 header = rec['result_header']
                 rcontent = rec['result_content']
                 iodict = rec['io']
                 if isinstance(rcontent, str):
                     rcontent = self.session.unpack(rcontent)
 
                 md = self.metadata[msg_id]
                 md_msg = dict(
                     content=rcontent,
                     parent_header=parent,
                     header=header,
                     metadata=rec['result_metadata'],
                 )
                 md.update(self._extract_metadata(md_msg))
                 if rec.get('received'):
                     md['received'] = rec['received']
                 md.update(iodict)
                 
                 if rcontent['status'] == 'ok':
                     if header['msg_type'] == 'apply_reply':
                         res,buffers = serialize.unserialize_object(buffers)
                     elif header['msg_type'] == 'execute_reply':
                         res = ExecuteReply(msg_id, rcontent, md)
                     else:
                         raise KeyError("unhandled msg type: %r" % header['msg_type'])
                 else:
                     res = self._unwrap_exception(rcontent)
                     failures.append(res)
 
                 self.results[msg_id] = res
                 content[msg_id] = res
 
         if len(theids) == 1 and failures:
             raise failures[0]
 
         error.collect_exceptions(failures, "result_status")
         return content
 
     @spin_first
     def queue_status(self, targets='all', verbose=False):
         """Fetch the status of engine queues.
 
         Parameters
         ----------
 
         targets : int/str/list of ints/strs
                 the engines whose states are to be queried.
                 default : all
         verbose : bool
                 Whether to return lengths only, or lists of ids for each element
         """
         if targets == 'all':
             # allow 'all' to be evaluated on the engine
             engine_ids = None
         else:
             engine_ids = self._build_targets(targets)[1]
         content = dict(targets=engine_ids, verbose=verbose)
         self.session.send(self._query_socket, "queue_request", content=content)
         idents,msg = self.session.recv(self._query_socket, 0)
         if self.debug:
             pprint(msg)
         content = msg['content']
         status = content.pop('status')
         if status != 'ok':
             raise self._unwrap_exception(content)
         content = rekey(content)
         if isinstance(targets, int):
             return content[targets]
         else:
             return content
 
     def _build_msgids_from_target(self, targets=None):
         """Build a list of msg_ids from the list of engine targets"""
         if not targets: # needed as _build_targets otherwise uses all engines
             return []
         target_ids = self._build_targets(targets)[0] 
         return filter(lambda md_id: self.metadata[md_id]["engine_uuid"] in target_ids, self.metadata)
     
     def _build_msgids_from_jobs(self, jobs=None):
         """Build a list of msg_ids from "jobs" """
         if not jobs:
             return []
         msg_ids = []
         if isinstance(jobs, string_types + (AsyncResult,)):
             jobs = [jobs]
         bad_ids = filter(lambda obj: not isinstance(obj, string_types + (AsyncResult)), jobs)
         if bad_ids:
             raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
         for j in jobs:
             if isinstance(j, AsyncResult):
                 msg_ids.extend(j.msg_ids)
             else:
                 msg_ids.append(j)
         return msg_ids        
         
     def purge_local_results(self, jobs=[], targets=[]):
         """Clears the client caches of results and frees such memory.
         
         Individual results can be purged by msg_id, or the entire
         history of specific targets can be purged.
 
         Use `purge_local_results('all')` to scrub everything from the Clients's db.
 
         The client must have no outstanding tasks before purging the caches.
         Raises `AssertionError` if there are still outstanding tasks.
 
         After this call all `AsyncResults` are invalid and should be discarded.
 
         If you must "reget" the results, you can still do so by using
         `client.get_result(msg_id)` or `client.get_result(asyncresult)`. This will
         redownload the results from the hub if they are still available
         (i.e `client.purge_hub_results(...)` has not been called.        
 
         Parameters
         ----------
 
         jobs : str or list of str or AsyncResult objects
                 the msg_ids whose results should be purged.
         targets : int/str/list of ints/strs
                 The targets, by int_id, whose entire results are to be purged.
 
                 default : None
         """
         assert not self.outstanding, "Can't purge a client with outstanding tasks!"
         
         if not targets and not jobs:
             raise ValueError("Must specify at least one of `targets` and `jobs`")
                 
         if jobs == 'all':
             self.results.clear()
             self.metadata.clear()
             return
         else:
             msg_ids = []
             msg_ids.extend(self._build_msgids_from_target(targets))
             msg_ids.extend(self._build_msgids_from_jobs(jobs))
             map(self.results.pop, msg_ids)
             map(self.metadata.pop, msg_ids)
 
 
     @spin_first
     def purge_hub_results(self, jobs=[], targets=[]):
         """Tell the Hub to forget results.
 
         Individual results can be purged by msg_id, or the entire
         history of specific targets can be purged.
 
         Use `purge_results('all')` to scrub everything from the Hub's db.
 
         Parameters
         ----------
 
         jobs : str or list of str or AsyncResult objects
                 the msg_ids whose results should be forgotten.
         targets : int/str/list of ints/strs
                 The targets, by int_id, whose entire history is to be purged.
 
                 default : None
         """
         if not targets and not jobs:
             raise ValueError("Must specify at least one of `targets` and `jobs`")
         if targets:
             targets = self._build_targets(targets)[1]
 
         # construct msg_ids from jobs
         if jobs == 'all':
             msg_ids = jobs
         else:
             msg_ids = self._build_msgids_from_jobs(jobs)
 
         content = dict(engine_ids=targets, msg_ids=msg_ids)
         self.session.send(self._query_socket, "purge_request", content=content)
         idents, msg = self.session.recv(self._query_socket, 0)
         if self.debug:
             pprint(msg)
         content = msg['content']
         if content['status'] != 'ok':
             raise self._unwrap_exception(content)
 
     def purge_results(self,  jobs=[], targets=[]):
         """Clears the cached results from both the hub and the local client
                 
         Individual results can be purged by msg_id, or the entire
         history of specific targets can be purged.
 
         Use `purge_results('all')` to scrub every cached result from both the Hub's and 
         the Client's db.
         
         Equivalent to calling both `purge_hub_results()` and `purge_client_results()` with 
         the same arguments.
 
         Parameters
         ----------
 
         jobs : str or list of str or AsyncResult objects
                 the msg_ids whose results should be forgotten.
         targets : int/str/list of ints/strs
                 The targets, by int_id, whose entire history is to be purged.
 
                 default : None
         """
         self.purge_local_results(jobs=jobs, targets=targets)
         self.purge_hub_results(jobs=jobs, targets=targets)
 
     def purge_everything(self):
         """Clears all content from previous Tasks from both the hub and the local client
         
         In addition to calling `purge_results("all")` it also deletes the history and 
         other bookkeeping lists.        
         """
         self.purge_results("all")
         self.history = []
         self.session.digest_history.clear()
 
     @spin_first
     def hub_history(self):
         """Get the Hub's history
 
         Just like the Client, the Hub has a history, which is a list of msg_ids.
         This will contain the history of all clients, and, depending on configuration,
         may contain history across multiple cluster sessions.
 
         Any msg_id returned here is a valid argument to `get_result`.
 
         Returns
         -------
 
         msg_ids : list of strs
                 list of all msg_ids, ordered by task submission time.
         """
 
         self.session.send(self._query_socket, "history_request", content={})
         idents, msg = self.session.recv(self._query_socket, 0)
 
         if self.debug:
             pprint(msg)
         content = msg['content']
         if content['status'] != 'ok':
             raise self._unwrap_exception(content)
         else:
             return content['history']
 
     @spin_first
     def db_query(self, query, keys=None):
         """Query the Hub's TaskRecord database
 
         This will return a list of task record dicts that match `query`
 
         Parameters
         ----------
 
         query : mongodb query dict
             The search dict. See mongodb query docs for details.
         keys : list of strs [optional]
             The subset of keys to be returned.  The default is to fetch everything but buffers.
             'msg_id' will *always* be included.
         """
         if isinstance(keys, string_types):
             keys = [keys]
         content = dict(query=query, keys=keys)
         self.session.send(self._query_socket, "db_request", content=content)
         idents, msg = self.session.recv(self._query_socket, 0)
         if self.debug:
             pprint(msg)
         content = msg['content']
         if content['status'] != 'ok':
             raise self._unwrap_exception(content)
 
         records = content['records']
 
         buffer_lens = content['buffer_lens']
         result_buffer_lens = content['result_buffer_lens']
         buffers = msg['buffers']
         has_bufs = buffer_lens is not None
         has_rbufs = result_buffer_lens is not None
         for i,rec in enumerate(records):
             # relink buffers
             if has_bufs:
                 blen = buffer_lens[i]
                 rec['buffers'], buffers = buffers[:blen],buffers[blen:]
             if has_rbufs:
                 blen = result_buffer_lens[i]
                 rec['result_buffers'], buffers = buffers[:blen],buffers[blen:]
 
         return records
 
 __all__ = [ 'Client' ]

IPython/parallel/util.py

@@ -1,369 +1,369 b''
 """some generic utilities for dealing with classes, urls, and serialization
 
 Authors:
 
 * Min RK
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Standard library imports.
 import logging
 import os
 import re
 import stat
 import socket
 import sys
 from signal import signal, SIGINT, SIGABRT, SIGTERM
 try:
     from signal import SIGKILL
 except ImportError:
     SIGKILL=None
 
 try:
     import cPickle
     pickle = cPickle
 except:
     cPickle = None
     import pickle
 
 # System library imports
 import zmq
 from zmq.log import handlers
 
 from IPython.external.decorator import decorator
 
 # IPython imports
 from IPython.config.application import Application
 from IPython.utils.localinterfaces import localhost, is_public_ip, public_ips
 from IPython.utils.py3compat import string_types
 from IPython.kernel.zmq.log import EnginePUBHandler
 from IPython.kernel.zmq.serialize import (
     unserialize_object, serialize_object, pack_apply_message, unpack_apply_message
 )
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 class Namespace(dict):
     """Subclass of dict for attribute access to keys."""
     
     def __getattr__(self, key):
         """getattr aliased to getitem"""
         if key in self.iterkeys():
             return self[key]
         else:
             raise NameError(key)
 
     def __setattr__(self, key, value):
         """setattr aliased to setitem, with strict"""
         if hasattr(dict, key):
             raise KeyError("Cannot override dict keys %r"%key)
         self[key] = value
     
 
 class ReverseDict(dict):
     """simple double-keyed subset of dict methods."""
     
     def __init__(self, *args, **kwargs):
         dict.__init__(self, *args, **kwargs)
         self._reverse = dict()
         for key, value in self.iteritems():
             self._reverse[value] = key
     
     def __getitem__(self, key):
         try:
             return dict.__getitem__(self, key)
         except KeyError:
             return self._reverse[key]
     
     def __setitem__(self, key, value):
         if key in self._reverse:
             raise KeyError("Can't have key %r on both sides!"%key)
         dict.__setitem__(self, key, value)
         self._reverse[value] = key
     
     def pop(self, key):
         value = dict.pop(self, key)
         self._reverse.pop(value)
         return value
     
     def get(self, key, default=None):
         try:
             return self[key]
         except KeyError:
             return default
 
 #-----------------------------------------------------------------------------
 # Functions
 #-----------------------------------------------------------------------------
 
 @decorator
 def log_errors(f, self, *args, **kwargs):
     """decorator to log unhandled exceptions raised in a method.
     
     For use wrapping on_recv callbacks, so that exceptions
     do not cause the stream to be closed.
     """
     try:
         return f(self, *args, **kwargs)
     except Exception:
         self.log.error("Uncaught exception in %r" % f, exc_info=True)
     
 
 def is_url(url):
     """boolean check for whether a string is a zmq url"""
     if '://' not in url:
         return False
     proto, addr = url.split('://', 1)
     if proto.lower() not in ['tcp','pgm','epgm','ipc','inproc']:
         return False
     return True
 
 def validate_url(url):
     """validate a url for zeromq"""
     if not isinstance(url, string_types):
         raise TypeError("url must be a string, not %r"%type(url))
     url = url.lower()
     
     proto_addr = url.split('://')
     assert len(proto_addr) == 2, 'Invalid url: %r'%url
     proto, addr = proto_addr
     assert proto in ['tcp','pgm','epgm','ipc','inproc'], "Invalid protocol: %r"%proto
     
     # domain pattern adapted from http://www.regexlib.com/REDetails.aspx?regexp_id=391
     # author: Remi Sabourin
     pat = re.compile(r'^([\w\d]([\w\d\-]{0,61}[\w\d])?\.)*[\w\d]([\w\d\-]{0,61}[\w\d])?$')
     
     if proto == 'tcp':
         lis = addr.split(':')
         assert len(lis) == 2, 'Invalid url: %r'%url
         addr,s_port = lis
         try:
             port = int(s_port)
         except ValueError:
             raise AssertionError("Invalid port %r in url: %r"%(port, url))
         
         assert addr == '*' or pat.match(addr) is not None, 'Invalid url: %r'%url
         
     else:
         # only validate tcp urls currently
         pass
     
     return True
 
 
 def validate_url_container(container):
     """validate a potentially nested collection of urls."""
     if isinstance(container, string_types):
         url = container
         return validate_url(url)
     elif isinstance(container, dict):
         container = container.itervalues()
     
     for element in container:
         validate_url_container(element)
 
 
 def split_url(url):
     """split a zmq url (tcp://ip:port) into ('tcp','ip','port')."""
     proto_addr = url.split('://')
     assert len(proto_addr) == 2, 'Invalid url: %r'%url
     proto, addr = proto_addr
     lis = addr.split(':')
     assert len(lis) == 2, 'Invalid url: %r'%url
     addr,s_port = lis
     return proto,addr,s_port
     
 def disambiguate_ip_address(ip, location=None):
     """turn multi-ip interfaces '0.0.0.0' and '*' into connectable
     ones, based on the location (default interpretation of location is localhost)."""
     if ip in ('0.0.0.0', '*'):
         if location is None or is_public_ip(location) or not public_ips():
             # If location is unspecified or cannot be determined, assume local
             ip = localhost()
         elif location:
             return location
     return ip
 
 def disambiguate_url(url, location=None):
     """turn multi-ip interfaces '0.0.0.0' and '*' into connectable
     ones, based on the location (default interpretation is localhost).
     
     This is for zeromq urls, such as tcp://*:10101."""
     try:
         proto,ip,port = split_url(url)
     except AssertionError:
         # probably not tcp url; could be ipc, etc.
         return url
     
     ip = disambiguate_ip_address(ip,location)
     
     return "%s://%s:%s"%(proto,ip,port)
 
 
 #--------------------------------------------------------------------------
 # helpers for implementing old MEC API via view.apply
 #--------------------------------------------------------------------------
 
 def interactive(f):
     """decorator for making functions appear as interactively defined.
     This results in the function being linked to the user_ns as globals()
     instead of the module globals().
     """
     f.__module__ = '__main__'
     return f
 
 @interactive
 def _push(**ns):
     """helper method for implementing `client.push` via `client.apply`"""
     user_ns = globals()
     tmp = '_IP_PUSH_TMP_'
     while tmp in user_ns:
         tmp = tmp + '_'
     try:
         for name, value in ns.iteritems():
             user_ns[tmp] = value
             exec("%s = %s" % (name, tmp), user_ns)
     finally:
         user_ns.pop(tmp, None)
 
 @interactive
 def _pull(keys):
     """helper method for implementing `client.pull` via `client.apply`"""
     if isinstance(keys, (list,tuple, set)):
         return map(lambda key: eval(key, globals()), keys)
     else:
         return eval(keys, globals())
 
 @interactive
 def _execute(code):
     """helper method for implementing `client.execute` via `client.apply`"""
     exec(code, globals())
 
 #--------------------------------------------------------------------------
 # extra process management utilities
 #--------------------------------------------------------------------------
 
 _random_ports = set()
 
 def select_random_ports(n):
     """Selects and return n random ports that are available."""
     ports = []
-    for i in xrange(n):
+    for i in range(n):
         sock = socket.socket()
         sock.bind(('', 0))
         while sock.getsockname()[1] in _random_ports:
             sock.close()
             sock = socket.socket()
             sock.bind(('', 0))
         ports.append(sock)
     for i, sock in enumerate(ports):
         port = sock.getsockname()[1]
         sock.close()
         ports[i] = port
         _random_ports.add(port)
     return ports
 
 def signal_children(children):
     """Relay interupt/term signals to children, for more solid process cleanup."""
     def terminate_children(sig, frame):
         log = Application.instance().log
         log.critical("Got signal %i, terminating children..."%sig)
         for child in children:
             child.terminate()
         
         sys.exit(sig != SIGINT)
         # sys.exit(sig)
     for sig in (SIGINT, SIGABRT, SIGTERM):
         signal(sig, terminate_children)
 
 def generate_exec_key(keyfile):
     import uuid
     newkey = str(uuid.uuid4())
     with open(keyfile, 'w') as f:
         # f.write('ipython-key ')
         f.write(newkey+'\n')
     # set user-only RW permissions (0600)
     # this will have no effect on Windows
     os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
 
 
 def integer_loglevel(loglevel):
     try:
         loglevel = int(loglevel)
     except ValueError:
         if isinstance(loglevel, str):
             loglevel = getattr(logging, loglevel)
     return loglevel
 
 def connect_logger(logname, context, iface, root="ip", loglevel=logging.DEBUG):
     logger = logging.getLogger(logname)
     if any([isinstance(h, handlers.PUBHandler) for h in logger.handlers]):
         # don't add a second PUBHandler
         return
     loglevel = integer_loglevel(loglevel)
     lsock = context.socket(zmq.PUB)
     lsock.connect(iface)
     handler = handlers.PUBHandler(lsock)
     handler.setLevel(loglevel)
     handler.root_topic = root
     logger.addHandler(handler)
     logger.setLevel(loglevel)
 
 def connect_engine_logger(context, iface, engine, loglevel=logging.DEBUG):
     logger = logging.getLogger()
     if any([isinstance(h, handlers.PUBHandler) for h in logger.handlers]):
         # don't add a second PUBHandler
         return
     loglevel = integer_loglevel(loglevel)
     lsock = context.socket(zmq.PUB)
     lsock.connect(iface)
     handler = EnginePUBHandler(engine, lsock)
     handler.setLevel(loglevel)
     logger.addHandler(handler)
     logger.setLevel(loglevel)
     return logger
 
 def local_logger(logname, loglevel=logging.DEBUG):
     loglevel = integer_loglevel(loglevel)
     logger = logging.getLogger(logname)
     if any([isinstance(h, logging.StreamHandler) for h in logger.handlers]):
         # don't add a second StreamHandler
         return
     handler = logging.StreamHandler()
     handler.setLevel(loglevel)
     formatter = logging.Formatter("%(asctime)s.%(msecs).03d [%(name)s] %(message)s",
                 datefmt="%Y-%m-%d %H:%M:%S")
     handler.setFormatter(formatter)
 
     logger.addHandler(handler)
     logger.setLevel(loglevel)
     return logger
 
 def set_hwm(sock, hwm=0):
     """set zmq High Water Mark on a socket
     
     in a way that always works for various pyzmq / libzmq versions.
     """
     import zmq
     
     for key in ('HWM', 'SNDHWM', 'RCVHWM'):
         opt = getattr(zmq, key, None)
         if opt is None:
             continue
         try:
             sock.setsockopt(opt, hwm)
         except zmq.ZMQError:
             pass
 
         
  No newline at end of file

IPython/qt/console/ansi_code_processor.py

@@ -1,378 +1,378 b''
 """ Utilities for processing ANSI escape codes and special ASCII characters.
 """
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Standard library imports
 from collections import namedtuple
 import re
 
 # System library imports
 from IPython.external.qt import QtGui
 
 # Local imports
 from IPython.utils.py3compat import string_types
 
 #-----------------------------------------------------------------------------
 # Constants and datatypes
 #-----------------------------------------------------------------------------
 
 # An action for erase requests (ED and EL commands).
 EraseAction = namedtuple('EraseAction', ['action', 'area', 'erase_to'])
 
 # An action for cursor move requests (CUU, CUD, CUF, CUB, CNL, CPL, CHA, CUP,
 # and HVP commands).
 # FIXME: Not implemented in AnsiCodeProcessor.
 MoveAction = namedtuple('MoveAction', ['action', 'dir', 'unit', 'count'])
 
 # An action for scroll requests (SU and ST) and form feeds.
 ScrollAction = namedtuple('ScrollAction', ['action', 'dir', 'unit', 'count'])
 
 # An action for the carriage return character
 CarriageReturnAction = namedtuple('CarriageReturnAction', ['action'])
 
 # An action for the \n character
 NewLineAction = namedtuple('NewLineAction', ['action'])
 
 # An action for the beep character
 BeepAction = namedtuple('BeepAction', ['action'])
 
 # An action for backspace
 BackSpaceAction = namedtuple('BackSpaceAction', ['action'])
 
 # Regular expressions.
 CSI_COMMANDS = 'ABCDEFGHJKSTfmnsu'
 CSI_SUBPATTERN = '\[(.*?)([%s])' % CSI_COMMANDS
 OSC_SUBPATTERN = '\](.*?)[\x07\x1b]'
 ANSI_PATTERN = ('\x01?\x1b(%s|%s)\x02?' % \
                 (CSI_SUBPATTERN, OSC_SUBPATTERN))
 ANSI_OR_SPECIAL_PATTERN = re.compile('(\a|\b|\r(?!\n)|\r?\n)|(?:%s)' % ANSI_PATTERN)
 SPECIAL_PATTERN = re.compile('([\f])')
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 class AnsiCodeProcessor(object):
     """ Translates special ASCII characters and ANSI escape codes into readable
         attributes. It also supports a few non-standard, xterm-specific codes.
     """
 
     # Whether to increase intensity or set boldness for SGR code 1.
     # (Different terminals handle this in different ways.)
     bold_text_enabled = False
 
     # We provide an empty default color map because subclasses will likely want
     # to use a custom color format.
     default_color_map = {}
 
     #---------------------------------------------------------------------------
     # AnsiCodeProcessor interface
     #---------------------------------------------------------------------------
 
     def __init__(self):
         self.actions = []
         self.color_map = self.default_color_map.copy()
         self.reset_sgr()
 
     def reset_sgr(self):
         """ Reset graphics attributs to their default values.
         """
         self.intensity = 0
         self.italic = False
         self.bold = False
         self.underline = False
         self.foreground_color = None
         self.background_color = None
 
     def split_string(self, string):
         """ Yields substrings for which the same escape code applies.
         """
         self.actions = []
         start = 0
 
         # strings ending with \r are assumed to be ending in \r\n since
         # \n is appended to output strings automatically.  Accounting
         # for that, here.
         last_char = '\n' if len(string) > 0 and string[-1] == '\n' else None
         string = string[:-1] if last_char is not None else string
 
         for match in ANSI_OR_SPECIAL_PATTERN.finditer(string):
             raw = string[start:match.start()]
             substring = SPECIAL_PATTERN.sub(self._replace_special, raw)
             if substring or self.actions:
                 yield substring
                 self.actions = []
             start = match.end()
 
             groups = filter(lambda x: x is not None, match.groups())
             g0 = groups[0]
             if g0 == '\a':
                 self.actions.append(BeepAction('beep'))
                 yield None
                 self.actions = []
             elif g0 == '\r':
                 self.actions.append(CarriageReturnAction('carriage-return'))
                 yield None
                 self.actions = []
             elif g0 == '\b':
                 self.actions.append(BackSpaceAction('backspace'))
                 yield None
                 self.actions = []
             elif g0 == '\n' or g0 == '\r\n':
                 self.actions.append(NewLineAction('newline'))
                 yield g0
                 self.actions = []
             else:
                 params = [ param for param in groups[1].split(';') if param ]
                 if g0.startswith('['):
                     # Case 1: CSI code.
                     try:
                         params = map(int, params)
                     except ValueError:
                         # Silently discard badly formed codes.
                         pass
                     else:
                         self.set_csi_code(groups[2], params)
 
                 elif g0.startswith(']'):
                     # Case 2: OSC code.
                     self.set_osc_code(params)
 
         raw = string[start:]
         substring = SPECIAL_PATTERN.sub(self._replace_special, raw)
         if substring or self.actions:
             yield substring
 
         if last_char is not None:
             self.actions.append(NewLineAction('newline'))
             yield last_char
 
     def set_csi_code(self, command, params=[]):
         """ Set attributes based on CSI (Control Sequence Introducer) code.
 
         Parameters
         ----------
         command : str
             The code identifier, i.e. the final character in the sequence.
 
         params : sequence of integers, optional
             The parameter codes for the command.
         """
         if command == 'm':   # SGR - Select Graphic Rendition
             if params:
                 self.set_sgr_code(params)
             else:
                 self.set_sgr_code([0])
 
         elif (command == 'J' or # ED - Erase Data
               command == 'K'):  # EL - Erase in Line
             code = params[0] if params else 0
             if 0 <= code <= 2:
                 area = 'screen' if command == 'J' else 'line'
                 if code == 0:
                     erase_to = 'end'
                 elif code == 1:
                     erase_to = 'start'
                 elif code == 2:
                     erase_to = 'all'
                 self.actions.append(EraseAction('erase', area, erase_to))
 
         elif (command == 'S' or # SU - Scroll Up
               command == 'T'):  # SD - Scroll Down
             dir = 'up' if command == 'S' else 'down'
             count = params[0] if params else 1
             self.actions.append(ScrollAction('scroll', dir, 'line', count))
 
     def set_osc_code(self, params):
         """ Set attributes based on OSC (Operating System Command) parameters.
 
         Parameters
         ----------
         params : sequence of str
             The parameters for the command.
         """
         try:
             command = int(params.pop(0))
         except (IndexError, ValueError):
             return
 
         if command == 4:
             # xterm-specific: set color number to color spec.
             try:
                 color = int(params.pop(0))
                 spec = params.pop(0)
                 self.color_map[color] = self._parse_xterm_color_spec(spec)
             except (IndexError, ValueError):
                 pass
 
     def set_sgr_code(self, params):
         """ Set attributes based on SGR (Select Graphic Rendition) codes.
 
         Parameters
         ----------
         params : sequence of ints
             A list of SGR codes for one or more SGR commands. Usually this
             sequence will have one element per command, although certain
             xterm-specific commands requires multiple elements.
         """
         # Always consume the first parameter.
         if not params:
             return
         code = params.pop(0)
 
         if code == 0:
             self.reset_sgr()
         elif code == 1:
             if self.bold_text_enabled:
                 self.bold = True
             else:
                 self.intensity = 1
         elif code == 2:
             self.intensity = 0
         elif code == 3:
             self.italic = True
         elif code == 4:
             self.underline = True
         elif code == 22:
             self.intensity = 0
             self.bold = False
         elif code == 23:
             self.italic = False
         elif code == 24:
             self.underline = False
         elif code >= 30 and code <= 37:
             self.foreground_color = code - 30
         elif code == 38 and params and params.pop(0) == 5:
             # xterm-specific: 256 color support.
             if params:
                 self.foreground_color = params.pop(0)
         elif code == 39:
             self.foreground_color = None
         elif code >= 40 and code <= 47:
             self.background_color = code - 40
         elif code == 48 and params and params.pop(0) == 5:
             # xterm-specific: 256 color support.
             if params:
                 self.background_color = params.pop(0)
         elif code == 49:
             self.background_color = None
 
         # Recurse with unconsumed parameters.
         self.set_sgr_code(params)
 
     #---------------------------------------------------------------------------
     # Protected interface
     #---------------------------------------------------------------------------
 
     def _parse_xterm_color_spec(self, spec):
         if spec.startswith('rgb:'):
             return tuple(map(lambda x: int(x, 16), spec[4:].split('/')))
         elif spec.startswith('rgbi:'):
             return tuple(map(lambda x: int(float(x) * 255),
                              spec[5:].split('/')))
         elif spec == '?':
             raise ValueError('Unsupported xterm color spec')
         return spec
 
     def _replace_special(self, match):
         special = match.group(1)
         if special == '\f':
             self.actions.append(ScrollAction('scroll', 'down', 'page', 1))
         return ''
 
 
 class QtAnsiCodeProcessor(AnsiCodeProcessor):
     """ Translates ANSI escape codes into QTextCharFormats.
     """
 
     # A map from ANSI color codes to SVG color names or RGB(A) tuples.
     darkbg_color_map = {
         0  : 'black',       # black
         1  : 'darkred',     # red
         2  : 'darkgreen',   # green
         3  : 'brown',       # yellow
         4  : 'darkblue',    # blue
         5  : 'darkviolet',  # magenta
         6  : 'steelblue',   # cyan
         7  : 'grey',        # white
         8  : 'grey',        # black (bright)
         9  : 'red',         # red (bright)
         10 : 'lime',        # green (bright)
         11 : 'yellow',      # yellow (bright)
         12 : 'deepskyblue', # blue (bright)
         13 : 'magenta',     # magenta (bright)
         14 : 'cyan',        # cyan (bright)
         15 : 'white' }      # white (bright)
 
     # Set the default color map for super class.
     default_color_map = darkbg_color_map.copy()
 
     def get_color(self, color, intensity=0):
         """ Returns a QColor for a given color code, or None if one cannot be
             constructed.
         """
         if color is None:
             return None
 
         # Adjust for intensity, if possible.
         if color < 8 and intensity > 0:
             color += 8
 
         constructor = self.color_map.get(color, None)
         if isinstance(constructor, string_types):
             # If this is an X11 color name, we just hope there is a close SVG
             # color name. We could use QColor's static method
             # 'setAllowX11ColorNames()', but this is global and only available
             # on X11. It seems cleaner to aim for uniformity of behavior.
             return QtGui.QColor(constructor)
 
         elif isinstance(constructor, (tuple, list)):
             return QtGui.QColor(*constructor)
 
         return None
 
     def get_format(self):
         """ Returns a QTextCharFormat that encodes the current style attributes.
         """
         format = QtGui.QTextCharFormat()
 
         # Set foreground color
         qcolor = self.get_color(self.foreground_color, self.intensity)
         if qcolor is not None:
             format.setForeground(qcolor)
 
         # Set background color
         qcolor = self.get_color(self.background_color, self.intensity)
         if qcolor is not None:
             format.setBackground(qcolor)
 
         # Set font weight/style options
         if self.bold:
             format.setFontWeight(QtGui.QFont.Bold)
         else:
             format.setFontWeight(QtGui.QFont.Normal)
         format.setFontItalic(self.italic)
         format.setFontUnderline(self.underline)
 
         return format
 
     def set_background_color(self, color):
         """ Given a background color (a QColor), attempt to set a color map
             that will be aesthetically pleasing.
         """
         # Set a new default color map.
         self.default_color_map = self.darkbg_color_map.copy()
 
         if color.value() >= 127:
             # Colors appropriate for a terminal with a light background. For
             # now, only use non-bright colors...
-            for i in xrange(8):
+            for i in range(8):
                 self.default_color_map[i + 8] = self.default_color_map[i]
 
             # ...and replace white with black.
             self.default_color_map[7] = self.default_color_map[15] = 'black'
 
         # Update the current color map with the new defaults.
         self.color_map.update(self.default_color_map)

IPython/utils/data.py

@@ -1,35 +1,37 b''
 # encoding: utf-8
 """Utilities for working with data structures like lists, dicts and tuples.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
+from .py3compat import xrange
+
 def uniq_stable(elems):
     """uniq_stable(elems) -> list
 
     Return from an iterable, a list of all the unique elements in the input,
     but maintaining the order in which they first appear.
 
     Note: All elements in the input must be hashable for this routine
     to work, as it internally uses a set for efficiency reasons.
     """
     seen = set()
     return [x for x in elems if x not in seen and not seen.add(x)]
 
 
 def flatten(seq):
     """Flatten a list of lists (NOT recursive, only works for 2d lists)."""
 
     return [x for subseq in seq for x in subseq]
     
 
 def chop(seq, size):
     """Chop a sequence into chunks of the given size."""
     return [seq[i:i+size] for i in xrange(0,len(seq),size)]
 
 

IPython/utils/py3compat.py

@@ -1,207 +1,210 b''
 # coding: utf-8
 """Compatibility tricks for Python 3. Mainly to do with unicode."""
 import functools
 import sys
 import re
 import types
 
 from .encoding import DEFAULT_ENCODING
 
 orig_open = open
 
 def no_code(x, encoding=None):
     return x
 
 def decode(s, encoding=None):
     encoding = encoding or DEFAULT_ENCODING
     return s.decode(encoding, "replace")
 
 def encode(u, encoding=None):
     encoding = encoding or DEFAULT_ENCODING
     return u.encode(encoding, "replace")
 
 
 def cast_unicode(s, encoding=None):
     if isinstance(s, bytes):
         return decode(s, encoding)
     return s
 
 def cast_bytes(s, encoding=None):
     if not isinstance(s, bytes):
         return encode(s, encoding)
     return s
 
 def _modify_str_or_docstring(str_change_func):
     @functools.wraps(str_change_func)
     def wrapper(func_or_str):
         if isinstance(func_or_str, string_types):
             func = None
             doc = func_or_str
         else:
             func = func_or_str
             doc = func.__doc__
         
         doc = str_change_func(doc)
         
         if func:
             func.__doc__ = doc
             return func
         return doc
     return wrapper
 
 def safe_unicode(e):
     """unicode(e) with various fallbacks. Used for exceptions, which may not be
     safe to call unicode() on.
     """
     try:
         return unicode_type(e)
     except UnicodeError:
         pass
 
     try:
         return str_to_unicode(str(e))
     except UnicodeError:
         pass
 
     try:
         return str_to_unicode(repr(e))
     except UnicodeError:
         pass
 
     return u'Unrecoverably corrupt evalue'
 
 if sys.version_info[0] >= 3:
     PY3 = True
     
     input = input
     builtin_mod_name = "builtins"
     import builtins as builtin_mod
     
     str_to_unicode = no_code
     unicode_to_str = no_code
     str_to_bytes = encode
     bytes_to_str = decode
     cast_bytes_py2 = no_code
     
     string_types = (str,)
     unicode_type = str
     
     def isidentifier(s, dotted=False):
         if dotted:
             return all(isidentifier(a) for a in s.split("."))
         return s.isidentifier()
     
     open = orig_open
+    xrange = range
     
     MethodType = types.MethodType
     
     def execfile(fname, glob, loc=None):
         loc = loc if (loc is not None) else glob
         with open(fname, 'rb') as f:
             exec(compile(f.read(), fname, 'exec'), glob, loc)
     
     # Refactor print statements in doctests.
     _print_statement_re = re.compile(r"\bprint (?P<expr>.*)$", re.MULTILINE)
     def _print_statement_sub(match):
         expr = match.groups('expr')
         return "print(%s)" % expr
     
     @_modify_str_or_docstring
     def doctest_refactor_print(doc):
         """Refactor 'print x' statements in a doctest to print(x) style. 2to3
         unfortunately doesn't pick up on our doctests.
         
         Can accept a string or a function, so it can be used as a decorator."""
         return _print_statement_re.sub(_print_statement_sub, doc)
     
     # Abstract u'abc' syntax:
     @_modify_str_or_docstring
     def u_format(s):
         """"{u}'abc'" --> "'abc'" (Python 3)
         
         Accepts a string or a function, so it can be used as a decorator."""
         return s.format(u='')
 
 else:
     PY3 = False
     
     input = raw_input
     builtin_mod_name = "__builtin__"
     import __builtin__ as builtin_mod
     
     str_to_unicode = decode
     unicode_to_str = encode
     str_to_bytes = no_code
     bytes_to_str = no_code
     cast_bytes_py2 = cast_bytes
     
     string_types = (str, unicode)
     unicode_type = unicode
     
     import re
     _name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$")
     def isidentifier(s, dotted=False):
         if dotted:
             return all(isidentifier(a) for a in s.split("."))
         return bool(_name_re.match(s))
     
     class open(object):
         """Wrapper providing key part of Python 3 open() interface."""
         def __init__(self, fname, mode="r", encoding="utf-8"):
             self.f = orig_open(fname, mode)
             self.enc = encoding
         
         def write(self, s):
             return self.f.write(s.encode(self.enc))
         
         def read(self, size=-1):
             return self.f.read(size).decode(self.enc)
         
         def close(self):
             return self.f.close()
         
         def __enter__(self):
             return self
         
         def __exit__(self, etype, value, traceback):
             self.f.close()
     
+    xrange = xrange
+
     def MethodType(func, instance):
         return types.MethodType(func, instance, type(instance))
     
     # don't override system execfile on 2.x:
     execfile = execfile
     
     def doctest_refactor_print(func_or_str):
         return func_or_str
 
 
     # Abstract u'abc' syntax:
     @_modify_str_or_docstring
     def u_format(s):
         """"{u}'abc'" --> "u'abc'" (Python 2)
         
         Accepts a string or a function, so it can be used as a decorator."""
         return s.format(u='u')
 
     if sys.platform == 'win32':
         def execfile(fname, glob=None, loc=None):
             loc = loc if (loc is not None) else glob
             # The rstrip() is necessary b/c trailing whitespace in files will
             # cause an IndentationError in Python 2.6 (this was fixed in 2.7,
             # but we still support 2.6).  See issue 1027.
             scripttext = builtin_mod.open(fname).read().rstrip() + '\n'
             # compile converts unicode filename to str assuming
             # ascii. Let's do the conversion before calling compile
             if isinstance(fname, unicode):
                 filename = unicode_to_str(fname)
             else:
                 filename = fname
             exec(compile(scripttext, filename, 'exec'), glob, loc)
     else:
         def execfile(fname, *where):
             if isinstance(fname, unicode):
                 filename = fname.encode(sys.getfilesystemencoding())
             else:
                 filename = fname
             builtin_mod.execfile(filename, *where)

IPython/utils/text.py

@@ -1,759 +1,758 b''
 # encoding: utf-8
 """
 Utilities for working with strings and text.
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.utils.text
    :parts: 3
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import re
 import sys
 import textwrap
 from string import Formatter
 
 from IPython.external.path import path
 from IPython.testing.skipdoctest import skip_doctest_py3, skip_doctest
 from IPython.utils import py3compat
 
-
 #-----------------------------------------------------------------------------
 # Declarations
 #-----------------------------------------------------------------------------
 
 # datetime.strftime date format for ipython
 if sys.platform == 'win32':
     date_format = "%B %d, %Y"
 else:
     date_format = "%B %-d, %Y"
 
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 class LSString(str):
     """String derivative with a special access attributes.
 
     These are normal strings, but with the special attributes:
 
         .l (or .list) : value as list (split on newlines).
         .n (or .nlstr): original value (the string itself).
         .s (or .spstr): value as whitespace-separated string.
         .p (or .paths): list of path objects
 
     Any values which require transformations are computed only once and
     cached.
 
     Such strings are very useful to efficiently interact with the shell, which
     typically only understands whitespace-separated options for commands."""
 
     def get_list(self):
         try:
             return self.__list
         except AttributeError:
             self.__list = self.split('\n')
             return self.__list
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = self.replace('\n',' ')
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         return self
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self.split('\n') if os.path.exists(p)]
             return self.__paths
 
     p = paths = property(get_paths)
 
 # FIXME: We need to reimplement type specific displayhook and then add this
 # back as a custom printer. This should also be moved outside utils into the
 # core.
 
 # def print_lsstring(arg):
 #     """ Prettier (non-repr-like) and more informative printer for LSString """
 #     print "LSString (.p, .n, .l, .s available). Value:"
 #     print arg
 #
 #
 # print_lsstring = result_display.when_type(LSString)(print_lsstring)
 
 
 class SList(list):
     """List derivative with a special access attributes.
 
     These are normal lists, but with the special attributes:
 
         .l (or .list) : value as list (the list itself).
         .n (or .nlstr): value as a string, joined on newlines.
         .s (or .spstr): value as a string, joined on spaces.
         .p (or .paths): list of path objects
 
     Any values which require transformations are computed only once and
     cached."""
 
     def get_list(self):
         return self
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = ' '.join(self)
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         try:
             return self.__nlstr
         except AttributeError:
             self.__nlstr = '\n'.join(self)
             return self.__nlstr
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self if os.path.exists(p)]
             return self.__paths
 
     p = paths = property(get_paths)
 
     def grep(self, pattern, prune = False, field = None):
         """ Return all strings matching 'pattern' (a regex or callable)
 
         This is case-insensitive. If prune is true, return all items
         NOT matching the pattern.
 
         If field is specified, the match must occur in the specified
         whitespace-separated field.
 
         Examples::
 
             a.grep( lambda x: x.startswith('C') )
             a.grep('Cha.*log', prune=1)
             a.grep('chm', field=-1)
         """
 
         def match_target(s):
             if field is None:
                 return s
             parts = s.split()
             try:
                 tgt = parts[field]
                 return tgt
             except IndexError:
                 return ""
 
         if isinstance(pattern, py3compat.string_types):
             pred = lambda x : re.search(pattern, x, re.IGNORECASE)
         else:
             pred = pattern
         if not prune:
             return SList([el for el in self if pred(match_target(el))])
         else:
             return SList([el for el in self if not pred(match_target(el))])
 
     def fields(self, *fields):
         """ Collect whitespace-separated fields from string list
 
         Allows quick awk-like usage of string lists.
 
         Example data (in var a, created by 'a = !ls -l')::
             -rwxrwxrwx  1 ville None      18 Dec 14  2006 ChangeLog
             drwxrwxrwx+ 6 ville None       0 Oct 24 18:05 IPython
 
         a.fields(0) is ['-rwxrwxrwx', 'drwxrwxrwx+']
         a.fields(1,0) is ['1 -rwxrwxrwx', '6 drwxrwxrwx+']
         (note the joining by space).
         a.fields(-1) is ['ChangeLog', 'IPython']
 
         IndexErrors are ignored.
 
         Without args, fields() just split()'s the strings.
         """
         if len(fields) == 0:
             return [el.split() for el in self]
 
         res = SList()
         for el in [f.split() for f in self]:
             lineparts = []
 
             for fd in fields:
                 try:
                     lineparts.append(el[fd])
                 except IndexError:
                     pass
             if lineparts:
                 res.append(" ".join(lineparts))
 
         return res
 
     def sort(self,field= None,  nums = False):
         """ sort by specified fields (see fields())
 
         Example::
             a.sort(1, nums = True)
 
         Sorts a by second field, in numerical order (so that 21 > 3)
 
         """
 
         #decorate, sort, undecorate
         if field is not None:
             dsu = [[SList([line]).fields(field),  line] for line in self]
         else:
             dsu = [[line,  line] for line in self]
         if nums:
             for i in range(len(dsu)):
                 numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
                 try:
                     n = int(numstr)
                 except ValueError:
                     n = 0;
                 dsu[i][0] = n
 
 
         dsu.sort()
         return SList([t[1] for t in dsu])
 
 
 # FIXME: We need to reimplement type specific displayhook and then add this
 # back as a custom printer. This should also be moved outside utils into the
 # core.
 
 # def print_slist(arg):
 #     """ Prettier (non-repr-like) and more informative printer for SList """
 #     print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
 #     if hasattr(arg,  'hideonce') and arg.hideonce:
 #         arg.hideonce = False
 #         return
 #
 #     nlprint(arg)   # This was a nested list printer, now removed.
 #
 # print_slist = result_display.when_type(SList)(print_slist)
 
 
 def indent(instr,nspaces=4, ntabs=0, flatten=False):
     """Indent a string a given number of spaces or tabstops.
 
     indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
 
     Parameters
     ----------
 
     instr : basestring
         The string to be indented.
     nspaces : int (default: 4)
         The number of spaces to be indented.
     ntabs : int (default: 0)
         The number of tabs to be indented.
     flatten : bool (default: False)
         Whether to scrub existing indentation.  If True, all lines will be
         aligned to the same indentation.  If False, existing indentation will
         be strictly increased.
 
     Returns
     -------
 
     str|unicode : string indented by ntabs and nspaces.
 
     """
     if instr is None:
         return
     ind = '\t'*ntabs+' '*nspaces
     if flatten:
         pat = re.compile(r'^\s*', re.MULTILINE)
     else:
         pat = re.compile(r'^', re.MULTILINE)
     outstr = re.sub(pat, ind, instr)
     if outstr.endswith(os.linesep+ind):
         return outstr[:-len(ind)]
     else:
         return outstr
 
 
 def list_strings(arg):
     """Always return a list of strings, given a string or list of strings
     as input.
 
     :Examples:
 
         In [7]: list_strings('A single string')
         Out[7]: ['A single string']
 
         In [8]: list_strings(['A single string in a list'])
         Out[8]: ['A single string in a list']
 
         In [9]: list_strings(['A','list','of','strings'])
         Out[9]: ['A', 'list', 'of', 'strings']
     """
 
     if isinstance(arg, py3compat.string_types): return [arg]
     else: return arg
 
 
 def marquee(txt='',width=78,mark='*'):
     """Return the input string centered in a 'marquee'.
 
     :Examples:
 
         In [16]: marquee('A test',40)
         Out[16]: '**************** A test ****************'
 
         In [17]: marquee('A test',40,'-')
         Out[17]: '---------------- A test ----------------'
 
         In [18]: marquee('A test',40,' ')
         Out[18]: '                 A test                 '
 
     """
     if not txt:
         return (mark*width)[:width]
     nmark = (width-len(txt)-2)//len(mark)//2
     if nmark < 0: nmark =0
     marks = mark*nmark
     return '%s %s %s' % (marks,txt,marks)
 
 
 ini_spaces_re = re.compile(r'^(\s+)')
 
 def num_ini_spaces(strng):
     """Return the number of initial spaces in a string"""
 
     ini_spaces = ini_spaces_re.match(strng)
     if ini_spaces:
         return ini_spaces.end()
     else:
         return 0
 
 
 def format_screen(strng):
     """Format a string for screen printing.
 
     This removes some latex-type format codes."""
     # Paragraph continue
     par_re = re.compile(r'\\$',re.MULTILINE)
     strng = par_re.sub('',strng)
     return strng
 
 
 def dedent(text):
     """Equivalent of textwrap.dedent that ignores unindented first line.
 
     This means it will still dedent strings like:
     '''foo
     is a bar
     '''
 
     For use in wrap_paragraphs.
     """
 
     if text.startswith('\n'):
         # text starts with blank line, don't ignore the first line
         return textwrap.dedent(text)
 
     # split first line
     splits = text.split('\n',1)
     if len(splits) == 1:
         # only one line
         return textwrap.dedent(text)
 
     first, rest = splits
     # dedent everything but the first line
     rest = textwrap.dedent(rest)
     return '\n'.join([first, rest])
 
 
 def wrap_paragraphs(text, ncols=80):
     """Wrap multiple paragraphs to fit a specified width.
 
     This is equivalent to textwrap.wrap, but with support for multiple
     paragraphs, as separated by empty lines.
 
     Returns
     -------
 
     list of complete paragraphs, wrapped to fill `ncols` columns.
     """
     paragraph_re = re.compile(r'\n(\s*\n)+', re.MULTILINE)
     text = dedent(text).strip()
     paragraphs = paragraph_re.split(text)[::2] # every other entry is space
     out_ps = []
     indent_re = re.compile(r'\n\s+', re.MULTILINE)
     for p in paragraphs:
         # presume indentation that survives dedent is meaningful formatting,
         # so don't fill unless text is flush.
         if indent_re.search(p) is None:
             # wrap paragraph
             p = textwrap.fill(p, ncols)
         out_ps.append(p)
     return out_ps
 
 
 def long_substr(data):
     """Return the longest common substring in a list of strings.
     
     Credit: http://stackoverflow.com/questions/2892931/longest-common-substring-from-more-than-two-strings-python
     """
     substr = ''
     if len(data) > 1 and len(data[0]) > 0:
         for i in range(len(data[0])):
             for j in range(len(data[0])-i+1):
                 if j > len(substr) and all(data[0][i:i+j] in x for x in data):
                     substr = data[0][i:i+j]
     elif len(data) == 1:
         substr = data[0]
     return substr
 
 
 def strip_email_quotes(text):
     """Strip leading email quotation characters ('>').
 
     Removes any combination of leading '>' interspersed with whitespace that
     appears *identically* in all lines of the input text.
 
     Parameters
     ----------
     text : str
 
     Examples
     --------
 
     Simple uses::
 
         In [2]: strip_email_quotes('> > text')
         Out[2]: 'text'
 
         In [3]: strip_email_quotes('> > text\\n> > more')
         Out[3]: 'text\\nmore'
 
     Note how only the common prefix that appears in all lines is stripped::
 
         In [4]: strip_email_quotes('> > text\\n> > more\\n> more...')
         Out[4]: '> text\\n> more\\nmore...'
 
     So if any line has no quote marks ('>') , then none are stripped from any
     of them ::
     
         In [5]: strip_email_quotes('> > text\\n> > more\\nlast different')
         Out[5]: '> > text\\n> > more\\nlast different'
     """
     lines = text.splitlines()
     matches = set()
     for line in lines:
         prefix = re.match(r'^(\s*>[ >]*)', line)
         if prefix:
             matches.add(prefix.group(1))
         else:
             break
     else:
         prefix = long_substr(list(matches))
         if prefix:
             strip = len(prefix)
             text = '\n'.join([ ln[strip:] for ln in lines])
     return text
 
 
 class EvalFormatter(Formatter):
     """A String Formatter that allows evaluation of simple expressions.
     
     Note that this version interprets a : as specifying a format string (as per
     standard string formatting), so if slicing is required, you must explicitly
     create a slice.
     
     This is to be used in templating cases, such as the parallel batch
     script templates, where simple arithmetic on arguments is useful.
 
     Examples
     --------
     
     In  [1]: f = EvalFormatter()
     In  [2]: f.format('{n//4}', n=8)
     Out [2]: '2'
     
     In  [3]: f.format("{greeting[slice(2,4)]}", greeting="Hello")
     Out [3]: 'll'
     """
     def get_field(self, name, args, kwargs):
         v = eval(name, kwargs)
         return v, name
 
 
 @skip_doctest_py3
 class FullEvalFormatter(Formatter):
     """A String Formatter that allows evaluation of simple expressions.
     
     Any time a format key is not found in the kwargs,
     it will be tried as an expression in the kwargs namespace.
     
     Note that this version allows slicing using [1:2], so you cannot specify
     a format string. Use :class:`EvalFormatter` to permit format strings.
     
     Examples
     --------
     
     In [1]: f = FullEvalFormatter()
     In [2]: f.format('{n//4}', n=8)
     Out[2]: u'2'
     
     In [3]: f.format('{list(range(5))[2:4]}')
     Out[3]: u'[2, 3]'
 
     In [4]: f.format('{3*2}')
     Out[4]: u'6'
     """
     # copied from Formatter._vformat with minor changes to allow eval
     # and replace the format_spec code with slicing
     def _vformat(self, format_string, args, kwargs, used_args, recursion_depth):
         if recursion_depth < 0:
             raise ValueError('Max string recursion exceeded')
         result = []
         for literal_text, field_name, format_spec, conversion in \
                 self.parse(format_string):
 
             # output the literal text
             if literal_text:
                 result.append(literal_text)
 
             # if there's a field, output it
             if field_name is not None:
                 # this is some markup, find the object and do
                 # the formatting
 
                 if format_spec:
                     # override format spec, to allow slicing:
                     field_name = ':'.join([field_name, format_spec])
 
                 # eval the contents of the field for the object
                 # to be formatted
                 obj = eval(field_name, kwargs)
 
                 # do any conversion on the resulting object
                 obj = self.convert_field(obj, conversion)
 
                 # format the object and append to the result
                 result.append(self.format_field(obj, ''))
 
         return u''.join(py3compat.cast_unicode(s) for s in result)
 
 
 @skip_doctest_py3
 class DollarFormatter(FullEvalFormatter):
     """Formatter allowing Itpl style $foo replacement, for names and attribute
     access only. Standard {foo} replacement also works, and allows full
     evaluation of its arguments.
 
     Examples
     --------
     In [1]: f = DollarFormatter()
     In [2]: f.format('{n//4}', n=8)
     Out[2]: u'2'
     
     In [3]: f.format('23 * 76 is $result', result=23*76)
     Out[3]: u'23 * 76 is 1748'
     
     In [4]: f.format('$a or {b}', a=1, b=2)
     Out[4]: u'1 or 2'
     """
     _dollar_pattern = re.compile("(.*?)\$(\$?[\w\.]+)")
     def parse(self, fmt_string):
         for literal_txt, field_name, format_spec, conversion \
                     in Formatter.parse(self, fmt_string):
             
             # Find $foo patterns in the literal text.
             continue_from = 0
             txt = ""
             for m in self._dollar_pattern.finditer(literal_txt):
                 new_txt, new_field = m.group(1,2)
                 # $$foo --> $foo
                 if new_field.startswith("$"):
                     txt += new_txt + new_field
                 else:
                     yield (txt + new_txt, new_field, "", None)
                     txt = ""
                 continue_from = m.end()
             
             # Re-yield the {foo} style pattern
             yield (txt + literal_txt[continue_from:], field_name, format_spec, conversion)
 
 #-----------------------------------------------------------------------------
 # Utils to columnize a list of string
 #-----------------------------------------------------------------------------
 
 def _chunks(l, n):
     """Yield successive n-sized chunks from l."""
-    for i in xrange(0, len(l), n):
+    for i in py3compat.xrange(0, len(l), n):
         yield l[i:i+n]
 
 
 def _find_optimal(rlist , separator_size=2 , displaywidth=80):
     """Calculate optimal info to columnize a list of string"""
     for nrow in range(1, len(rlist)+1) :
         chk = map(max,_chunks(rlist, nrow))
         sumlength = sum(chk)
         ncols = len(chk)
         if sumlength+separator_size*(ncols-1) <= displaywidth :
             break;
     return {'columns_numbers' : ncols,
             'optimal_separator_width':(displaywidth - sumlength)/(ncols-1) if (ncols -1) else 0,
             'rows_numbers' : nrow,
             'columns_width' : chk
            }
 
 
 def _get_or_default(mylist, i, default=None):
     """return list item number, or default if don't exist"""
     if i >= len(mylist):
         return default
     else :
         return mylist[i]
 
 
 @skip_doctest
 def compute_item_matrix(items, empty=None, *args, **kwargs) :
     """Returns a nested list, and info to columnize items
 
     Parameters
     ----------
 
     items :
         list of strings to columize
     empty : (default None)
         default value to fill list if needed
     separator_size : int (default=2)
         How much caracters will be used as a separation between each columns.
     displaywidth : int (default=80)
         The width of the area onto wich the columns should enter
 
     Returns
     -------
 
     Returns a tuple of (strings_matrix, dict_info)
 
     strings_matrix :
 
         nested list of string, the outer most list contains as many list as
         rows, the innermost lists have each as many element as colums. If the
         total number of elements in `items` does not equal the product of
         rows*columns, the last element of some lists are filled with `None`.
 
     dict_info :
         some info to make columnize easier:
 
         columns_numbers : number of columns
         rows_numbers    : number of rows
         columns_width   : list of with of each columns
         optimal_separator_width : best separator width between columns
 
     Examples
     --------
 
     In [1]: l = ['aaa','b','cc','d','eeeee','f','g','h','i','j','k','l']
        ...: compute_item_matrix(l,displaywidth=12)
     Out[1]:
         ([['aaa', 'f', 'k'],
         ['b', 'g', 'l'],
         ['cc', 'h', None],
         ['d', 'i', None],
         ['eeeee', 'j', None]],
         {'columns_numbers': 3,
         'columns_width': [5, 1, 1],
         'optimal_separator_width': 2,
         'rows_numbers': 5})
 
     """
     info = _find_optimal(map(len, items), *args, **kwargs)
     nrow, ncol = info['rows_numbers'], info['columns_numbers']
     return ([[ _get_or_default(items, c*nrow+i, default=empty) for c in range(ncol) ] for i in range(nrow) ], info)
 
 
 def columnize(items, separator='  ', displaywidth=80):
     """ Transform a list of strings into a single string with columns.
 
     Parameters
     ----------
     items : sequence of strings
         The strings to process.
 
     separator : str, optional [default is two spaces]
         The string that separates columns.
 
     displaywidth : int, optional [default is 80]
         Width of the display in number of characters.
 
     Returns
     -------
     The formatted string.
     """
     if not items :
         return '\n'
     matrix, info = compute_item_matrix(items, separator_size=len(separator), displaywidth=displaywidth)
     fmatrix = [filter(None, x) for x in matrix]
     sjoin = lambda x : separator.join([ y.ljust(w, ' ') for y, w in zip(x, info['columns_width'])])
     return '\n'.join(map(sjoin, fmatrix))+'\n'
 
 
 def get_text_list(list_, last_sep=' and ', sep=", ", wrap_item_with=""):
     """
     Return a string with a natural enumeration of items
 
     >>> get_text_list(['a', 'b', 'c', 'd'])
     'a, b, c and d'
     >>> get_text_list(['a', 'b', 'c'], ' or ')
     'a, b or c'
     >>> get_text_list(['a', 'b', 'c'], ', ')
     'a, b, c'
     >>> get_text_list(['a', 'b'], ' or ')
     'a or b'
     >>> get_text_list(['a'])
     'a'
     >>> get_text_list([])
     ''
     >>> get_text_list(['a', 'b'], wrap_item_with="`")
     '`a` and `b`'
     >>> get_text_list(['a', 'b', 'c', 'd'], " = ", sep=" + ")
     'a + b + c = d'
     """
     if len(list_) == 0:
         return ''
     if wrap_item_with:
         list_ = ['%s%s%s' % (wrap_item_with, item, wrap_item_with) for
                  item in list_]
     if len(list_) == 1:
         return list_[0]
     return '%s%s%s' % (
         sep.join(i for i in list_[:-1]),
         last_sep, list_[-1])
  No newline at end of file

IPython/utils/timing.py

@@ -1,116 +1,118 b''
 # encoding: utf-8
 """
 Utilities for timing code execution.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import time
 
+from .py3compat import xrange
+
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 # If possible (Unix), use the resource module instead of time.clock()
 try:
     import resource
     def clocku():
         """clocku() -> floating point number
 
         Return the *USER* CPU time in seconds since the start of the process.
         This is done via a call to resource.getrusage, so it avoids the
         wraparound problems in time.clock()."""
 
         return resource.getrusage(resource.RUSAGE_SELF)[0]
 
     def clocks():
         """clocks() -> floating point number
 
         Return the *SYSTEM* CPU time in seconds since the start of the process.
         This is done via a call to resource.getrusage, so it avoids the
         wraparound problems in time.clock()."""
 
         return resource.getrusage(resource.RUSAGE_SELF)[1]
 
     def clock():
         """clock() -> floating point number
 
         Return the *TOTAL USER+SYSTEM* CPU time in seconds since the start of
         the process.  This is done via a call to resource.getrusage, so it
         avoids the wraparound problems in time.clock()."""
 
         u,s = resource.getrusage(resource.RUSAGE_SELF)[:2]
         return u+s
 
     def clock2():
         """clock2() -> (t_user,t_system)
 
         Similar to clock(), but return a tuple of user/system times."""
         return resource.getrusage(resource.RUSAGE_SELF)[:2]
 except ImportError:
     # There is no distinction of user/system time under windows, so we just use
     # time.clock() for everything...
     clocku = clocks = clock = time.clock
     def clock2():
         """Under windows, system CPU time can't be measured.
 
         This just returns clock() and zero."""
         return time.clock(),0.0
 
     
 def timings_out(reps,func,*args,**kw):
     """timings_out(reps,func,*args,**kw) -> (t_total,t_per_call,output)
 
     Execute a function reps times, return a tuple with the elapsed total
     CPU time in seconds, the time per call and the function's output.
 
     Under Unix, the return value is the sum of user+system time consumed by
     the process, computed via the resource module.  This prevents problems
     related to the wraparound effect which the time.clock() function has.
 
     Under Windows the return value is in wall clock seconds. See the
     documentation for the time module for more details."""
 
     reps = int(reps)
     assert reps >=1, 'reps must be >= 1'
     if reps==1:
         start = clock()
         out = func(*args,**kw)
         tot_time = clock()-start
     else:
         rng = xrange(reps-1) # the last time is executed separately to store output
         start = clock()
         for dummy in rng: func(*args,**kw)
         out = func(*args,**kw)  # one last time
         tot_time = clock()-start
     av_time = tot_time / reps
     return tot_time,av_time,out
 
 
 def timings(reps,func,*args,**kw):
     """timings(reps,func,*args,**kw) -> (t_total,t_per_call)
 
     Execute a function reps times, return a tuple with the elapsed total CPU
     time in seconds and the time per call. These are just the first two values
     in timings_out()."""
 
     return timings_out(reps,func,*args,**kw)[0:2]
 
 
 def timing(func,*args,**kw):
     """timing(func,*args,**kw) -> t_total
 
     Execute a function once, return the elapsed total CPU time in
     seconds. This is just the first value in timings_out()."""
 
     return timings_out(1,func,*args,**kw)[0]