remove now-obsolete use of skip_doctest outside core

IPython/extensions/rmagic.py

@@ -1,707 +1,702 b''
 # -*- coding: utf-8 -*-
 """
 ======
 Rmagic
 ======
 
 Magic command interface for interactive work with R via rpy2
 
 .. note::
 
   The ``rpy2`` package needs to be installed separately. It
   can be obtained using ``easy_install`` or ``pip``.
 
   You will also need a working copy of R.
 
 Usage
 =====
 
 To enable the magics below, execute ``%load_ext rmagic``.
 
 ``%R``
 
 {R_DOC}
 
 ``%Rpush``
 
 {RPUSH_DOC}
 
 ``%Rpull``
 
 {RPULL_DOC}
 
 ``%Rget``
 
 {RGET_DOC}
 
 """
 from __future__ import print_function
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2012 The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 import sys
 import tempfile
 from glob import glob
 from shutil import rmtree
 import warnings
 
 # numpy and rpy2 imports
 
 import numpy as np
 
 import rpy2.rinterface as ri
 import rpy2.robjects as ro
 try:
     from rpy2.robjects import pandas2ri
     pandas2ri.activate()
 except ImportError:
     pandas2ri = None
     from rpy2.robjects import numpy2ri
     numpy2ri.activate()
 
 # IPython imports
 
 from IPython.core.displaypub import publish_display_data
 from IPython.core.magic import (Magics, magics_class, line_magic,
                                 line_cell_magic, needs_local_scope)
-from IPython.testing.skipdoctest import skip_doctest
 from IPython.core.magic_arguments import (
     argument, magic_arguments, parse_argstring
 )
 from simplegeneric import generic
 from IPython.utils.py3compat import (str_to_unicode, unicode_to_str, PY3,
                                      unicode_type)
 from IPython.utils.text import dedent
 
 class RInterpreterError(ri.RRuntimeError):
     """An error when running R code in a %%R magic cell."""
     def __init__(self, line, err, stdout):
         self.line = line
         self.err = err.rstrip()
         self.stdout = stdout.rstrip()
     
     def __unicode__(self):
         s = 'Failed to parse and evaluate line %r.\nR error message: %r' % \
                 (self.line, self.err)
         if self.stdout and (self.stdout != self.err):
             s += '\nR stdout:\n' + self.stdout
         return s
     
     if PY3:
         __str__ = __unicode__
     else:
         def __str__(self):
             return unicode_to_str(unicode(self), 'utf-8')
 
 def Rconverter(Robj, dataframe=False):
     """
     Convert an object in R's namespace to one suitable
     for ipython's namespace.
 
     For a data.frame, it tries to return a structured array.
     It first checks for colnames, then names.
     If all are NULL, it returns np.asarray(Robj), else
     it tries to construct a recarray
 
     Parameters
     ----------
 
     Robj: an R object returned from rpy2
     """
     is_data_frame = ro.r('is.data.frame')
     colnames = ro.r('colnames')
     rownames = ro.r('rownames') # with pandas, these could be used for the index
     names = ro.r('names')
 
     if dataframe:
         as_data_frame = ro.r('as.data.frame')
         cols = colnames(Robj)
         _names = names(Robj)
         if cols != ri.NULL:
             Robj = as_data_frame(Robj)
             names = tuple(np.array(cols))
         elif _names != ri.NULL:
             names = tuple(np.array(_names))
         else: # failed to find names
             return np.asarray(Robj)
         Robj = np.rec.fromarrays(Robj, names = names)
     return np.asarray(Robj)
 
 @generic
 def pyconverter(pyobj):
     """Convert Python objects to R objects. Add types using the decorator:
     
     @pyconverter.when_type
     """
     return pyobj
 
 # The default conversion for lists seems to make them a nested list. That has
 # some advantages, but is rarely convenient, so for interactive use, we convert
 # lists to a numpy array, which becomes an R vector.
 @pyconverter.when_type(list)
 def pyconverter_list(pyobj):
     return np.asarray(pyobj)
 
 if pandas2ri is None:
     # pandas2ri was new in rpy2 2.3.3, so for now we'll fallback to pandas'
     # conversion function.
     try:
         from pandas import DataFrame
         from pandas.rpy.common import convert_to_r_dataframe
         @pyconverter.when_type(DataFrame)
         def pyconverter_dataframe(pyobj):
             return convert_to_r_dataframe(pyobj, strings_as_factors=True)
     except ImportError:
         pass
 
 @magics_class
 class RMagics(Magics):
     """A set of magics useful for interactive work with R via rpy2.
     """
 
     def __init__(self, shell, Rconverter=Rconverter,
                  pyconverter=pyconverter,
                  cache_display_data=False):
         """
         Parameters
         ----------
 
         shell : IPython shell
         
         Rconverter : callable
             To be called on values taken from R before putting them in the
             IPython namespace.
 
         pyconverter : callable
             To be called on values in ipython namespace before 
             assigning to variables in rpy2.
 
         cache_display_data : bool
             If True, the published results of the final call to R are 
             cached in the variable 'display_cache'.
 
         """
         super(RMagics, self).__init__(shell)
         self.cache_display_data = cache_display_data
 
         self.r = ro.R()
 
         self.Rstdout_cache = []
         self.pyconverter = pyconverter
         self.Rconverter = Rconverter
 
     def eval(self, line):
         '''
         Parse and evaluate a line of R code with rpy2.
         Returns the output to R's stdout() connection, 
         the value generated by evaluating the code, and a
         boolean indicating whether the return value would be
         visible if the line of code were evaluated in an R REPL.
         
         R Code evaluation and visibility determination are
         done via an R call of the form withVisible({<code>})
         
         '''
         old_writeconsole = ri.get_writeconsole()
         ri.set_writeconsole(self.write_console)
         try:
             res = ro.r("withVisible({%s\n})" % line)
             value = res[0] #value (R object)
             visible = ro.conversion.ri2py(res[1])[0] #visible (boolean)
         except (ri.RRuntimeError, ValueError) as exception:
             warning_or_other_msg = self.flush() # otherwise next return seems to have copy of error
             raise RInterpreterError(line, str_to_unicode(str(exception)), warning_or_other_msg)
         text_output = self.flush()
         ri.set_writeconsole(old_writeconsole)
         return text_output, value, visible
 
     def write_console(self, output):
         '''
         A hook to capture R's stdout in a cache.
         '''
         self.Rstdout_cache.append(output)
 
     def flush(self):
         '''
         Flush R's stdout cache to a string, returning the string.
         '''
         value = ''.join([str_to_unicode(s, 'utf-8') for s in self.Rstdout_cache])
         self.Rstdout_cache = []
         return value
 
-    @skip_doctest
     @needs_local_scope
     @line_magic
     def Rpush(self, line, local_ns=None):
         '''
         A line-level magic for R that pushes
         variables from python to rpy2. The line should be made up
         of whitespace separated variable names in the IPython
         namespace::
 
             In [7]: import numpy as np
 
             In [8]: X = np.array([4.5,6.3,7.9])
 
             In [9]: X.mean()
             Out[9]: 6.2333333333333343
 
             In [10]: %Rpush X
 
             In [11]: %R mean(X)
             Out[11]: array([ 6.23333333])
 
         '''
         if local_ns is None:
             local_ns = {}
 
         inputs = line.split(' ')
         for input in inputs:
             try:
                 val = local_ns[input]
             except KeyError:
                 try:
                     val = self.shell.user_ns[input]
                 except KeyError:
                     # reraise the KeyError as a NameError so that it looks like
                     # the standard python behavior when you use an unnamed
                     # variable
                     raise NameError("name '%s' is not defined" % input)
 
             self.r.assign(input, self.pyconverter(val))
 
-    @skip_doctest
     @magic_arguments()
     @argument(
         '-d', '--as_dataframe', action='store_true',
         default=False,
         help='Convert objects to data.frames before returning to ipython.'
         )
     @argument(
         'outputs',
         nargs='*',
         )
     @line_magic
     def Rpull(self, line):
         '''
         A line-level magic for R that pulls
         variables from python to rpy2::
 
             In [18]: _ = %R x = c(3,4,6.7); y = c(4,6,7); z = c('a',3,4)
 
             In [19]: %Rpull x  y z
 
             In [20]: x
             Out[20]: array([ 3. ,  4. ,  6.7])
 
             In [21]: y
             Out[21]: array([ 4.,  6.,  7.])
 
             In [22]: z
             Out[22]:
             array(['a', '3', '4'],
                   dtype='|S1')
 
 
         If --as_dataframe, then each object is returned as a structured array
         after first passed through "as.data.frame" in R before
         being calling self.Rconverter. 
         This is useful when a structured array is desired as output, or
         when the object in R has mixed data types. 
         See the %%R docstring for more examples.
 
         Notes
         -----
 
         Beware that R names can have '.' so this is not fool proof.
         To avoid this, don't name your R objects with '.'s...
 
         '''
         args = parse_argstring(self.Rpull, line)
         outputs = args.outputs
         for output in outputs:
             self.shell.push({output:self.Rconverter(self.r(output),dataframe=args.as_dataframe)})
 
-    @skip_doctest
     @magic_arguments()
     @argument(
         '-d', '--as_dataframe', action='store_true',
         default=False,
         help='Convert objects to data.frames before returning to ipython.'
         )
     @argument(
         'output',
         nargs=1,
         type=str,
         )
     @line_magic
     def Rget(self, line):
         '''
         Return an object from rpy2, possibly as a structured array (if possible).
         Similar to Rpull except only one argument is accepted and the value is 
         returned rather than pushed to self.shell.user_ns::
 
             In [3]: dtype=[('x', '<i4'), ('y', '<f8'), ('z', '|S1')]
 
             In [4]: datapy = np.array([(1, 2.9, 'a'), (2, 3.5, 'b'), (3, 2.1, 'c'), (4, 5, 'e')], dtype=dtype)
 
             In [5]: %R -i datapy
 
             In [6]: %Rget datapy
             Out[6]: 
             array([['1', '2', '3', '4'],
                    ['2', '3', '2', '5'],
                    ['a', 'b', 'c', 'e']], 
                   dtype='|S1')
 
             In [7]: %Rget -d datapy
             Out[7]: 
             array([(1, 2.9, 'a'), (2, 3.5, 'b'), (3, 2.1, 'c'), (4, 5.0, 'e')], 
                   dtype=[('x', '<i4'), ('y', '<f8'), ('z', '|S1')])
 
         '''
         args = parse_argstring(self.Rget, line)
         output = args.output
         return self.Rconverter(self.r(output[0]),dataframe=args.as_dataframe)
 
 
-    @skip_doctest
     @magic_arguments()
     @argument(
         '-i', '--input', action='append',
         help='Names of input variable from shell.user_ns to be assigned to R variables of the same names after calling self.pyconverter. Multiple names can be passed separated only by commas with no whitespace.'
         )
     @argument(
         '-o', '--output', action='append',
         help='Names of variables to be pushed from rpy2 to shell.user_ns after executing cell body and applying self.Rconverter. Multiple names can be passed separated only by commas with no whitespace.'
         )
     @argument(
         '-w', '--width', type=int,
         help='Width of png plotting device sent as an argument to *png* in R.'
         )
     @argument(
         '-h', '--height', type=int,
         help='Height of png plotting device sent as an argument to *png* in R.'
         )
 
     @argument(
         '-d', '--dataframe', action='append',
         help='Convert these objects to data.frames and return as structured arrays.'
         )
     @argument(
         '-u', '--units', type=unicode_type, choices=["px", "in", "cm", "mm"],
         help='Units of png plotting device sent as an argument to *png* in R. One of ["px", "in", "cm", "mm"].'
         )
     @argument(
         '-r', '--res', type=int,
         help='Resolution of png plotting device sent as an argument to *png* in R. Defaults to 72 if *units* is one of ["in", "cm", "mm"].'
         )
     @argument(
         '-p', '--pointsize', type=int,
         help='Pointsize of png plotting device sent as an argument to *png* in R.'
         )
     @argument(
         '-b', '--bg',
         help='Background of png plotting device sent as an argument to *png* in R.'
         )
     @argument(
         '-n', '--noreturn',
         help='Force the magic to not return anything.',
         action='store_true',
         default=False
         )
     @argument(
         'code',
         nargs='*',
         )
     @needs_local_scope
     @line_cell_magic
     def R(self, line, cell=None, local_ns=None):
         '''
         Execute code in R, and pull some of the results back into the Python namespace.
 
         In line mode, this will evaluate an expression and convert the returned value to a Python object.
         The return value is determined by rpy2's behaviour of returning the result of evaluating the
         final line. 
 
         Multiple R lines can be executed by joining them with semicolons::
 
             In [9]: %R X=c(1,4,5,7); sd(X); mean(X)
             Out[9]: array([ 4.25])
 
         In cell mode, this will run a block of R code. The resulting value
         is printed if it would printed be when evaluating the same code 
         within a standard R REPL.
         
         Nothing is returned to python by default in cell mode::
 
             In [10]: %%R
                ....: Y = c(2,4,3,9)
                ....: summary(lm(Y~X))
 
             Call:
             lm(formula = Y ~ X)
 
             Residuals:
                 1     2     3     4
              0.88 -0.24 -2.28  1.64
 
             Coefficients:
                         Estimate Std. Error t value Pr(>|t|)
             (Intercept)   0.0800     2.3000   0.035    0.975
             X             1.0400     0.4822   2.157    0.164
 
             Residual standard error: 2.088 on 2 degrees of freedom
             Multiple R-squared: 0.6993,Adjusted R-squared: 0.549
             F-statistic: 4.651 on 1 and 2 DF,  p-value: 0.1638
 
         In the notebook, plots are published as the output of the cell::
 
             %R plot(X, Y)
 
         will create a scatter plot of X bs Y.
 
         If cell is not None and line has some R code, it is prepended to
         the R code in cell.
 
         Objects can be passed back and forth between rpy2 and python via the -i -o flags in line::
 
             In [14]: Z = np.array([1,4,5,10])
 
             In [15]: %R -i Z mean(Z)
             Out[15]: array([ 5.])
 
 
             In [16]: %R -o W W=Z*mean(Z)
             Out[16]: array([  5.,  20.,  25.,  50.])
 
             In [17]: W
             Out[17]: array([  5.,  20.,  25.,  50.])
 
         The return value is determined by these rules:
 
         * If the cell is not None, the magic returns None.
 
         * If the cell evaluates as False, the resulting value is returned
           unless the final line prints something to the console, in
           which case None is returned.
 
         * If the final line results in a NULL value when evaluated
           by rpy2, then None is returned.
 
         * No attempt is made to convert the final value to a structured array.
           Use the --dataframe flag or %Rget to push / return a structured array.
 
         * If the -n flag is present, there is no return value.
 
         * A trailing ';' will also result in no return value as the last
           value in the line is an empty string.
 
         The --dataframe argument will attempt to return structured arrays. 
         This is useful for dataframes with
         mixed data types. Note also that for a data.frame, 
         if it is returned as an ndarray, it is transposed::
 
             In [18]: dtype=[('x', '<i4'), ('y', '<f8'), ('z', '|S1')]
 
             In [19]: datapy = np.array([(1, 2.9, 'a'), (2, 3.5, 'b'), (3, 2.1, 'c'), (4, 5, 'e')], dtype=dtype)
 
             In [20]: %%R -o datar
             datar = datapy
                ....: 
 
             In [21]: datar
             Out[21]: 
             array([['1', '2', '3', '4'],
                    ['2', '3', '2', '5'],
                    ['a', 'b', 'c', 'e']], 
                   dtype='|S1')
 
             In [22]: %%R -d datar
             datar = datapy
                ....: 
 
             In [23]: datar
             Out[23]: 
             array([(1, 2.9, 'a'), (2, 3.5, 'b'), (3, 2.1, 'c'), (4, 5.0, 'e')], 
                   dtype=[('x', '<i4'), ('y', '<f8'), ('z', '|S1')])
 
         The --dataframe argument first tries colnames, then names.
         If both are NULL, it returns an ndarray (i.e. unstructured)::
 
             In [1]: %R mydata=c(4,6,8.3); NULL
 
             In [2]: %R -d mydata
 
             In [3]: mydata
             Out[3]: array([ 4. ,  6. ,  8.3])
 
             In [4]: %R names(mydata) = c('a','b','c'); NULL
 
             In [5]: %R -d mydata
 
             In [6]: mydata
             Out[6]: 
             array((4.0, 6.0, 8.3), 
                   dtype=[('a', '<f8'), ('b', '<f8'), ('c', '<f8')])
 
             In [7]: %R -o mydata
 
             In [8]: mydata
             Out[8]: array([ 4. ,  6. ,  8.3])
 
         '''
 
         args = parse_argstring(self.R, line)
 
         # arguments 'code' in line are prepended to
         # the cell lines
 
         if cell is None:
             code = ''
             return_output = True
             line_mode = True
         else:
             code = cell
             return_output = False
             line_mode = False
 
         code = ' '.join(args.code) + code
 
         # if there is no local namespace then default to an empty dict
         if local_ns is None:
             local_ns = {}
 
         if args.input:
             for input in ','.join(args.input).split(','):
                 try:
                     val = local_ns[input]
                 except KeyError:
                     try:
                         val = self.shell.user_ns[input]
                     except KeyError:
                         raise NameError("name '%s' is not defined" % input)
                 self.r.assign(input, self.pyconverter(val))
 
         if getattr(args, 'units') is not None:
             if args.units != "px" and getattr(args, 'res') is None:
                 args.res = 72
             args.units = '"%s"' % args.units
 
         png_argdict = dict([(n, getattr(args, n)) for n in ['units', 'res', 'height', 'width', 'bg', 'pointsize']])
         png_args = ','.join(['%s=%s' % (o,v) for o, v in png_argdict.items() if v is not None])
         # execute the R code in a temporary directory
 
         tmpd = tempfile.mkdtemp()
         self.r('png("%s/Rplots%%03d.png",%s)' % (tmpd.replace('\\', '/'), png_args))
 
         text_output = ''
         try:
             if line_mode:
                 for line in code.split(';'):
                     text_result, result, visible = self.eval(line)
                     text_output += text_result
                 if text_result:
                     # the last line printed something to the console so we won't return it
                     return_output = False
             else:
                 text_result, result, visible = self.eval(code)
                 text_output += text_result
                 if visible:
                     old_writeconsole = ri.get_writeconsole()
                     ri.set_writeconsole(self.write_console) 
                     ro.r.show(result)
                     text_output += self.flush()
                     ri.set_writeconsole(old_writeconsole)
         
         except RInterpreterError as e:
             print(e.stdout)
             if not e.stdout.endswith(e.err):
                 print(e.err)
             rmtree(tmpd)
             return
         finally:
             self.r('dev.off()')
 
         # read out all the saved .png files
 
         images = [open(imgfile, 'rb').read() for imgfile in glob("%s/Rplots*png" % tmpd)]
 
         # now publish the images
         # mimicking IPython/zmq/pylab/backend_inline.py
         fmt = 'png'
         mimetypes = { 'png' : 'image/png', 'svg' : 'image/svg+xml' }
         mime = mimetypes[fmt]
 
         # publish the printed R objects, if any
 
         display_data = []
         if text_output:
             display_data.append(('RMagic.R', {'text/plain':text_output}))
 
         # flush text streams before sending figures, helps a little with output
         for image in images:
             # synchronization in the console (though it's a bandaid, not a real sln)
             sys.stdout.flush(); sys.stderr.flush()
             display_data.append(('RMagic.R', {mime: image}))
 
         # kill the temporary directory
         rmtree(tmpd)
 
         # try to turn every output into a numpy array
         # this means that output are assumed to be castable
         # as numpy arrays
 
         if args.output:
             for output in ','.join(args.output).split(','):
                 self.shell.push({output:self.Rconverter(self.r(output), dataframe=False)})
 
         if args.dataframe:
             for output in ','.join(args.dataframe).split(','):
                 self.shell.push({output:self.Rconverter(self.r(output), dataframe=True)})
 
         for tag, disp_d in display_data:
             publish_display_data(data=disp_d, source=tag)
 
         # this will keep a reference to the display_data
         # which might be useful to other objects who happen to use
         # this method
 
         if self.cache_display_data:
             self.display_cache = display_data
 
         # if in line mode and return_output, return the result as an ndarray
         if return_output and not args.noreturn:
             if result != ri.NULL:
                 return self.Rconverter(result, dataframe=False)
 
 __doc__ = __doc__.format(
                 R_DOC = dedent(RMagics.R.__doc__),
                 RPUSH_DOC = dedent(RMagics.Rpush.__doc__),
                 RPULL_DOC = dedent(RMagics.Rpull.__doc__),
                 RGET_DOC = dedent(RMagics.Rget.__doc__)
 )
 
 
 def load_ipython_extension(ip):
     """Load the extension in IPython."""
     warnings.warn("The rmagic extension in IPython is deprecated in favour of "
             "rpy2.ipython. If available, that will be loaded instead.\n"
             "http://rpy.sourceforge.net/")
     try:
         import rpy2.ipython
     except ImportError:
         pass  # Fall back to our own implementation for now
     else:
         return rpy2.ipython.load_ipython_extension(ip)
 
     ip.register_magics(RMagics)
     # Initialising rpy2 interferes with readline. Since, at this point, we've
     # probably just loaded rpy2, we reset the delimiters. See issue gh-2759.
     if ip.has_readline:
         ip.readline.set_completer_delims(ip.readline_delims)

IPython/extensions/storemagic.py

@@ -1,243 +1,241 b''
 # -*- coding: utf-8 -*-
 """
 %store magic for lightweight persistence.
 
 Stores variables, aliases and macros in IPython's database.
 
 To automatically restore stored variables at startup, add this to your
 :file:`ipython_config.py` file::
 
   c.StoreMagics.autorestore = True
 """
 from __future__ import print_function
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2012, The IPython Development Team.
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib
 import inspect, os, sys, textwrap
 
 # Our own
 from IPython.core.error import UsageError
 from IPython.core.magic import Magics, magics_class, line_magic
-from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.traitlets import Bool
 from IPython.utils.py3compat import string_types
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 def restore_aliases(ip):
     staliases = ip.db.get('stored_aliases', {})
     for k,v in staliases.items():
         #print "restore alias",k,v # dbg
         #self.alias_table[k] = v
         ip.alias_manager.define_alias(k,v)
 
 
 def refresh_variables(ip):
     db = ip.db
     for key in db.keys('autorestore/*'):
         # strip autorestore
         justkey = os.path.basename(key)
         try:
             obj = db[key]
         except KeyError:
             print("Unable to restore variable '%s', ignoring (use %%store -d to forget!)" % justkey)
             print("The error was:", sys.exc_info()[0])
         else:
             #print "restored",justkey,"=",obj #dbg
             ip.user_ns[justkey] = obj
 
 
 def restore_dhist(ip):
     ip.user_ns['_dh'] = ip.db.get('dhist',[])
 
 
 def restore_data(ip):
     refresh_variables(ip)
     restore_aliases(ip)
     restore_dhist(ip)
 
 
 @magics_class
 class StoreMagics(Magics):
     """Lightweight persistence for python variables.
 
     Provides the %store magic."""
     
     autorestore = Bool(False, config=True, help=
         """If True, any %store-d variables will be automatically restored
         when IPython starts.
         """
     )
     
     def __init__(self, shell):
         super(StoreMagics, self).__init__(shell=shell)
         self.shell.configurables.append(self)
         if self.autorestore:
             restore_data(self.shell)
 
-    @skip_doctest
     @line_magic
     def store(self, parameter_s=''):
         """Lightweight persistence for python variables.
 
         Example::
 
           In [1]: l = ['hello',10,'world']
           In [2]: %store l
           In [3]: exit
 
           (IPython session is closed and started again...)
 
           ville@badger:~$ ipython
           In [1]: l
           NameError: name 'l' is not defined
           In [2]: %store -r
           In [3]: l
           Out[3]: ['hello', 10, 'world']
 
         Usage:
 
         * ``%store``          - Show list of all variables and their current
                                 values
         * ``%store spam``     - Store the *current* value of the variable spam
                                 to disk
         * ``%store -d spam``  - Remove the variable and its value from storage
         * ``%store -z``       - Remove all variables from storage
         * ``%store -r``       - Refresh all variables from store (overwrite
                                 current vals)
         * ``%store -r spam bar`` - Refresh specified variables from store
                                    (delete current val)
         * ``%store foo >a.txt``  - Store value of foo to new file a.txt
         * ``%store foo >>a.txt`` - Append value of foo to file a.txt
 
         It should be noted that if you change the value of a variable, you
         need to %store it again if you want to persist the new value.
 
         Note also that the variables will need to be pickleable; most basic
         python types can be safely %store'd.
 
         Also aliases can be %store'd across sessions.
         """
 
         opts,argsl = self.parse_options(parameter_s,'drz',mode='string')
         args = argsl.split(None,1)
         ip = self.shell
         db = ip.db
         # delete
         if 'd' in opts:
             try:
                 todel = args[0]
             except IndexError:
                 raise UsageError('You must provide the variable to forget')
             else:
                 try:
                     del db['autorestore/' + todel]
                 except:
                     raise UsageError("Can't delete variable '%s'" % todel)
         # reset
         elif 'z' in opts:
             for k in db.keys('autorestore/*'):
                 del db[k]
 
         elif 'r' in opts:
             if args:
                 for arg in args:
                     try:
                         obj = db['autorestore/' + arg]
                     except KeyError:
                         print("no stored variable %s" % arg)
                     else:
                         ip.user_ns[arg] = obj
             else:
                 restore_data(ip)
 
         # run without arguments -> list variables & values
         elif not args:
             vars = db.keys('autorestore/*')
             vars.sort()
             if vars:
                 size = max(map(len, vars))
             else:
                 size = 0
 
             print('Stored variables and their in-db values:')
             fmt = '%-'+str(size)+'s -> %s'
             get = db.get
             for var in vars:
                 justkey = os.path.basename(var)
                 # print 30 first characters from every var
                 print(fmt % (justkey, repr(get(var, '<unavailable>'))[:50]))
 
         # default action - store the variable
         else:
             # %store foo >file.txt or >>file.txt
             if len(args) > 1 and args[1].startswith('>'):
                 fnam = os.path.expanduser(args[1].lstrip('>').lstrip())
                 if args[1].startswith('>>'):
                     fil = open(fnam, 'a')
                 else:
                     fil = open(fnam, 'w')
                 obj = ip.ev(args[0])
                 print("Writing '%s' (%s) to file '%s'." % (args[0],
                   obj.__class__.__name__, fnam))
 
 
                 if not isinstance (obj, string_types):
                     from pprint import pprint
                     pprint(obj, fil)
                 else:
                     fil.write(obj)
                     if not obj.endswith('\n'):
                         fil.write('\n')
 
                 fil.close()
                 return
 
             # %store foo
             try:
                 obj = ip.user_ns[args[0]]
             except KeyError:
                 # it might be an alias
                 name = args[0]
                 try:
                     cmd = ip.alias_manager.retrieve_alias(name)
                 except ValueError:
                     raise UsageError("Unknown variable '%s'" % name)
                 
                 staliases = db.get('stored_aliases',{})
                 staliases[name] = cmd
                 db['stored_aliases'] = staliases
                 print("Alias stored: %s (%s)" % (name, cmd))
                 return
 
             else:
                 modname = getattr(inspect.getmodule(obj), '__name__', '')
                 if modname == '__main__':
                     print(textwrap.dedent("""\
                     Warning:%s is %s
                     Proper storage of interactively declared classes (or instances
                     of those classes) is not possible! Only instances
                     of classes in real modules on file system can be %%store'd.
                     """ % (args[0], obj) ))
                     return
                 #pickled = pickle.dumps(obj)
                 db[ 'autorestore/' + args[0] ] = obj
                 print("Stored '%s' (%s)" % (args[0], obj.__class__.__name__))
 
 
 def load_ipython_extension(ip):
     """Load the extension in IPython."""
     ip.register_magics(StoreMagics)
     

IPython/html/widgets/widget_link.py

@@ -1,111 +1,108 b''
 """Link and DirectionalLink classes.
 
 Propagate changes between widgets on the javascript side
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from .widget import Widget
-from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.traitlets import Unicode, Tuple, List,Instance, TraitError
 
 class WidgetTraitTuple(Tuple):
     """Traitlet for validating a single (Widget, 'trait_name') pair"""
     
     def __init__(self, **kwargs):
         super(WidgetTraitTuple, self).__init__(Instance(Widget), Unicode, **kwargs)
     
     def validate_elements(self, obj, value):
         value = super(WidgetTraitTuple, self).validate_elements(obj, value)
         widget, trait_name = value
         trait = widget.traits().get(trait_name)
         trait_repr = "%s.%s" % (widget.__class__.__name__, trait_name)
         # Can't raise TraitError because the parent will swallow the message
         # and throw it away in a new, less informative TraitError
         if trait is None:
             raise TypeError("No such trait: %s" % trait_repr)
         elif not trait.get_metadata('sync'):
             raise TypeError("%s cannot be synced" % trait_repr)
         
         return value
 
 
 class Link(Widget):
     """Link Widget
     
     one trait:
     widgets, a list of (widget, 'trait_name') tuples which should be linked in the frontend.
     """
     _model_name = Unicode('LinkModel', sync=True)
     widgets = List(WidgetTraitTuple, sync=True)
 
     def __init__(self, widgets, **kwargs):
         if len(widgets) < 2:
             raise TypeError("Require at least two widgets to link")
         kwargs['widgets'] = widgets
         super(Link, self).__init__(**kwargs)
 
     # for compatibility with traitlet links
     def unlink(self):
         self.close()
 
 
-@skip_doctest
 def jslink(*args):
     """Link traits from different widgets together on the frontend so they remain in sync.
 
     Parameters
     ----------
     *args : two or more (Widget, 'trait_name') tuples that should be kept in sync.
 
     Examples
     --------
 
     >>> c = link((widget1, 'value'), (widget2, 'value'), (widget3, 'value'))
     """
     return Link(widgets=args)
 
 
 class DirectionalLink(Widget):
     """A directional link
     
     source: a (Widget, 'trait_name') tuple for the source trait
     targets: one or more (Widget, 'trait_name') tuples that should be updated
     when the source trait changes.
     """
     _model_name = Unicode('DirectionalLinkModel', sync=True)
     targets = List(WidgetTraitTuple, sync=True)
     source = WidgetTraitTuple(sync=True)
 
     # Does not quite behave like other widgets but reproduces
     # the behavior of IPython.utils.traitlets.directional_link
     def __init__(self, source, targets, **kwargs):
         if len(targets) < 1:
             raise TypeError("Require at least two widgets to link")
         
         kwargs['source'] = source
         kwargs['targets'] = targets
         super(DirectionalLink, self).__init__(**kwargs)
 
     # for compatibility with traitlet links
     def unlink(self):
         self.close()
 
-@skip_doctest
 def jsdlink(source, *targets):
     """Link the trait of a source widget with traits of target widgets in the frontend.
 
     Parameters
     ----------
     source : a (Widget, 'trait_name') tuple for the source trait
     *targets : one or more (Widget, 'trait_name') tuples that should be updated
     when the source trait changes.
 
     Examples
     --------
 
     >>> c = dlink((src_widget, 'value'), (tgt_widget1, 'value'), (tgt_widget2, 'value'))
     """
     return DirectionalLink(source=source, targets=targets)
 

IPython/html/widgets/widget_output.py

@@ -1,78 +1,76 b''
 """Output class.  
 
 Represents a widget that can be used to display output within the widget area.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from .widget import DOMWidget
 import sys
 from IPython.utils.traitlets import Unicode, List
 from IPython.display import clear_output
-from IPython.testing.skipdoctest import skip_doctest
 from IPython.kernel.zmq.session import Message
 
-@skip_doctest
 class Output(DOMWidget):
     """Widget used as a context manager to display output.
 
     This widget can capture and display stdout, stderr, and rich output.  To use
     it, create an instance of it and display it.  Then use it as a context
     manager.  Any output produced while in it's context will be captured and
     displayed in it instead of the standard output area.
 
     Example
         from IPython.html import widgets
         from IPython.display import display
         out = widgets.Output()
         display(out)
         
         print('prints to output area')
 
         with out:
             print('prints to output widget')"""
     _view_name = Unicode('OutputView', sync=True)
 
     def clear_output(self, *pargs, **kwargs):
         with self:
             clear_output(*pargs, **kwargs)
 
     def __enter__(self):
         """Called upon entering output widget context manager."""
         self._flush()
         kernel = get_ipython().kernel
         session = kernel.session
         send = session.send
         self._original_send = send
         self._session = session
 
         def send_hook(stream, msg_or_type, content=None, parent=None, ident=None,
              buffers=None, track=False, header=None, metadata=None): 
 
             # Handle both prebuild messages and unbuilt messages.
             if isinstance(msg_or_type, (Message, dict)):
                 msg_type = msg_or_type['msg_type']
                 msg = dict(msg_or_type)
             else:
                 msg_type = msg_or_type
                 msg = session.msg(msg_type, content=content, parent=parent, 
                     header=header, metadata=metadata)
 
             # If this is a message type that we want to forward, forward it.
             if stream is kernel.iopub_socket and msg_type in ['clear_output', 'stream', 'display_data']:
                 self.send(msg)
             else: 
                 send(stream, msg, ident=ident, buffers=buffers, track=track)
 
         session.send = send_hook
 
     def __exit__(self, exception_type, exception_value, traceback):
         """Called upon exiting output widget context manager."""
         self._flush()
         self._session.send = self._original_send
 
     def _flush(self):
         """Flush stdout and stderr buffers."""
         sys.stdout.flush()
         sys.stderr.flush()

IPython/lib/lexers.py

@@ -1,512 +1,510 b''
 # -*- coding: utf-8 -*-
 """
 Defines a variety of Pygments lexers for highlighting IPython code.
 
 This includes:
 
     IPythonLexer, IPython3Lexer
         Lexers for pure IPython (python + magic/shell commands)
 
     IPythonPartialTracebackLexer, IPythonTracebackLexer
         Supports 2.x and 3.x via keyword `python3`.  The partial traceback
         lexer reads everything but the Python code appearing in a traceback.
         The full lexer combines the partial lexer with an IPython lexer.
 
     IPythonConsoleLexer
         A lexer for IPython console sessions, with support for tracebacks.
 
     IPyLexer
         A friendly lexer which examines the first line of text and from it,
         decides whether to use an IPython lexer or an IPython console lexer.
         This is probably the only lexer that needs to be explicitly added
         to Pygments.
 
 """
 #-----------------------------------------------------------------------------
 # Copyright (c) 2013, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 # Standard library
 import re
 
 # Third party
 from pygments.lexers import BashLexer, PythonLexer, Python3Lexer
 from pygments.lexer import (
     Lexer, DelegatingLexer, RegexLexer, do_insertions, bygroups, using,
 )
 from pygments.token import (
     Comment, Generic, Keyword, Literal, Name, Operator, Other, Text, Error,
 )
 from pygments.util import get_bool_opt
 
 # Local
-from IPython.testing.skipdoctest import skip_doctest
 
 line_re = re.compile('.*?\n')
 
 __all__ = ['build_ipy_lexer', 'IPython3Lexer', 'IPythonLexer',
            'IPythonPartialTracebackLexer', 'IPythonTracebackLexer',
            'IPythonConsoleLexer', 'IPyLexer']
 
 ipython_tokens = [
   (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)),
   (r'(?s)(^\s*)(%%!)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(BashLexer))),
   (r"(%%?)(\w+)(\?\??)$",  bygroups(Operator, Keyword, Operator)),
   (r"\b(\?\??)(\s*)$",  bygroups(Operator, Text)),
   (r'(%)(sx|sc|system)(.*)(\n)', bygroups(Operator, Keyword,
                                        using(BashLexer), Text)),
   (r'(%)(\w+)(.*\n)', bygroups(Operator, Keyword, Text)),
   (r'^(!!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
   (r'(!)(?!=)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
   (r'^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)', bygroups(Text, Operator, Text)),
 ]
 
 def build_ipy_lexer(python3):
     """Builds IPython lexers depending on the value of `python3`.
 
     The lexer inherits from an appropriate Python lexer and then adds
     information about IPython specific keywords (i.e. magic commands,
     shell commands, etc.)
 
     Parameters
     ----------
     python3 : bool
         If `True`, then build an IPython lexer from a Python 3 lexer.
 
     """
     # It would be nice to have a single IPython lexer class which takes
     # a boolean `python3`.  But since there are two Python lexer classes,
     # we will also have two IPython lexer classes.
     if python3:
         PyLexer = Python3Lexer
         clsname = 'IPython3Lexer'
         name = 'IPython3'
         aliases = ['ipython3']
         doc = """IPython3 Lexer"""
     else:
         PyLexer = PythonLexer
         clsname = 'IPythonLexer'
         name = 'IPython'
         aliases = ['ipython2', 'ipython']
         doc = """IPython Lexer"""
 
     tokens = PyLexer.tokens.copy()
     tokens['root'] = ipython_tokens + tokens['root']
 
     attrs = {'name': name, 'aliases': aliases, 'filenames': [],
              '__doc__': doc, 'tokens': tokens}
 
     return type(name, (PyLexer,), attrs)
 
 
 IPython3Lexer = build_ipy_lexer(python3=True)
 IPythonLexer = build_ipy_lexer(python3=False)
 
 
 class IPythonPartialTracebackLexer(RegexLexer):
     """
     Partial lexer for IPython tracebacks.
 
     Handles all the non-python output. This works for both Python 2.x and 3.x.
 
     """
     name = 'IPython Partial Traceback'
 
     tokens = {
         'root': [
             # Tracebacks for syntax errors have a different style.
             # For both types of tracebacks, we mark the first line with
             # Generic.Traceback.  For syntax errors, we mark the filename
             # as we mark the filenames for non-syntax tracebacks.
             #
             # These two regexps define how IPythonConsoleLexer finds a
             # traceback.
             #
             ## Non-syntax traceback
             (r'^(\^C)?(-+\n)', bygroups(Error, Generic.Traceback)),
             ## Syntax traceback
             (r'^(  File)(.*)(, line )(\d+\n)',
              bygroups(Generic.Traceback, Name.Namespace,
                       Generic.Traceback, Literal.Number.Integer)),
 
             # (Exception Identifier)(Whitespace)(Traceback Message)
             (r'(?u)(^[^\d\W]\w*)(\s*)(Traceback.*?\n)',
              bygroups(Name.Exception, Generic.Whitespace, Text)),
             # (Module/Filename)(Text)(Callee)(Function Signature)
             # Better options for callee and function signature?
             (r'(.*)( in )(.*)(\(.*\)\n)',
              bygroups(Name.Namespace, Text, Name.Entity, Name.Tag)),
             # Regular line: (Whitespace)(Line Number)(Python Code)
             (r'(\s*?)(\d+)(.*?\n)',
              bygroups(Generic.Whitespace, Literal.Number.Integer, Other)),
             # Emphasized line: (Arrow)(Line Number)(Python Code)
             # Using Exception token so arrow color matches the Exception.
             (r'(-*>?\s?)(\d+)(.*?\n)',
              bygroups(Name.Exception, Literal.Number.Integer, Other)),
             # (Exception Identifier)(Message)
             (r'(?u)(^[^\d\W]\w*)(:.*?\n)',
              bygroups(Name.Exception, Text)),
             # Tag everything else as Other, will be handled later.
             (r'.*\n', Other),
         ],
     }
 
 
 class IPythonTracebackLexer(DelegatingLexer):
     """
     IPython traceback lexer.
 
     For doctests, the tracebacks can be snipped as much as desired with the
     exception to the lines that designate a traceback. For non-syntax error
     tracebacks, this is the line of hyphens. For syntax error tracebacks,
     this is the line which lists the File and line number.
 
     """
     # The lexer inherits from DelegatingLexer.  The "root" lexer is an
     # appropriate IPython lexer, which depends on the value of the boolean
     # `python3`.  First, we parse with the partial IPython traceback lexer.
     # Then, any code marked with the "Other" token is delegated to the root
     # lexer.
     #
     name = 'IPython Traceback'
     aliases = ['ipythontb']
 
     def __init__(self, **options):
         self.python3 = get_bool_opt(options, 'python3', False)
         if self.python3:
             self.aliases = ['ipython3tb']
         else:
             self.aliases = ['ipython2tb', 'ipythontb']
 
         if self.python3:
             IPyLexer = IPython3Lexer
         else:
             IPyLexer = IPythonLexer
 
         DelegatingLexer.__init__(self, IPyLexer,
                                  IPythonPartialTracebackLexer, **options)
 
-@skip_doctest
 class IPythonConsoleLexer(Lexer):
     """
     An IPython console lexer for IPython code-blocks and doctests, such as:
 
     .. code-block:: rst
 
         .. code-block:: ipythonconsole
 
             In [1]: a = 'foo'
 
             In [2]: a
             Out[2]: 'foo'
 
             In [3]: print a
             foo
 
             In [4]: 1 / 0
 
 
     Support is also provided for IPython exceptions:
 
     .. code-block:: rst
 
         .. code-block:: ipythonconsole
 
             In [1]: raise Exception
 
             ---------------------------------------------------------------------------
             Exception                                 Traceback (most recent call last)
             <ipython-input-1-fca2ab0ca76b> in <module>()
             ----> 1 raise Exception
 
             Exception:
 
     """
     name = 'IPython console session'
     aliases = ['ipythonconsole']
     mimetypes = ['text/x-ipython-console']
 
     # The regexps used to determine what is input and what is output.
     # The default prompts for IPython are:
     #
     #     c.PromptManager.in_template  = 'In [\#]: '
     #     c.PromptManager.in2_template = '   .\D.: '
     #     c.PromptManager.out_template = 'Out[\#]: '
     #
     in1_regex = r'In \[[0-9]+\]: '
     in2_regex = r'   \.\.+\.: '
     out_regex = r'Out\[[0-9]+\]: '
 
     #: The regex to determine when a traceback starts.
     ipytb_start = re.compile(r'^(\^C)?(-+\n)|^(  File)(.*)(, line )(\d+\n)')
 
     def __init__(self, **options):
         """Initialize the IPython console lexer.
 
         Parameters
         ----------
         python3 : bool
             If `True`, then the console inputs are parsed using a Python 3
             lexer. Otherwise, they are parsed using a Python 2 lexer.
         in1_regex : RegexObject
             The compiled regular expression used to detect the start
             of inputs. Although the IPython configuration setting may have a
             trailing whitespace, do not include it in the regex. If `None`,
             then the default input prompt is assumed.
         in2_regex : RegexObject
             The compiled regular expression used to detect the continuation
             of inputs. Although the IPython configuration setting may have a
             trailing whitespace, do not include it in the regex. If `None`,
             then the default input prompt is assumed.
         out_regex : RegexObject
             The compiled regular expression used to detect outputs. If `None`,
             then the default output prompt is assumed.
 
         """
         self.python3 = get_bool_opt(options, 'python3', False)
         if self.python3:
             self.aliases = ['ipython3console']
         else:
             self.aliases = ['ipython2console', 'ipythonconsole']
 
         in1_regex = options.get('in1_regex', self.in1_regex)
         in2_regex = options.get('in2_regex', self.in2_regex)
         out_regex = options.get('out_regex', self.out_regex)
 
         # So that we can work with input and output prompts which have been
         # rstrip'd (possibly by editors) we also need rstrip'd variants. If
         # we do not do this, then such prompts will be tagged as 'output'.
         # The reason can't just use the rstrip'd variants instead is because
         # we want any whitespace associated with the prompt to be inserted
         # with the token. This allows formatted code to be modified so as hide
         # the appearance of prompts, with the whitespace included. One example
         # use of this is in copybutton.js from the standard lib Python docs.
         in1_regex_rstrip = in1_regex.rstrip() + '\n'
         in2_regex_rstrip = in2_regex.rstrip() + '\n'
         out_regex_rstrip = out_regex.rstrip() + '\n'
 
         # Compile and save them all.
         attrs = ['in1_regex', 'in2_regex', 'out_regex',
                  'in1_regex_rstrip', 'in2_regex_rstrip', 'out_regex_rstrip']
         for attr in attrs:
             self.__setattr__(attr, re.compile(locals()[attr]))
 
         Lexer.__init__(self, **options)
 
         if self.python3:
             pylexer = IPython3Lexer
             tblexer = IPythonTracebackLexer
         else:
             pylexer = IPythonLexer
             tblexer = IPythonTracebackLexer
 
         self.pylexer = pylexer(**options)
         self.tblexer = tblexer(**options)
 
         self.reset()
 
     def reset(self):
         self.mode = 'output'
         self.index = 0
         self.buffer = u''
         self.insertions = []
 
     def buffered_tokens(self):
         """
         Generator of unprocessed tokens after doing insertions and before
         changing to a new state.
 
         """
         if self.mode == 'output':
             tokens = [(0, Generic.Output, self.buffer)]
         elif self.mode == 'input':
             tokens = self.pylexer.get_tokens_unprocessed(self.buffer)
         else: # traceback
             tokens = self.tblexer.get_tokens_unprocessed(self.buffer)
 
         for i, t, v in do_insertions(self.insertions, tokens):
             # All token indexes are relative to the buffer.
             yield self.index + i, t, v
 
         # Clear it all
         self.index += len(self.buffer)
         self.buffer = u''
         self.insertions = []
 
     def get_mci(self, line):
         """
         Parses the line and returns a 3-tuple: (mode, code, insertion).
 
         `mode` is the next mode (or state) of the lexer, and is always equal
         to 'input', 'output', or 'tb'.
 
         `code` is a portion of the line that should be added to the buffer
         corresponding to the next mode and eventually lexed by another lexer.
         For example, `code` could be Python code if `mode` were 'input'.
 
         `insertion` is a 3-tuple (index, token, text) representing an
         unprocessed "token" that will be inserted into the stream of tokens
         that are created from the buffer once we change modes. This is usually
         the input or output prompt.
 
         In general, the next mode depends on current mode and on the contents
         of `line`.
 
         """
         # To reduce the number of regex match checks, we have multiple
         # 'if' blocks instead of 'if-elif' blocks.
 
         # Check for possible end of input
         in2_match = self.in2_regex.match(line)
         in2_match_rstrip = self.in2_regex_rstrip.match(line)
         if (in2_match and in2_match.group().rstrip() == line.rstrip()) or \
            in2_match_rstrip:
             end_input = True
         else:
             end_input = False
         if end_input and self.mode != 'tb':
             # Only look for an end of input when not in tb mode.
             # An ellipsis could appear within the traceback.
             mode = 'output'
             code = u''
             insertion = (0, Generic.Prompt, line)
             return mode, code, insertion
 
         # Check for output prompt
         out_match = self.out_regex.match(line)
         out_match_rstrip = self.out_regex_rstrip.match(line)
         if out_match or out_match_rstrip:
             mode = 'output'
             if out_match:
                 idx = out_match.end()
             else:
                 idx = out_match_rstrip.end()
             code = line[idx:]
             # Use the 'heading' token for output.  We cannot use Generic.Error
             # since it would conflict with exceptions.
             insertion = (0, Generic.Heading, line[:idx])
             return mode, code, insertion
 
 
         # Check for input or continuation prompt (non stripped version)
         in1_match = self.in1_regex.match(line)
         if in1_match or (in2_match and self.mode != 'tb'):
             # New input or when not in tb, continued input.
             # We do not check for continued input when in tb since it is
             # allowable to replace a long stack with an ellipsis.
             mode = 'input'
             if in1_match:
                 idx = in1_match.end()
             else: # in2_match
                 idx = in2_match.end()
             code = line[idx:]
             insertion = (0, Generic.Prompt, line[:idx])
             return mode, code, insertion
 
         # Check for input or continuation prompt (stripped version)
         in1_match_rstrip = self.in1_regex_rstrip.match(line)
         if in1_match_rstrip or (in2_match_rstrip and self.mode != 'tb'):
             # New input or when not in tb, continued input.
             # We do not check for continued input when in tb since it is
             # allowable to replace a long stack with an ellipsis.
             mode = 'input'
             if in1_match_rstrip:
                 idx = in1_match_rstrip.end()
             else: # in2_match
                 idx = in2_match_rstrip.end()
             code = line[idx:]
             insertion = (0, Generic.Prompt, line[:idx])
             return mode, code, insertion
 
         # Check for traceback
         if self.ipytb_start.match(line):
             mode = 'tb'
             code = line
             insertion = None
             return mode, code, insertion
 
         # All other stuff...
         if self.mode in ('input', 'output'):
             # We assume all other text is output. Multiline input that
             # does not use the continuation marker cannot be detected.
             # For example, the 3 in the following is clearly output:
             #
             #    In [1]: print 3
             #    3
             #
             # But the following second line is part of the input:
             #
             #    In [2]: while True:
             #        print True
             #
             # In both cases, the 2nd line will be 'output'.
             #
             mode = 'output'
         else:
             mode = 'tb'
 
         code = line
         insertion = None
 
         return mode, code, insertion
 
     def get_tokens_unprocessed(self, text):
         self.reset()
         for match in line_re.finditer(text):
             line = match.group()
             mode, code, insertion = self.get_mci(line)
 
             if mode != self.mode:
                 # Yield buffered tokens before transitioning to new mode.
                 for token in self.buffered_tokens():
                     yield token
                 self.mode = mode
 
             if insertion:
                 self.insertions.append((len(self.buffer), [insertion]))
             self.buffer += code
         else:
             for token in self.buffered_tokens():
                 yield token
 
 class IPyLexer(Lexer):
     """
     Primary lexer for all IPython-like code.
 
     This is a simple helper lexer.  If the first line of the text begins with
     "In \[[0-9]+\]:", then the entire text is parsed with an IPython console
     lexer. If not, then the entire text is parsed with an IPython lexer.
 
     The goal is to reduce the number of lexers that are registered
     with Pygments.
 
     """
     name = 'IPy session'
     aliases = ['ipy']
 
     def __init__(self, **options):
         self.python3 = get_bool_opt(options, 'python3', False)
         if self.python3:
             self.aliases = ['ipy3']
         else:
             self.aliases = ['ipy2', 'ipy']
 
         Lexer.__init__(self, **options)
 
         self.IPythonLexer = IPythonLexer(**options)
         self.IPythonConsoleLexer = IPythonConsoleLexer(**options)
 
     def get_tokens_unprocessed(self, text):
         # Search for the input prompt anywhere...this allows code blocks to
         # begin with comments as well.
         if re.match(r'.*(In \[[0-9]+\]:)', text.strip(), re.DOTALL):
             lex = self.IPythonConsoleLexer
         else:
             lex = self.IPythonLexer
         for token in lex.get_tokens_unprocessed(text):
             yield token
 

IPython/lib/security.py

@@ -1,116 +1,114 b''
 """
 Password generation for the IPython notebook.
 """
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 # Stdlib
 import getpass
 import hashlib
 import random
 
 # Our own
 from IPython.core.error import UsageError
-from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.py3compat import cast_bytes, str_to_bytes
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # Length of the salt in nr of hex chars, which implies salt_len * 4
 # bits of randomness.
 salt_len = 12
 
 #-----------------------------------------------------------------------------
 # Functions
 #-----------------------------------------------------------------------------
 
-@skip_doctest
 def passwd(passphrase=None, algorithm='sha1'):
     """Generate hashed password and salt for use in notebook configuration.
 
     In the notebook configuration, set `c.NotebookApp.password` to
     the generated string.
 
     Parameters
     ----------
     passphrase : str
         Password to hash.  If unspecified, the user is asked to input
         and verify a password.
     algorithm : str
         Hashing algorithm to use (e.g, 'sha1' or any argument supported
         by :func:`hashlib.new`).
 
     Returns
     -------
     hashed_passphrase : str
         Hashed password, in the format 'hash_algorithm:salt:passphrase_hash'.
 
     Examples
     --------
     >>> passwd('mypassword')
     'sha1:7cf3:b7d6da294ea9592a9480c8f52e63cd42cfb9dd12'
 
     """
     if passphrase is None:
         for i in range(3):
             p0 = getpass.getpass('Enter password: ')
             p1 = getpass.getpass('Verify password: ')
             if p0 == p1:
                 passphrase = p0
                 break
             else:
                 print('Passwords do not match.')
         else:
             raise UsageError('No matching passwords found. Giving up.')
 
     h = hashlib.new(algorithm)
     salt = ('%0' + str(salt_len) + 'x') % random.getrandbits(4 * salt_len)
     h.update(cast_bytes(passphrase, 'utf-8') + str_to_bytes(salt, 'ascii'))
 
     return ':'.join((algorithm, salt, h.hexdigest()))
 
 
 def passwd_check(hashed_passphrase, passphrase):
     """Verify that a given passphrase matches its hashed version.
 
     Parameters
     ----------
     hashed_passphrase : str
         Hashed password, in the format returned by `passwd`.
     passphrase : str
         Passphrase to validate.
 
     Returns
     -------
     valid : bool
         True if the passphrase matches the hash.
 
     Examples
     --------
     >>> from IPython.lib.security import passwd_check
     >>> passwd_check('sha1:0e112c3ddfce:a68df677475c2b47b6e86d0467eec97ac5f4b85a',
     ...              'mypassword')
     True
 
     >>> passwd_check('sha1:0e112c3ddfce:a68df677475c2b47b6e86d0467eec97ac5f4b85a',
     ...              'anotherpassword')
     False
     """
     try:
         algorithm, salt, pw_digest = hashed_passphrase.split(':', 2)
     except (ValueError, TypeError):
         return False
 
     try:
         h = hashlib.new(algorithm)
     except ValueError:
         return False
 
     if len(pw_digest) == 0:
         return False
 
     h.update(cast_bytes(passphrase, 'utf-8') + cast_bytes(salt, 'ascii'))
 
     return h.hexdigest() == pw_digest

IPython/terminal/interactiveshell.py

@@ -1,642 +1,640 b''
 # -*- coding: utf-8 -*-
 """Subclass of InteractiveShell for terminal based frontends."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001 Janko Hauser <[e-mail]>
 #  Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 import bdb
 import os
 import sys
 
 from IPython.core.error import TryNext, UsageError
 from IPython.core.usage import interactive_usage
 from IPython.core.inputsplitter import IPythonInputSplitter
 from IPython.core.interactiveshell import InteractiveShell, InteractiveShellABC
 from IPython.core.magic import Magics, magics_class, line_magic
 from IPython.lib.clipboard import ClipboardEmpty
-from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.encoding import get_stream_enc
 from IPython.utils import py3compat
 from IPython.utils.terminal import toggle_set_term_title, set_term_title
 from IPython.utils.process import abbrev_cwd
 from IPython.utils.warn import warn, error
 from IPython.utils.text import num_ini_spaces, SList, strip_email_quotes
 from IPython.utils.traitlets import Integer, CBool, Unicode
 
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
 def get_default_editor():
     try:
         ed = os.environ['EDITOR']
         if not py3compat.PY3:
             ed = ed.decode()
         return ed
     except KeyError:
         pass
     except UnicodeError:
         warn("$EDITOR environment variable is not pure ASCII. Using platform "
              "default editor.")
 
     if os.name == 'posix':
         return 'vi'  # the only one guaranteed to be there!
     else:
         return 'notepad' # same in Windows!
 
 def get_pasted_lines(sentinel, l_input=py3compat.input, quiet=False):
     """ Yield pasted lines until the user enters the given sentinel value.
     """
     if not quiet:
         print("Pasting code; enter '%s' alone on the line to stop or use Ctrl-D." \
               % sentinel)
         prompt = ":"
     else:
         prompt = ""
     while True:
         try:
             l = py3compat.str_to_unicode(l_input(prompt))
             if l == sentinel:
                 return
             else:
                 yield l
         except EOFError:
             print('<EOF>')
             return
 
 
 #------------------------------------------------------------------------
 # Terminal-specific magics
 #------------------------------------------------------------------------
 
 @magics_class
 class TerminalMagics(Magics):
     def __init__(self, shell):
         super(TerminalMagics, self).__init__(shell)
         self.input_splitter = IPythonInputSplitter()
         
     def store_or_execute(self, block, name):
         """ Execute a block, or store it in a variable, per the user's request.
         """
         if name:
             # If storing it for further editing
             self.shell.user_ns[name] = SList(block.splitlines())
             print("Block assigned to '%s'" % name)
         else:
             b = self.preclean_input(block)
             self.shell.user_ns['pasted_block'] = b
             self.shell.using_paste_magics = True
             try:
                 self.shell.run_cell(b)
             finally:
                 self.shell.using_paste_magics = False
     
     def preclean_input(self, block):
         lines = block.splitlines()
         while lines and not lines[0].strip():
             lines = lines[1:]
         return strip_email_quotes('\n'.join(lines))
 
     def rerun_pasted(self, name='pasted_block'):
         """ Rerun a previously pasted command.
         """
         b = self.shell.user_ns.get(name)
 
         # Sanity checks
         if b is None:
             raise UsageError('No previous pasted block available')
         if not isinstance(b, py3compat.string_types):
             raise UsageError(
                 "Variable 'pasted_block' is not a string, can't execute")
 
         print("Re-executing '%s...' (%d chars)"% (b.split('\n',1)[0], len(b)))
         self.shell.run_cell(b)
 
     @line_magic
     def autoindent(self, parameter_s = ''):
         """Toggle autoindent on/off (if available)."""
 
         self.shell.set_autoindent()
         print("Automatic indentation is:",['OFF','ON'][self.shell.autoindent])
 
-    @skip_doctest
     @line_magic
     def cpaste(self, parameter_s=''):
         """Paste & execute a pre-formatted code block from clipboard.
 
         You must terminate the block with '--' (two minus-signs) or Ctrl-D
         alone on the line. You can also provide your own sentinel with '%paste
         -s %%' ('%%' is the new sentinel for this operation).
 
         The block is dedented prior to execution to enable execution of method
         definitions. '>' and '+' characters at the beginning of a line are
         ignored, to allow pasting directly from e-mails, diff files and
         doctests (the '...' continuation prompt is also stripped).  The
         executed block is also assigned to variable named 'pasted_block' for
         later editing with '%edit pasted_block'.
 
         You can also pass a variable name as an argument, e.g. '%cpaste foo'.
         This assigns the pasted block to variable 'foo' as string, without
         dedenting or executing it (preceding >>> and + is still stripped)
 
         '%cpaste -r' re-executes the block previously entered by cpaste.
         '%cpaste -q' suppresses any additional output messages.
 
         Do not be alarmed by garbled output on Windows (it's a readline bug).
         Just press enter and type -- (and press enter again) and the block
         will be what was just pasted.
 
         IPython statements (magics, shell escapes) are not supported (yet).
 
         See also
         --------
         paste: automatically pull code from clipboard.
 
         Examples
         --------
         ::
 
           In [8]: %cpaste
           Pasting code; enter '--' alone on the line to stop.
           :>>> a = ["world!", "Hello"]
           :>>> print " ".join(sorted(a))
           :--
           Hello world!
         """
         opts, name = self.parse_options(parameter_s, 'rqs:', mode='string')
         if 'r' in opts:
             self.rerun_pasted()
             return
 
         quiet = ('q' in opts)
 
         sentinel = opts.get('s', u'--')
         block = '\n'.join(get_pasted_lines(sentinel, quiet=quiet))
         self.store_or_execute(block, name)
 
     @line_magic
     def paste(self, parameter_s=''):
         """Paste & execute a pre-formatted code block from clipboard.
 
         The text is pulled directly from the clipboard without user
         intervention and printed back on the screen before execution (unless
         the -q flag is given to force quiet mode).
 
         The block is dedented prior to execution to enable execution of method
         definitions. '>' and '+' characters at the beginning of a line are
         ignored, to allow pasting directly from e-mails, diff files and
         doctests (the '...' continuation prompt is also stripped).  The
         executed block is also assigned to variable named 'pasted_block' for
         later editing with '%edit pasted_block'.
 
         You can also pass a variable name as an argument, e.g. '%paste foo'.
         This assigns the pasted block to variable 'foo' as string, without
         executing it (preceding >>> and + is still stripped).
 
         Options:
 
           -r: re-executes the block previously entered by cpaste.
 
           -q: quiet mode: do not echo the pasted text back to the terminal.
 
         IPython statements (magics, shell escapes) are not supported (yet).
 
         See also
         --------
         cpaste: manually paste code into terminal until you mark its end.
         """
         opts, name = self.parse_options(parameter_s, 'rq', mode='string')
         if 'r' in opts:
             self.rerun_pasted()
             return
         try:
             block = self.shell.hooks.clipboard_get()
         except TryNext as clipboard_exc:
             message = getattr(clipboard_exc, 'args')
             if message:
                 error(message[0])
             else:
                 error('Could not get text from the clipboard.')
             return
         except ClipboardEmpty:
             raise UsageError("The clipboard appears to be empty")
 
         # By default, echo back to terminal unless quiet mode is requested
         if 'q' not in opts:
             write = self.shell.write
             write(self.shell.pycolorize(block))
             if not block.endswith('\n'):
                 write('\n')
             write("## -- End pasted text --\n")
 
         self.store_or_execute(block, name)
 
     # Class-level: add a '%cls' magic only on Windows
     if sys.platform == 'win32':
         @line_magic
         def cls(self, s):
             """Clear screen.
             """
             os.system("cls")
 
 #-----------------------------------------------------------------------------
 # Main class
 #-----------------------------------------------------------------------------
 
 class TerminalInteractiveShell(InteractiveShell):
 
     autoedit_syntax = CBool(False, config=True,
         help="auto editing of files with syntax errors.")
     confirm_exit = CBool(True, config=True,
         help="""
         Set to confirm when you try to exit IPython with an EOF (Control-D
         in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
         you can force a direct exit without any confirmation.""",
     )
     # This display_banner only controls whether or not self.show_banner()
     # is called when mainloop/interact are called.  The default is False
     # because for the terminal based application, the banner behavior
     # is controlled by the application.
     display_banner = CBool(False) # This isn't configurable!
     embedded = CBool(False)
     embedded_active = CBool(False)
     editor = Unicode(get_default_editor(), config=True,
         help="Set the editor used by IPython (default to $EDITOR/vi/notepad)."
     )
     pager = Unicode('less', config=True,
         help="The shell program to be used for paging.")
 
     screen_length = Integer(0, config=True,
         help=
         """Number of lines of your screen, used to control printing of very
         long strings.  Strings longer than this number of lines will be sent
         through a pager instead of directly printed.  The default value for
         this is 0, which means IPython will auto-detect your screen size every
         time it needs to print certain potentially long strings (this doesn't
         change the behavior of the 'print' keyword, it's only triggered
         internally). If for some reason this isn't working well (it needs
         curses support), specify it yourself. Otherwise don't change the
         default.""",
     )
     term_title = CBool(False, config=True,
         help="Enable auto setting the terminal title."
     )
     usage = Unicode(interactive_usage)
     
     # This `using_paste_magics` is used to detect whether the code is being
     # executed via paste magics functions
     using_paste_magics = CBool(False)
 
     # In the terminal, GUI control is done via PyOS_InputHook
     @staticmethod
     def enable_gui(gui=None, app=None):
         """Switch amongst GUI input hooks by name.
         """
         # Deferred import
         from IPython.lib.inputhook import enable_gui as real_enable_gui
         try:
             return real_enable_gui(gui, app)
         except ValueError as e:
             raise UsageError("%s" % e)
     
     system = InteractiveShell.system_raw
 
     #-------------------------------------------------------------------------
     # Overrides of init stages
     #-------------------------------------------------------------------------
 
     def init_display_formatter(self):
         super(TerminalInteractiveShell, self).init_display_formatter()
         # terminal only supports plaintext
         self.display_formatter.active_types = ['text/plain']
 
     #-------------------------------------------------------------------------
     # Things related to the terminal
     #-------------------------------------------------------------------------
 
     @property
     def usable_screen_length(self):
         if self.screen_length == 0:
             return 0
         else:
             num_lines_bot = self.separate_in.count('\n')+1
             return self.screen_length - num_lines_bot
 
     def _term_title_changed(self, name, new_value):
         self.init_term_title()
 
     def init_term_title(self):
         # Enable or disable the terminal title.
         if self.term_title:
             toggle_set_term_title(True)
             set_term_title('IPython: ' + abbrev_cwd())
         else:
             toggle_set_term_title(False)
 
     #-------------------------------------------------------------------------
     # Things related to aliases
     #-------------------------------------------------------------------------
 
     def init_alias(self):
         # The parent class defines aliases that can be safely used with any
         # frontend.
         super(TerminalInteractiveShell, self).init_alias()
 
         # Now define aliases that only make sense on the terminal, because they
         # need direct access to the console in a way that we can't emulate in
         # GUI or web frontend
         if os.name == 'posix':
             aliases = [('clear', 'clear'), ('more', 'more'), ('less', 'less'),
                        ('man', 'man')]
         else :
             aliases = []
 
         for name, cmd in aliases:
             self.alias_manager.soft_define_alias(name, cmd)
 
     #-------------------------------------------------------------------------
     # Mainloop and code execution logic
     #-------------------------------------------------------------------------
 
     def mainloop(self, display_banner=None):
         """Start the mainloop.
 
         If an optional banner argument is given, it will override the
         internally created default banner.
         """
 
         with self.builtin_trap, self.display_trap:
 
             while 1:
                 try:
                     self.interact(display_banner=display_banner)
                     #self.interact_with_readline()
                     # XXX for testing of a readline-decoupled repl loop, call
                     # interact_with_readline above
                     break
                 except KeyboardInterrupt:
                     # this should not be necessary, but KeyboardInterrupt
                     # handling seems rather unpredictable...
                     self.write("\nKeyboardInterrupt in interact()\n")
 
     def _replace_rlhist_multiline(self, source_raw, hlen_before_cell):
         """Store multiple lines as a single entry in history"""
 
         # do nothing without readline or disabled multiline
         if not self.has_readline or not self.multiline_history:
             return hlen_before_cell
 
         # windows rl has no remove_history_item
         if not hasattr(self.readline, "remove_history_item"):
             return hlen_before_cell
 
         # skip empty cells
         if not source_raw.rstrip():
             return hlen_before_cell
 
         # nothing changed do nothing, e.g. when rl removes consecutive dups
         hlen = self.readline.get_current_history_length()
         if hlen == hlen_before_cell:
             return hlen_before_cell
 
         for i in range(hlen - hlen_before_cell):
             self.readline.remove_history_item(hlen - i - 1)
         stdin_encoding = get_stream_enc(sys.stdin, 'utf-8')
         self.readline.add_history(py3compat.unicode_to_str(source_raw.rstrip(),
                                     stdin_encoding))
         return self.readline.get_current_history_length()
 
     def interact(self, display_banner=None):
         """Closely emulate the interactive Python console."""
 
         # batch run -> do not interact
         if self.exit_now:
             return
 
         if display_banner is None:
             display_banner = self.display_banner
 
         if isinstance(display_banner, py3compat.string_types):
             self.show_banner(display_banner)
         elif display_banner:
             self.show_banner()
 
         more = False
 
         if self.has_readline:
             self.readline_startup_hook(self.pre_readline)
             hlen_b4_cell = self.readline.get_current_history_length()
         else:
             hlen_b4_cell = 0
         # exit_now is set by a call to %Exit or %Quit, through the
         # ask_exit callback.
 
         while not self.exit_now:
             self.hooks.pre_prompt_hook()
             if more:
                 try:
                     prompt = self.prompt_manager.render('in2')
                 except:
                     self.showtraceback()
                 if self.autoindent:
                     self.rl_do_indent = True
 
             else:
                 try:
                     prompt = self.separate_in + self.prompt_manager.render('in')
                 except:
                     self.showtraceback()
             try:
                 line = self.raw_input(prompt)
                 if self.exit_now:
                     # quick exit on sys.std[in|out] close
                     break
                 if self.autoindent:
                     self.rl_do_indent = False
 
             except KeyboardInterrupt:
                 #double-guard against keyboardinterrupts during kbdint handling
                 try:
                     self.write('\n' + self.get_exception_only())
                     source_raw = self.input_splitter.raw_reset()
                     hlen_b4_cell = \
                         self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
                     more = False
                 except KeyboardInterrupt:
                     pass
             except EOFError:
                 if self.autoindent:
                     self.rl_do_indent = False
                     if self.has_readline:
                         self.readline_startup_hook(None)
                 self.write('\n')
                 self.exit()
             except bdb.BdbQuit:
                 warn('The Python debugger has exited with a BdbQuit exception.\n'
                      'Because of how pdb handles the stack, it is impossible\n'
                      'for IPython to properly format this particular exception.\n'
                      'IPython will resume normal operation.')
             except:
                 # exceptions here are VERY RARE, but they can be triggered
                 # asynchronously by signal handlers, for example.
                 self.showtraceback()
             else:
                 try:
                     self.input_splitter.push(line)
                     more = self.input_splitter.push_accepts_more()
                 except SyntaxError:
                     # Run the code directly - run_cell takes care of displaying
                     # the exception.
                     more = False
                 if (self.SyntaxTB.last_syntax_error and
                     self.autoedit_syntax):
                     self.edit_syntax_error()
                 if not more:
                     source_raw = self.input_splitter.raw_reset()
                     self.run_cell(source_raw, store_history=True)
                     hlen_b4_cell = \
                         self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
 
         # Turn off the exit flag, so the mainloop can be restarted if desired
         self.exit_now = False
 
     def raw_input(self, prompt=''):
         """Write a prompt and read a line.
 
         The returned line does not include the trailing newline.
         When the user enters the EOF key sequence, EOFError is raised.
 
         Parameters
         ----------
 
         prompt : str, optional
           A string to be printed to prompt the user.
         """
         # raw_input expects str, but we pass it unicode sometimes
         prompt = py3compat.cast_bytes_py2(prompt)
 
         try:
             line = py3compat.str_to_unicode(self.raw_input_original(prompt))
         except ValueError:
             warn("\n********\nYou or a %run:ed script called sys.stdin.close()"
                  " or sys.stdout.close()!\nExiting IPython!\n")
             self.ask_exit()
             return ""
 
         # Try to be reasonably smart about not re-indenting pasted input more
         # than necessary.  We do this by trimming out the auto-indent initial
         # spaces, if the user's actual input started itself with whitespace.
         if self.autoindent:
             if num_ini_spaces(line) > self.indent_current_nsp:
                 line = line[self.indent_current_nsp:]
                 self.indent_current_nsp = 0
 
         return line
 
     #-------------------------------------------------------------------------
     # Methods to support auto-editing of SyntaxErrors.
     #-------------------------------------------------------------------------
 
     def edit_syntax_error(self):
         """The bottom half of the syntax error handler called in the main loop.
 
         Loop until syntax error is fixed or user cancels.
         """
 
         while self.SyntaxTB.last_syntax_error:
             # copy and clear last_syntax_error
             err = self.SyntaxTB.clear_err_state()
             if not self._should_recompile(err):
                 return
             try:
                 # may set last_syntax_error again if a SyntaxError is raised
                 self.safe_execfile(err.filename,self.user_ns)
             except:
                 self.showtraceback()
             else:
                 try:
                     f = open(err.filename)
                     try:
                         # This should be inside a display_trap block and I
                         # think it is.
                         sys.displayhook(f.read())
                     finally:
                         f.close()
                 except:
                     self.showtraceback()
 
     def _should_recompile(self,e):
         """Utility routine for edit_syntax_error"""
 
         if e.filename in ('<ipython console>','<input>','<string>',
                           '<console>','<BackgroundJob compilation>',
                           None):
 
             return False
         try:
             if (self.autoedit_syntax and
                 not self.ask_yes_no('Return to editor to correct syntax error? '
                               '[Y/n] ','y')):
                 return False
         except EOFError:
             return False
 
         def int0(x):
             try:
                 return int(x)
             except TypeError:
                 return 0
         # always pass integer line and offset values to editor hook
         try:
             self.hooks.fix_error_editor(e.filename,
                 int0(e.lineno),int0(e.offset),e.msg)
         except TryNext:
             warn('Could not open editor')
             return False
         return True
 
     #-------------------------------------------------------------------------
     # Things related to exiting
     #-------------------------------------------------------------------------
 
     def ask_exit(self):
         """ Ask the shell to exit. Can be overiden and used as a callback. """
         self.exit_now = True
 
     def exit(self):
         """Handle interactive exit.
 
         This method calls the ask_exit callback."""
         if self.confirm_exit:
             if self.ask_yes_no('Do you really want to exit ([y]/n)?','y'):
                 self.ask_exit()
         else:
             self.ask_exit()
 
     #-------------------------------------------------------------------------
     # Things related to magics
     #-------------------------------------------------------------------------
 
     def init_magics(self):
         super(TerminalInteractiveShell, self).init_magics()
         self.register_magics(TerminalMagics)
 
     def showindentationerror(self):
         super(TerminalInteractiveShell, self).showindentationerror()
         if not self.using_paste_magics:
             print("If you want to paste code into IPython, try the "
                 "%paste and %cpaste magic functions.")
 
 
 InteractiveShellABC.register(TerminalInteractiveShell)

IPython/testing/tests/test_decorators.py

@@ -1,169 +1,165 b''
 """Tests for the decorators we've created for IPython.
 """
 from __future__ import print_function
 
 # Module imports
 # Std lib
 import inspect
 import sys
 
 # Third party
 import nose.tools as nt
 
 # Our own
 from IPython.testing import decorators as dec
-from IPython.testing.skipdoctest import skip_doctest
 
 #-----------------------------------------------------------------------------
 # Utilities
 
 # Note: copied from OInspect, kept here so the testing stuff doesn't create
 # circular dependencies and is easier to reuse.
 def getargspec(obj):
     """Get the names and default values of a function's arguments.
 
     A tuple of four things is returned: (args, varargs, varkw, defaults).
     'args' is a list of the argument names (it may contain nested lists).
     'varargs' and 'varkw' are the names of the * and ** arguments or None.
     'defaults' is an n-tuple of the default values of the last n arguments.
 
     Modified version of inspect.getargspec from the Python Standard
     Library."""
 
     if inspect.isfunction(obj):
         func_obj = obj
     elif inspect.ismethod(obj):
         func_obj = obj.__func__
     else:
         raise TypeError('arg is not a Python function')
     args, varargs, varkw = inspect.getargs(func_obj.__code__)
     return args, varargs, varkw, func_obj.__defaults__
 
 #-----------------------------------------------------------------------------
 # Testing functions
 
 @dec.as_unittest
 def trivial():
     """A trivial test"""
     pass
 
 
 @dec.skip
 def test_deliberately_broken():
     """A deliberately broken test - we want to skip this one."""
     1/0
 
 @dec.skip('Testing the skip decorator')
 def test_deliberately_broken2():
     """Another deliberately broken test - we want to skip this one."""
     1/0
 
 
 # Verify that we can correctly skip the doctest for a function at will, but
 # that the docstring itself is NOT destroyed by the decorator.
-@skip_doctest
 def doctest_bad(x,y=1,**k):
     """A function whose doctest we need to skip.
 
     >>> 1+1
     3
     """
     print('x:',x)
     print('y:',y)
     print('k:',k)
 
 
 def call_doctest_bad():
     """Check that we can still call the decorated functions.
     
     >>> doctest_bad(3,y=4)
     x: 3
     y: 4
     k: {}
     """
     pass
 
 
 def test_skip_dt_decorator():
     """Doctest-skipping decorator should preserve the docstring.
     """
     # Careful: 'check' must be a *verbatim* copy of the doctest_bad docstring!
     check = """A function whose doctest we need to skip.
 
     >>> 1+1
     3
     """
     # Fetch the docstring from doctest_bad after decoration.
     val = doctest_bad.__doc__
     
     nt.assert_equal(check,val,"doctest_bad docstrings don't match")
 
 
 # Doctest skipping should work for class methods too
 class FooClass(object):
     """FooClass
 
     Example:
 
     >>> 1+1
     2
     """
 
-    @skip_doctest
     def __init__(self,x):
         """Make a FooClass.
 
         Example:
 
         >>> f = FooClass(3)
         junk
         """
         print('Making a FooClass.')
         self.x = x
         
-    @skip_doctest
     def bar(self,y):
         """Example:
 
         >>> ff = FooClass(3)
         >>> ff.bar(0)
         boom!
         >>> 1/0
         bam!
         """
         return 1/y
 
     def baz(self,y):
         """Example:
 
         >>> ff2 = FooClass(3)
         Making a FooClass.
         >>> ff2.baz(3)
         True
         """
         return self.x==y
 
 
 def test_skip_dt_decorator2():
     """Doctest-skipping decorator should preserve function signature.
     """
     # Hardcoded correct answer
     dtargs = (['x', 'y'], None, 'k', (1,))
     # Introspect out the value
     dtargsr = getargspec(doctest_bad)
     assert dtargsr==dtargs, \
            "Incorrectly reconstructed args for doctest_bad: %s" % (dtargsr,)
 
 
 @dec.skip_linux
 def test_linux():
     nt.assert_false(sys.platform.startswith('linux'),"This test can't run under linux")
 
 @dec.skip_win32
 def test_win32():
     nt.assert_not_equal(sys.platform,'win32',"This test can't run under windows")
 
 @dec.skip_osx
 def test_osx():
     nt.assert_not_equal(sys.platform,'darwin',"This test can't run under osx")
 

IPython/utils/path.py

@@ -1,448 +1,446 b''
 # encoding: utf-8
 """
 Utilities for path handling.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import os
 import sys
 import errno
 import shutil
 import random
 import tempfile
 import glob
 from warnings import warn
 from hashlib import md5
 
-from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.process import system
 from IPython.utils import py3compat
 from IPython.utils.decorators import undoc
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 fs_encoding = sys.getfilesystemencoding()
 
 def _writable_dir(path):
     """Whether `path` is a directory, to which the user has write access."""
     return os.path.isdir(path) and os.access(path, os.W_OK)
 
 if sys.platform == 'win32':
-    @skip_doctest
     def _get_long_path_name(path):
         """Get a long path name (expand ~) on Windows using ctypes.
 
         Examples
         --------
 
         >>> get_long_path_name('c:\\docume~1')
         u'c:\\\\Documents and Settings'
 
         """
         try:
             import ctypes
         except ImportError:
             raise ImportError('you need to have ctypes installed for this to work')
         _GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
         _GetLongPathName.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p,
             ctypes.c_uint ]
 
         buf = ctypes.create_unicode_buffer(260)
         rv = _GetLongPathName(path, buf, 260)
         if rv == 0 or rv > 260:
             return path
         else:
             return buf.value
 else:
     def _get_long_path_name(path):
         """Dummy no-op."""
         return path
 
 
 
 def get_long_path_name(path):
     """Expand a path into its long form.
 
     On Windows this expands any ~ in the paths. On other platforms, it is
     a null operation.
     """
     return _get_long_path_name(path)
 
 
 def unquote_filename(name, win32=(sys.platform=='win32')):
     """ On Windows, remove leading and trailing quotes from filenames.
     """
     if win32:
         if name.startswith(("'", '"')) and name.endswith(("'", '"')):
             name = name[1:-1]
     return name
 
 def compress_user(path):
     """Reverse of :func:`os.path.expanduser`
     """
     home = os.path.expanduser('~')
     if path.startswith(home):
         path =  "~" + path[len(home):]
     return path
 
 def get_py_filename(name, force_win32=None):
     """Return a valid python filename in the current directory.
 
     If the given name is not a file, it adds '.py' and searches again.
     Raises IOError with an informative message if the file isn't found.
 
     On Windows, apply Windows semantics to the filename. In particular, remove
     any quoting that has been applied to it. This option can be forced for
     testing purposes.
     """
 
     name = os.path.expanduser(name)
     if force_win32 is None:
         win32 = (sys.platform == 'win32')
     else:
         win32 = force_win32
     name = unquote_filename(name, win32=win32)
     if not os.path.isfile(name) and not name.endswith('.py'):
         name += '.py'
     if os.path.isfile(name):
         return name
     else:
         raise IOError('File `%r` not found.' % name)
 
 
 def filefind(filename, path_dirs=None):
     """Find a file by looking through a sequence of paths.
 
     This iterates through a sequence of paths looking for a file and returns
     the full, absolute path of the first occurence of the file.  If no set of
     path dirs is given, the filename is tested as is, after running through
     :func:`expandvars` and :func:`expanduser`.  Thus a simple call::
 
         filefind('myfile.txt')
 
     will find the file in the current working dir, but::
 
         filefind('~/myfile.txt')
 
     Will find the file in the users home directory.  This function does not
     automatically try any paths, such as the cwd or the user's home directory.
 
     Parameters
     ----------
     filename : str
         The filename to look for.
     path_dirs : str, None or sequence of str
         The sequence of paths to look for the file in.  If None, the filename
         need to be absolute or be in the cwd.  If a string, the string is
         put into a sequence and the searched.  If a sequence, walk through
         each element and join with ``filename``, calling :func:`expandvars`
         and :func:`expanduser` before testing for existence.
 
     Returns
     -------
     Raises :exc:`IOError` or returns absolute path to file.
     """
 
     # If paths are quoted, abspath gets confused, strip them...
     filename = filename.strip('"').strip("'")
     # If the input is an absolute path, just check it exists
     if os.path.isabs(filename) and os.path.isfile(filename):
         return filename
 
     if path_dirs is None:
         path_dirs = ("",)
     elif isinstance(path_dirs, py3compat.string_types):
         path_dirs = (path_dirs,)
 
     for path in path_dirs:
         if path == '.': path = py3compat.getcwd()
         testname = expand_path(os.path.join(path, filename))
         if os.path.isfile(testname):
             return os.path.abspath(testname)
 
     raise IOError("File %r does not exist in any of the search paths: %r" %
                   (filename, path_dirs) )
 
 
 class HomeDirError(Exception):
     pass
 
 
 def get_home_dir(require_writable=False):
     """Return the 'home' directory, as a unicode string.
 
     Uses os.path.expanduser('~'), and checks for writability.
     
     See stdlib docs for how this is determined.
     $HOME is first priority on *ALL* platforms.
     
     Parameters
     ----------
     
     require_writable : bool [default: False]
         if True:
             guarantees the return value is a writable directory, otherwise
             raises HomeDirError
         if False:
             The path is resolved, but it is not guaranteed to exist or be writable.
     """
 
     homedir = os.path.expanduser('~')
     # Next line will make things work even when /home/ is a symlink to
     # /usr/home as it is on FreeBSD, for example
     homedir = os.path.realpath(homedir)
     
     if not _writable_dir(homedir) and os.name == 'nt':
         # expanduser failed, use the registry to get the 'My Documents' folder.
         try:
             try:
                 import winreg as wreg  # Py 3
             except ImportError:
                 import _winreg as wreg  # Py 2
             key = wreg.OpenKey(
                 wreg.HKEY_CURRENT_USER,
                 "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
             )
             homedir = wreg.QueryValueEx(key,'Personal')[0]
             key.Close()
         except:
             pass
     
     if (not require_writable) or _writable_dir(homedir):
         return py3compat.cast_unicode(homedir, fs_encoding)
     else:
         raise HomeDirError('%s is not a writable dir, '
                 'set $HOME environment variable to override' % homedir)
 
 def get_xdg_dir():
     """Return the XDG_CONFIG_HOME, if it is defined and exists, else None.
 
     This is only for non-OS X posix (Linux,Unix,etc.) systems.
     """
 
     env = os.environ
 
     if os.name == 'posix' and sys.platform != 'darwin':
         # Linux, Unix, AIX, etc.
         # use ~/.config if empty OR not set
         xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config')
         if xdg and _writable_dir(xdg):
             return py3compat.cast_unicode(xdg, fs_encoding)
 
     return None
 
 
 def get_xdg_cache_dir():
     """Return the XDG_CACHE_HOME, if it is defined and exists, else None.
 
     This is only for non-OS X posix (Linux,Unix,etc.) systems.
     """
 
     env = os.environ
 
     if os.name == 'posix' and sys.platform != 'darwin':
         # Linux, Unix, AIX, etc.
         # use ~/.cache if empty OR not set
         xdg = env.get("XDG_CACHE_HOME", None) or os.path.join(get_home_dir(), '.cache')
         if xdg and _writable_dir(xdg):
             return py3compat.cast_unicode(xdg, fs_encoding)
 
     return None
 
 
 @undoc
 def get_ipython_dir():
     warn("get_ipython_dir has moved to the IPython.paths module")
     from IPython.paths import get_ipython_dir
     return get_ipython_dir()
 
 @undoc
 def get_ipython_cache_dir():
     warn("get_ipython_cache_dir has moved to the IPython.paths module")
     from IPython.paths import get_ipython_cache_dir
     return get_ipython_cache_dir()
 
 @undoc
 def get_ipython_package_dir():
     warn("get_ipython_package_dir has moved to the IPython.paths module")
     from IPython.paths import get_ipython_package_dir
     return get_ipython_package_dir()
 
 @undoc
 def get_ipython_module_path(module_str):
     warn("get_ipython_module_path has moved to the IPython.paths module")
     from IPython.paths import get_ipython_module_path
     return get_ipython_module_path(module_str)
 
 @undoc
 def locate_profile(profile='default'):
     warn("locate_profile has moved to the IPython.paths module")
     from IPython.paths import locate_profile
     return locate_profile(profile=profile)
 
 def expand_path(s):
     """Expand $VARS and ~names in a string, like a shell
 
     :Examples:
 
        In [2]: os.environ['FOO']='test'
 
        In [3]: expand_path('variable FOO is $FOO')
        Out[3]: 'variable FOO is test'
     """
     # This is a pretty subtle hack. When expand user is given a UNC path
     # on Windows (\\server\share$\%username%), os.path.expandvars, removes
     # the $ to get (\\server\share\%username%). I think it considered $
     # alone an empty var. But, we need the $ to remains there (it indicates
     # a hidden share).
     if os.name=='nt':
         s = s.replace('$\\', 'IPYTHON_TEMP')
     s = os.path.expandvars(os.path.expanduser(s))
     if os.name=='nt':
         s = s.replace('IPYTHON_TEMP', '$\\')
     return s
 
 
 def unescape_glob(string):
     """Unescape glob pattern in `string`."""
     def unescape(s):
         for pattern in '*[]!?':
             s = s.replace(r'\{0}'.format(pattern), pattern)
         return s
     return '\\'.join(map(unescape, string.split('\\\\')))
 
 
 def shellglob(args):
     """
     Do glob expansion for each element in `args` and return a flattened list.
 
     Unmatched glob pattern will remain as-is in the returned list.
 
     """
     expanded = []
     # Do not unescape backslash in Windows as it is interpreted as
     # path separator:
     unescape = unescape_glob if sys.platform != 'win32' else lambda x: x
     for a in args:
         expanded.extend(glob.glob(a) or [unescape(a)])
     return expanded
 
 
 def target_outdated(target,deps):
     """Determine whether a target is out of date.
 
     target_outdated(target,deps) -> 1/0
 
     deps: list of filenames which MUST exist.
     target: single filename which may or may not exist.
 
     If target doesn't exist or is older than any file listed in deps, return
     true, otherwise return false.
     """
     try:
         target_time = os.path.getmtime(target)
     except os.error:
         return 1
     for dep in deps:
         dep_time = os.path.getmtime(dep)
         if dep_time > target_time:
             #print "For target",target,"Dep failed:",dep # dbg
             #print "times (dep,tar):",dep_time,target_time # dbg
             return 1
     return 0
 
 
 def target_update(target,deps,cmd):
     """Update a target with a given command given a list of dependencies.
 
     target_update(target,deps,cmd) -> runs cmd if target is outdated.
 
     This is just a wrapper around target_outdated() which calls the given
     command if target is outdated."""
 
     if target_outdated(target,deps):
         system(cmd)
 
 @undoc
 def filehash(path):
     """Make an MD5 hash of a file, ignoring any differences in line
     ending characters."""
     warn("filehash() is deprecated")
     with open(path, "rU") as f:
         return md5(py3compat.str_to_bytes(f.read())).hexdigest()
 
 ENOLINK = 1998
 
 def link(src, dst):
     """Hard links ``src`` to ``dst``, returning 0 or errno.
 
     Note that the special errno ``ENOLINK`` will be returned if ``os.link`` isn't
     supported by the operating system.
     """
 
     if not hasattr(os, "link"):
         return ENOLINK
     link_errno = 0
     try:
         os.link(src, dst)
     except OSError as e:
         link_errno = e.errno
     return link_errno
 
 
 def link_or_copy(src, dst):
     """Attempts to hardlink ``src`` to ``dst``, copying if the link fails.
 
     Attempts to maintain the semantics of ``shutil.copy``.
 
     Because ``os.link`` does not overwrite files, a unique temporary file
     will be used if the target already exists, then that file will be moved
     into place.
     """
 
     if os.path.isdir(dst):
         dst = os.path.join(dst, os.path.basename(src))
 
     link_errno = link(src, dst)
     if link_errno == errno.EEXIST:
         if os.stat(src).st_ino == os.stat(dst).st_ino:
             # dst is already a hard link to the correct file, so we don't need
             # to do anything else. If we try to link and rename the file
             # anyway, we get duplicate files - see http://bugs.python.org/issue21876
             return
 
         new_dst = dst + "-temp-%04X" %(random.randint(1, 16**4), )
         try:
             link_or_copy(src, new_dst)
         except:
             try:
                 os.remove(new_dst)
             except OSError:
                 pass
             raise
         os.rename(new_dst, dst)
     elif link_errno != 0:
         # Either link isn't supported, or the filesystem doesn't support
         # linking, or 'src' and 'dst' are on different filesystems.
         shutil.copy(src, dst)
 
 def ensure_dir_exists(path, mode=0o755):
     """ensure that a directory exists
     
     If it doesn't exist, try to create it and protect against a race condition
     if another process is doing the same.
     
     The default permissions are 755, which differ from os.makedirs default of 777.
     """
     if not os.path.exists(path):
         try:
             os.makedirs(path, mode=mode)
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
     elif not os.path.isdir(path):
         raise IOError("%r exists but is not a directory" % path)

IPython/utils/text.py

@@ -1,765 +1,764 b''
 # encoding: utf-8
 """
 Utilities for working with strings and text.
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.utils.text
    :parts: 3
 """
 
 import os
 import re
 import sys
 import textwrap
 from string import Formatter
 
 from IPython.testing.skipdoctest import skip_doctest_py3, skip_doctest
 from IPython.utils import py3compat
 
 # datetime.strftime date format for ipython
 if sys.platform == 'win32':
     date_format = "%B %d, %Y"
 else:
     date_format = "%B %-d, %Y"
 
 class LSString(str):
     """String derivative with a special access attributes.
 
     These are normal strings, but with the special attributes:
 
         .l (or .list) : value as list (split on newlines).
         .n (or .nlstr): original value (the string itself).
         .s (or .spstr): value as whitespace-separated string.
         .p (or .paths): list of path objects (requires path.py package)
 
     Any values which require transformations are computed only once and
     cached.
 
     Such strings are very useful to efficiently interact with the shell, which
     typically only understands whitespace-separated options for commands."""
 
     def get_list(self):
         try:
             return self.__list
         except AttributeError:
             self.__list = self.split('\n')
             return self.__list
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = self.replace('\n',' ')
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         return self
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         from path import path
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self.split('\n') if os.path.exists(p)]
             return self.__paths
 
     p = paths = property(get_paths)
 
 # FIXME: We need to reimplement type specific displayhook and then add this
 # back as a custom printer. This should also be moved outside utils into the
 # core.
 
 # def print_lsstring(arg):
 #     """ Prettier (non-repr-like) and more informative printer for LSString """
 #     print "LSString (.p, .n, .l, .s available). Value:"
 #     print arg
 #
 #
 # print_lsstring = result_display.when_type(LSString)(print_lsstring)
 
 
 class SList(list):
     """List derivative with a special access attributes.
 
     These are normal lists, but with the special attributes:
 
     * .l (or .list) : value as list (the list itself).
     * .n (or .nlstr): value as a string, joined on newlines.
     * .s (or .spstr): value as a string, joined on spaces.
     * .p (or .paths): list of path objects (requires path.py package)
 
     Any values which require transformations are computed only once and
     cached."""
 
     def get_list(self):
         return self
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = ' '.join(self)
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         try:
             return self.__nlstr
         except AttributeError:
             self.__nlstr = '\n'.join(self)
             return self.__nlstr
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         from path import path
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self if os.path.exists(p)]
             return self.__paths
 
     p = paths = property(get_paths)
 
     def grep(self, pattern, prune = False, field = None):
         """ Return all strings matching 'pattern' (a regex or callable)
 
         This is case-insensitive. If prune is true, return all items
         NOT matching the pattern.
 
         If field is specified, the match must occur in the specified
         whitespace-separated field.
 
         Examples::
 
             a.grep( lambda x: x.startswith('C') )
             a.grep('Cha.*log', prune=1)
             a.grep('chm', field=-1)
         """
 
         def match_target(s):
             if field is None:
                 return s
             parts = s.split()
             try:
                 tgt = parts[field]
                 return tgt
             except IndexError:
                 return ""
 
         if isinstance(pattern, py3compat.string_types):
             pred = lambda x : re.search(pattern, x, re.IGNORECASE)
         else:
             pred = pattern
         if not prune:
             return SList([el for el in self if pred(match_target(el))])
         else:
             return SList([el for el in self if not pred(match_target(el))])
 
     def fields(self, *fields):
         """ Collect whitespace-separated fields from string list
 
         Allows quick awk-like usage of string lists.
 
         Example data (in var a, created by 'a = !ls -l')::
 
             -rwxrwxrwx  1 ville None      18 Dec 14  2006 ChangeLog
             drwxrwxrwx+ 6 ville None       0 Oct 24 18:05 IPython
 
         * ``a.fields(0)`` is ``['-rwxrwxrwx', 'drwxrwxrwx+']``
         * ``a.fields(1,0)`` is ``['1 -rwxrwxrwx', '6 drwxrwxrwx+']``
           (note the joining by space).
         * ``a.fields(-1)`` is ``['ChangeLog', 'IPython']``
 
         IndexErrors are ignored.
 
         Without args, fields() just split()'s the strings.
         """
         if len(fields) == 0:
             return [el.split() for el in self]
 
         res = SList()
         for el in [f.split() for f in self]:
             lineparts = []
 
             for fd in fields:
                 try:
                     lineparts.append(el[fd])
                 except IndexError:
                     pass
             if lineparts:
                 res.append(" ".join(lineparts))
 
         return res
 
     def sort(self,field= None,  nums = False):
         """ sort by specified fields (see fields())
 
         Example::
 
             a.sort(1, nums = True)
 
         Sorts a by second field, in numerical order (so that 21 > 3)
 
         """
 
         #decorate, sort, undecorate
         if field is not None:
             dsu = [[SList([line]).fields(field),  line] for line in self]
         else:
             dsu = [[line,  line] for line in self]
         if nums:
             for i in range(len(dsu)):
                 numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
                 try:
                     n = int(numstr)
                 except ValueError:
                     n = 0;
                 dsu[i][0] = n
 
 
         dsu.sort()
         return SList([t[1] for t in dsu])
 
 
 # FIXME: We need to reimplement type specific displayhook and then add this
 # back as a custom printer. This should also be moved outside utils into the
 # core.
 
 # def print_slist(arg):
 #     """ Prettier (non-repr-like) and more informative printer for SList """
 #     print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
 #     if hasattr(arg,  'hideonce') and arg.hideonce:
 #         arg.hideonce = False
 #         return
 #
 #     nlprint(arg)   # This was a nested list printer, now removed.
 #
 # print_slist = result_display.when_type(SList)(print_slist)
 
 
 def indent(instr,nspaces=4, ntabs=0, flatten=False):
     """Indent a string a given number of spaces or tabstops.
 
     indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
 
     Parameters
     ----------
 
     instr : basestring
         The string to be indented.
     nspaces : int (default: 4)
         The number of spaces to be indented.
     ntabs : int (default: 0)
         The number of tabs to be indented.
     flatten : bool (default: False)
         Whether to scrub existing indentation.  If True, all lines will be
         aligned to the same indentation.  If False, existing indentation will
         be strictly increased.
 
     Returns
     -------
 
     str|unicode : string indented by ntabs and nspaces.
 
     """
     if instr is None:
         return
     ind = '\t'*ntabs+' '*nspaces
     if flatten:
         pat = re.compile(r'^\s*', re.MULTILINE)
     else:
         pat = re.compile(r'^', re.MULTILINE)
     outstr = re.sub(pat, ind, instr)
     if outstr.endswith(os.linesep+ind):
         return outstr[:-len(ind)]
     else:
         return outstr
 
 
 def list_strings(arg):
     """Always return a list of strings, given a string or list of strings
     as input.
 
     Examples
     --------
     ::
 
         In [7]: list_strings('A single string')
         Out[7]: ['A single string']
 
         In [8]: list_strings(['A single string in a list'])
         Out[8]: ['A single string in a list']
 
         In [9]: list_strings(['A','list','of','strings'])
         Out[9]: ['A', 'list', 'of', 'strings']
     """
 
     if isinstance(arg, py3compat.string_types): return [arg]
     else: return arg
 
 
 def marquee(txt='',width=78,mark='*'):
     """Return the input string centered in a 'marquee'.
 
     Examples
     --------
     ::
 
         In [16]: marquee('A test',40)
         Out[16]: '**************** A test ****************'
 
         In [17]: marquee('A test',40,'-')
         Out[17]: '---------------- A test ----------------'
 
         In [18]: marquee('A test',40,' ')
         Out[18]: '                 A test                 '
 
     """
     if not txt:
         return (mark*width)[:width]
     nmark = (width-len(txt)-2)//len(mark)//2
     if nmark < 0: nmark =0
     marks = mark*nmark
     return '%s %s %s' % (marks,txt,marks)
 
 
 ini_spaces_re = re.compile(r'^(\s+)')
 
 def num_ini_spaces(strng):
     """Return the number of initial spaces in a string"""
 
     ini_spaces = ini_spaces_re.match(strng)
     if ini_spaces:
         return ini_spaces.end()
     else:
         return 0
 
 
 def format_screen(strng):
     """Format a string for screen printing.
 
     This removes some latex-type format codes."""
     # Paragraph continue
     par_re = re.compile(r'\\$',re.MULTILINE)
     strng = par_re.sub('',strng)
     return strng
 
 
 def dedent(text):
     """Equivalent of textwrap.dedent that ignores unindented first line.
 
     This means it will still dedent strings like:
     '''foo
     is a bar
     '''
 
     For use in wrap_paragraphs.
     """
 
     if text.startswith('\n'):
         # text starts with blank line, don't ignore the first line
         return textwrap.dedent(text)
 
     # split first line
     splits = text.split('\n',1)
     if len(splits) == 1:
         # only one line
         return textwrap.dedent(text)
 
     first, rest = splits
     # dedent everything but the first line
     rest = textwrap.dedent(rest)
     return '\n'.join([first, rest])
 
 
 def wrap_paragraphs(text, ncols=80):
     """Wrap multiple paragraphs to fit a specified width.
 
     This is equivalent to textwrap.wrap, but with support for multiple
     paragraphs, as separated by empty lines.
 
     Returns
     -------
 
     list of complete paragraphs, wrapped to fill `ncols` columns.
     """
     paragraph_re = re.compile(r'\n(\s*\n)+', re.MULTILINE)
     text = dedent(text).strip()
     paragraphs = paragraph_re.split(text)[::2] # every other entry is space
     out_ps = []
     indent_re = re.compile(r'\n\s+', re.MULTILINE)
     for p in paragraphs:
         # presume indentation that survives dedent is meaningful formatting,
         # so don't fill unless text is flush.
         if indent_re.search(p) is None:
             # wrap paragraph
             p = textwrap.fill(p, ncols)
         out_ps.append(p)
     return out_ps
 
 
 def long_substr(data):
     """Return the longest common substring in a list of strings.
     
     Credit: http://stackoverflow.com/questions/2892931/longest-common-substring-from-more-than-two-strings-python
     """
     substr = ''
     if len(data) > 1 and len(data[0]) > 0:
         for i in range(len(data[0])):
             for j in range(len(data[0])-i+1):
                 if j > len(substr) and all(data[0][i:i+j] in x for x in data):
                     substr = data[0][i:i+j]
     elif len(data) == 1:
         substr = data[0]
     return substr
 
 
 def strip_email_quotes(text):
     """Strip leading email quotation characters ('>').
 
     Removes any combination of leading '>' interspersed with whitespace that
     appears *identically* in all lines of the input text.
 
     Parameters
     ----------
     text : str
 
     Examples
     --------
 
     Simple uses::
 
         In [2]: strip_email_quotes('> > text')
         Out[2]: 'text'
 
         In [3]: strip_email_quotes('> > text\\n> > more')
         Out[3]: 'text\\nmore'
 
     Note how only the common prefix that appears in all lines is stripped::
 
         In [4]: strip_email_quotes('> > text\\n> > more\\n> more...')
         Out[4]: '> text\\n> more\\nmore...'
 
     So if any line has no quote marks ('>') , then none are stripped from any
     of them ::
     
         In [5]: strip_email_quotes('> > text\\n> > more\\nlast different')
         Out[5]: '> > text\\n> > more\\nlast different'
     """
     lines = text.splitlines()
     matches = set()
     for line in lines:
         prefix = re.match(r'^(\s*>[ >]*)', line)
         if prefix:
             matches.add(prefix.group(1))
         else:
             break
     else:
         prefix = long_substr(list(matches))
         if prefix:
             strip = len(prefix)
             text = '\n'.join([ ln[strip:] for ln in lines])
     return text
 
 def strip_ansi(source):
     """
     Remove ansi escape codes from text.
     
     Parameters
     ----------
     source : str
         Source to remove the ansi from
     """
     return re.sub(r'\033\[(\d|;)+?m', '', source)
 
 
 class EvalFormatter(Formatter):
     """A String Formatter that allows evaluation of simple expressions.
     
     Note that this version interprets a : as specifying a format string (as per
     standard string formatting), so if slicing is required, you must explicitly
     create a slice.
     
     This is to be used in templating cases, such as the parallel batch
     script templates, where simple arithmetic on arguments is useful.
 
     Examples
     --------
     ::
 
         In [1]: f = EvalFormatter()
         In [2]: f.format('{n//4}', n=8)
         Out[2]: '2'
 
         In [3]: f.format("{greeting[slice(2,4)]}", greeting="Hello")
         Out[3]: 'll'
     """
     def get_field(self, name, args, kwargs):
         v = eval(name, kwargs)
         return v, name
 
 #XXX: As of Python 3.4, the format string parsing no longer splits on a colon
 # inside [], so EvalFormatter can handle slicing. Once we only support 3.4 and
 # above, it should be possible to remove FullEvalFormatter.
 
 @skip_doctest_py3
 class FullEvalFormatter(Formatter):
     """A String Formatter that allows evaluation of simple expressions.
     
     Any time a format key is not found in the kwargs,
     it will be tried as an expression in the kwargs namespace.
     
     Note that this version allows slicing using [1:2], so you cannot specify
     a format string. Use :class:`EvalFormatter` to permit format strings.
     
     Examples
     --------
     ::
 
         In [1]: f = FullEvalFormatter()
         In [2]: f.format('{n//4}', n=8)
         Out[2]: u'2'
 
         In [3]: f.format('{list(range(5))[2:4]}')
         Out[3]: u'[2, 3]'
 
         In [4]: f.format('{3*2}')
         Out[4]: u'6'
     """
     # copied from Formatter._vformat with minor changes to allow eval
     # and replace the format_spec code with slicing
     def _vformat(self, format_string, args, kwargs, used_args, recursion_depth):
         if recursion_depth < 0:
             raise ValueError('Max string recursion exceeded')
         result = []
         for literal_text, field_name, format_spec, conversion in \
                 self.parse(format_string):
 
             # output the literal text
             if literal_text:
                 result.append(literal_text)
 
             # if there's a field, output it
             if field_name is not None:
                 # this is some markup, find the object and do
                 # the formatting
 
                 if format_spec:
                     # override format spec, to allow slicing:
                     field_name = ':'.join([field_name, format_spec])
 
                 # eval the contents of the field for the object
                 # to be formatted
                 obj = eval(field_name, kwargs)
 
                 # do any conversion on the resulting object
                 obj = self.convert_field(obj, conversion)
 
                 # format the object and append to the result
                 result.append(self.format_field(obj, ''))
 
         return u''.join(py3compat.cast_unicode(s) for s in result)
 
 
 @skip_doctest_py3
 class DollarFormatter(FullEvalFormatter):
     """Formatter allowing Itpl style $foo replacement, for names and attribute
     access only. Standard {foo} replacement also works, and allows full
     evaluation of its arguments.
 
     Examples
     --------
     ::
 
         In [1]: f = DollarFormatter()
         In [2]: f.format('{n//4}', n=8)
         Out[2]: u'2'
 
         In [3]: f.format('23 * 76 is $result', result=23*76)
         Out[3]: u'23 * 76 is 1748'
 
         In [4]: f.format('$a or {b}', a=1, b=2)
         Out[4]: u'1 or 2'
     """
     _dollar_pattern = re.compile("(.*?)\$(\$?[\w\.]+)")
     def parse(self, fmt_string):
         for literal_txt, field_name, format_spec, conversion \
                     in Formatter.parse(self, fmt_string):
             
             # Find $foo patterns in the literal text.
             continue_from = 0
             txt = ""
             for m in self._dollar_pattern.finditer(literal_txt):
                 new_txt, new_field = m.group(1,2)
                 # $$foo --> $foo
                 if new_field.startswith("$"):
                     txt += new_txt + new_field
                 else:
                     yield (txt + new_txt, new_field, "", None)
                     txt = ""
                 continue_from = m.end()
             
             # Re-yield the {foo} style pattern
             yield (txt + literal_txt[continue_from:], field_name, format_spec, conversion)
 
 #-----------------------------------------------------------------------------
 # Utils to columnize a list of string
 #-----------------------------------------------------------------------------
 
 def _chunks(l, n):
     """Yield successive n-sized chunks from l."""
     for i in py3compat.xrange(0, len(l), n):
         yield l[i:i+n]
 
 
 def _find_optimal(rlist , separator_size=2 , displaywidth=80):
     """Calculate optimal info to columnize a list of string"""
     for nrow in range(1, len(rlist)+1) :
         chk = list(map(max,_chunks(rlist, nrow)))
         sumlength = sum(chk)
         ncols = len(chk)
         if sumlength+separator_size*(ncols-1) <= displaywidth :
             break;
     return {'columns_numbers' : ncols,
             'optimal_separator_width':(displaywidth - sumlength)/(ncols-1) if (ncols -1) else 0,
             'rows_numbers' : nrow,
             'columns_width' : chk
            }
 
 
 def _get_or_default(mylist, i, default=None):
     """return list item number, or default if don't exist"""
     if i >= len(mylist):
         return default
     else :
         return mylist[i]
 
 
-@skip_doctest
 def compute_item_matrix(items, empty=None, *args, **kwargs) :
     """Returns a nested list, and info to columnize items
 
     Parameters
     ----------
 
     items
         list of strings to columize
     empty : (default None)
         default value to fill list if needed
     separator_size : int (default=2)
         How much caracters will be used as a separation between each columns.
     displaywidth : int (default=80)
         The width of the area onto wich the columns should enter
 
     Returns
     -------
 
     strings_matrix
 
         nested list of string, the outer most list contains as many list as
         rows, the innermost lists have each as many element as colums. If the
         total number of elements in `items` does not equal the product of
         rows*columns, the last element of some lists are filled with `None`.
 
     dict_info
         some info to make columnize easier:
 
         columns_numbers
           number of columns
         rows_numbers
           number of rows
         columns_width
           list of with of each columns
         optimal_separator_width
           best separator width between columns
 
     Examples
     --------
     ::
 
         In [1]: l = ['aaa','b','cc','d','eeeee','f','g','h','i','j','k','l']
            ...: compute_item_matrix(l,displaywidth=12)
         Out[1]:
             ([['aaa', 'f', 'k'],
             ['b', 'g', 'l'],
             ['cc', 'h', None],
             ['d', 'i', None],
             ['eeeee', 'j', None]],
             {'columns_numbers': 3,
             'columns_width': [5, 1, 1],
             'optimal_separator_width': 2,
             'rows_numbers': 5})
     """
     info = _find_optimal(list(map(len, items)), *args, **kwargs)
     nrow, ncol = info['rows_numbers'], info['columns_numbers']
     return ([[ _get_or_default(items, c*nrow+i, default=empty) for c in range(ncol) ] for i in range(nrow) ], info)
 
 
 def columnize(items, separator='  ', displaywidth=80):
     """ Transform a list of strings into a single string with columns.
 
     Parameters
     ----------
     items : sequence of strings
         The strings to process.
 
     separator : str, optional [default is two spaces]
         The string that separates columns.
 
     displaywidth : int, optional [default is 80]
         Width of the display in number of characters.
 
     Returns
     -------
     The formatted string.
     """
     if not items :
         return '\n'
     matrix, info = compute_item_matrix(items, separator_size=len(separator), displaywidth=displaywidth)
     fmatrix = [filter(None, x) for x in matrix]
     sjoin = lambda x : separator.join([ y.ljust(w, ' ') for y, w in zip(x, info['columns_width'])])
     return '\n'.join(map(sjoin, fmatrix))+'\n'
 
 
 def get_text_list(list_, last_sep=' and ', sep=", ", wrap_item_with=""):
     """
     Return a string with a natural enumeration of items
 
     >>> get_text_list(['a', 'b', 'c', 'd'])
     'a, b, c and d'
     >>> get_text_list(['a', 'b', 'c'], ' or ')
     'a, b or c'
     >>> get_text_list(['a', 'b', 'c'], ', ')
     'a, b, c'
     >>> get_text_list(['a', 'b'], ' or ')
     'a or b'
     >>> get_text_list(['a'])
     'a'
     >>> get_text_list([])
     ''
     >>> get_text_list(['a', 'b'], wrap_item_with="`")
     '`a` and `b`'
     >>> get_text_list(['a', 'b', 'c', 'd'], " = ", sep=" + ")
     'a + b + c = d'
     """
     if len(list_) == 0:
         return ''
     if wrap_item_with:
         list_ = ['%s%s%s' % (wrap_item_with, item, wrap_item_with) for
                  item in list_]
     if len(list_) == 1:
         return list_[0]
     return '%s%s%s' % (
         sep.join(i for i in list_[:-1]),
         last_sep, list_[-1])
  No newline at end of file

ipython_kernel/zmqshell.py

@@ -1,486 +1,484 b''
 """A ZMQ-based subclass of InteractiveShell.
 
 This code is meant to ease the refactoring of the base InteractiveShell into
 something with a cleaner architecture for 2-process use, without actually
 breaking InteractiveShell itself.  So we're doing something a bit ugly, where
 we subclass and override what we want to fix.  Once this is working well, we
 can go back to the base class and refactor the code for a cleaner inheritance
 implementation that doesn't rely on so much monkeypatching.
 
 But this lets us maintain a fully working IPython as we develop the new
 machinery.  This should thus be thought of as scaffolding.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 
 import os
 import sys
 import time
 
 from zmq.eventloop import ioloop
 
 from IPython.core.interactiveshell import (
     InteractiveShell, InteractiveShellABC
 )
 from IPython.core import page
 from IPython.core.autocall import ZMQExitAutocall
 from IPython.core.displaypub import DisplayPublisher
 from IPython.core.error import UsageError
 from IPython.core.magics import MacroToEdit, CodeMagics
 from IPython.core.magic import magics_class, line_magic, Magics
 from IPython.core import payloadpage
 from IPython.core.usage import default_gui_banner
 from IPython.display import display, Javascript
 from ipython_kernel.inprocess.socket import SocketABC
 from ipython_kernel import (
     get_connection_file, get_connection_info, connect_qtconsole
 )
-from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import openpy
 from jupyter_client.jsonutil import json_clean, encode_images
 from IPython.utils.process import arg_split
 from IPython.utils import py3compat
 from IPython.utils.py3compat import unicode_type
 from IPython.utils.traitlets import Instance, Type, Dict, CBool, CBytes, Any
 from IPython.utils.warn import error
 from ipython_kernel.displayhook import ZMQShellDisplayHook
 from ipython_kernel.datapub import ZMQDataPublisher
 from ipython_kernel.session import extract_header
 from .session import Session
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 class ZMQDisplayPublisher(DisplayPublisher):
     """A display publisher that publishes data using a ZeroMQ PUB socket."""
 
     session = Instance(Session, allow_none=True)
     pub_socket = Instance(SocketABC, allow_none=True)
     parent_header = Dict({})
     topic = CBytes(b'display_data')
 
     def set_parent(self, parent):
         """Set the parent for outbound messages."""
         self.parent_header = extract_header(parent)
 
     def _flush_streams(self):
         """flush IO Streams prior to display"""
         sys.stdout.flush()
         sys.stderr.flush()
 
     def publish(self, data, metadata=None, source=None):
         self._flush_streams()
         if metadata is None:
             metadata = {}
         self._validate_data(data, metadata)
         content = {}
         content['data'] = encode_images(data)
         content['metadata'] = metadata
         self.session.send(
             self.pub_socket, u'display_data', json_clean(content),
             parent=self.parent_header, ident=self.topic,
         )
 
     def clear_output(self, wait=False):
         content = dict(wait=wait)
         self._flush_streams()
         self.session.send(
             self.pub_socket, u'clear_output', content,
             parent=self.parent_header, ident=self.topic,
         )
 
 @magics_class
 class KernelMagics(Magics):
     #------------------------------------------------------------------------
     # Magic overrides
     #------------------------------------------------------------------------
     # Once the base class stops inheriting from magic, this code needs to be
     # moved into a separate machinery as well.  For now, at least isolate here
     # the magics which this class needs to implement differently from the base
     # class, or that are unique to it.
 
     _find_edit_target = CodeMagics._find_edit_target
 
-    @skip_doctest
     @line_magic
     def edit(self, parameter_s='', last_call=['','']):
         """Bring up an editor and execute the resulting code.
 
         Usage:
           %edit [options] [args]
 
         %edit runs an external text editor. You will need to set the command for
         this editor via the ``TerminalInteractiveShell.editor`` option in your
         configuration file before it will work.
 
         This command allows you to conveniently edit multi-line code right in
         your IPython session.
 
         If called without arguments, %edit opens up an empty editor with a
         temporary file and will execute the contents of this file when you
         close it (don't forget to save it!).
 
         Options:
 
         -n <number>
           Open the editor at a specified line number. By default, the IPython
           editor hook uses the unix syntax 'editor +N filename', but you can
           configure this by providing your own modified hook if your favorite
           editor supports line-number specifications with a different syntax.
 
         -p
           Call the editor with the same data as the previous time it was used,
           regardless of how long ago (in your current session) it was.
 
         -r
           Use 'raw' input. This option only applies to input taken from the
           user's history.  By default, the 'processed' history is used, so that
           magics are loaded in their transformed version to valid Python.  If
           this option is given, the raw input as typed as the command line is
           used instead.  When you exit the editor, it will be executed by
           IPython's own processor.
 
         Arguments:
 
         If arguments are given, the following possibilites exist:
 
         - The arguments are numbers or pairs of colon-separated numbers (like
           1 4:8 9). These are interpreted as lines of previous input to be
           loaded into the editor. The syntax is the same of the %macro command.
 
         - If the argument doesn't start with a number, it is evaluated as a
           variable and its contents loaded into the editor. You can thus edit
           any string which contains python code (including the result of
           previous edits).
 
         - If the argument is the name of an object (other than a string),
           IPython will try to locate the file where it was defined and open the
           editor at the point where it is defined. You can use ``%edit function``
           to load an editor exactly at the point where 'function' is defined,
           edit it and have the file be executed automatically.
 
           If the object is a macro (see %macro for details), this opens up your
           specified editor with a temporary file containing the macro's data.
           Upon exit, the macro is reloaded with the contents of the file.
 
           Note: opening at an exact line is only supported under Unix, and some
           editors (like kedit and gedit up to Gnome 2.8) do not understand the
           '+NUMBER' parameter necessary for this feature. Good editors like
           (X)Emacs, vi, jed, pico and joe all do.
 
         - If the argument is not found as a variable, IPython will look for a
           file with that name (adding .py if necessary) and load it into the
           editor. It will execute its contents with execfile() when you exit,
           loading any code in the file into your interactive namespace.
 
         Unlike in the terminal, this is designed to use a GUI editor, and we do
         not know when it has closed. So the file you edit will not be
         automatically executed or printed.
 
         Note that %edit is also available through the alias %ed.
         """
 
         opts,args = self.parse_options(parameter_s,'prn:')
 
         try:
             filename, lineno, _ = CodeMagics._find_edit_target(self.shell, args, opts, last_call)
         except MacroToEdit as e:
             # TODO: Implement macro editing over 2 processes.
             print("Macro editing not yet implemented in 2-process model.")
             return
 
         # Make sure we send to the client an absolute path, in case the working
         # directory of client and kernel don't match
         filename = os.path.abspath(filename)
 
         payload = {
             'source' : 'edit_magic',
             'filename' : filename,
             'line_number' : lineno
         }
         self.shell.payload_manager.write_payload(payload)
 
     # A few magics that are adapted to the specifics of using pexpect and a
     # remote terminal
 
     @line_magic
     def clear(self, arg_s):
         """Clear the terminal."""
         if os.name == 'posix':
             self.shell.system("clear")
         else:
             self.shell.system("cls")
 
     if os.name == 'nt':
         # This is the usual name in windows
         cls = line_magic('cls')(clear)
 
     # Terminal pagers won't work over pexpect, but we do have our own pager
 
     @line_magic
     def less(self, arg_s):
         """Show a file through the pager.
 
         Files ending in .py are syntax-highlighted."""
         if not arg_s:
             raise UsageError('Missing filename.')
 
         if arg_s.endswith('.py'):
             cont = self.shell.pycolorize(openpy.read_py_file(arg_s, skip_encoding_cookie=False))
         else:
             cont = open(arg_s).read()
         page.page(cont)
 
     more = line_magic('more')(less)
 
     # Man calls a pager, so we also need to redefine it
     if os.name == 'posix':
         @line_magic
         def man(self, arg_s):
             """Find the man page for the given command and display in pager."""
             page.page(self.shell.getoutput('man %s | col -b' % arg_s,
                                            split=False))
 
     @line_magic
     def connect_info(self, arg_s):
         """Print information for connecting other clients to this kernel
 
         It will print the contents of this session's connection file, as well as
         shortcuts for local clients.
 
         In the simplest case, when called from the most recently launched kernel,
         secondary clients can be connected, simply with:
 
         $> ipython <app> --existing
 
         """
 
         from IPython.core.application import BaseIPythonApplication as BaseIPApp
 
         if BaseIPApp.initialized():
             app = BaseIPApp.instance()
             security_dir = app.profile_dir.security_dir
             profile = app.profile
         else:
             profile = 'default'
             security_dir = ''
 
         try:
             connection_file = get_connection_file()
             info = get_connection_info(unpack=False)
         except Exception as e:
             error("Could not get connection info: %r" % e)
             return
 
         # add profile flag for non-default profile
         profile_flag = "--profile %s" % profile if profile != 'default' else ""
 
         # if it's in the security dir, truncate to basename
         if security_dir == os.path.dirname(connection_file):
             connection_file = os.path.basename(connection_file)
 
 
         print (info + '\n')
         print ("Paste the above JSON into a file, and connect with:\n"
             "    $> ipython <app> --existing <file>\n"
             "or, if you are local, you can connect with just:\n"
             "    $> ipython <app> --existing {0} {1}\n"
             "or even just:\n"
             "    $> ipython <app> --existing {1}\n"
             "if this is the most recent IPython session you have started.".format(
             connection_file, profile_flag
             )
         )
 
     @line_magic
     def qtconsole(self, arg_s):
         """Open a qtconsole connected to this kernel.
 
         Useful for connecting a qtconsole to running notebooks, for better
         debugging.
         """
 
         # %qtconsole should imply bind_kernel for engines:
         try:
             from IPython.parallel import bind_kernel
         except ImportError:
             # technically possible, because parallel has higher pyzmq min-version
             pass
         else:
             bind_kernel()
 
         try:
             p = connect_qtconsole(argv=arg_split(arg_s, os.name=='posix'))
         except Exception as e:
             error("Could not start qtconsole: %r" % e)
             return
 
     @line_magic
     def autosave(self, arg_s):
         """Set the autosave interval in the notebook (in seconds).
 
         The default value is 120, or two minutes.
         ``%autosave 0`` will disable autosave.
 
         This magic only has an effect when called from the notebook interface.
         It has no effect when called in a startup file.
         """
 
         try:
             interval = int(arg_s)
         except ValueError:
             raise UsageError("%%autosave requires an integer, got %r" % arg_s)
 
         # javascript wants milliseconds
         milliseconds = 1000 * interval
         display(Javascript("IPython.notebook.set_autosave_interval(%i)" % milliseconds),
             include=['application/javascript']
         )
         if interval:
             print("Autosaving every %i seconds" % interval)
         else:
             print("Autosave disabled")
 
 
 class ZMQInteractiveShell(InteractiveShell):
     """A subclass of InteractiveShell for ZMQ."""
 
     displayhook_class = Type(ZMQShellDisplayHook)
     display_pub_class = Type(ZMQDisplayPublisher)
     data_pub_class = Type(ZMQDataPublisher)
     kernel = Any()
     parent_header = Any()
 
     def _banner1_default(self):
         return default_gui_banner
 
     # Override the traitlet in the parent class, because there's no point using
     # readline for the kernel. Can be removed when the readline code is moved
     # to the terminal frontend.
     colors_force = CBool(True)
     readline_use = CBool(False)
     # autoindent has no meaning in a zmqshell, and attempting to enable it
     # will print a warning in the absence of readline.
     autoindent = CBool(False)
 
     exiter = Instance(ZMQExitAutocall)
     def _exiter_default(self):
         return ZMQExitAutocall(self)
 
     def _exit_now_changed(self, name, old, new):
         """stop eventloop when exit_now fires"""
         if new:
             loop = ioloop.IOLoop.instance()
             loop.add_timeout(time.time()+0.1, loop.stop)
 
     keepkernel_on_exit = None
 
     # Over ZeroMQ, GUI control isn't done with PyOS_InputHook as there is no
     # interactive input being read; we provide event loop support in ipkernel
     @staticmethod
     def enable_gui(gui):
         from .eventloops import enable_gui as real_enable_gui
         try:
             real_enable_gui(gui)
         except ValueError as e:
             raise UsageError("%s" % e)
 
     def init_environment(self):
         """Configure the user's environment."""
         env = os.environ
         # These two ensure 'ls' produces nice coloring on BSD-derived systems
         env['TERM'] = 'xterm-color'
         env['CLICOLOR'] = '1'
         # Since normal pagers don't work at all (over pexpect we don't have
         # single-key control of the subprocess), try to disable paging in
         # subprocesses as much as possible.
         env['PAGER'] = 'cat'
         env['GIT_PAGER'] = 'cat'
 
     def init_hooks(self):
         super(ZMQInteractiveShell, self).init_hooks()
         self.set_hook('show_in_pager', page.as_hook(payloadpage.page), 99)
 
     def ask_exit(self):
         """Engage the exit actions."""
         self.exit_now = (not self.keepkernel_on_exit)
         payload = dict(
             source='ask_exit',
             keepkernel=self.keepkernel_on_exit,
             )
         self.payload_manager.write_payload(payload)
 
     def _showtraceback(self, etype, evalue, stb):
         # try to preserve ordering of tracebacks and print statements
         sys.stdout.flush()
         sys.stderr.flush()
 
         exc_content = {
             u'traceback' : stb,
             u'ename' : unicode_type(etype.__name__),
             u'evalue' : py3compat.safe_unicode(evalue),
         }
 
         dh = self.displayhook
         # Send exception info over pub socket for other clients than the caller
         # to pick up
         topic = None
         if dh.topic:
             topic = dh.topic.replace(b'execute_result', b'error')
 
         exc_msg = dh.session.send(dh.pub_socket, u'error', json_clean(exc_content), dh.parent_header, ident=topic)
 
         # FIXME - Hack: store exception info in shell object.  Right now, the
         # caller is reading this info after the fact, we need to fix this logic
         # to remove this hack.  Even uglier, we need to store the error status
         # here, because in the main loop, the logic that sets it is being
         # skipped because runlines swallows the exceptions.
         exc_content[u'status'] = u'error'
         self._reply_content = exc_content
         # /FIXME
 
         return exc_content
 
     def set_next_input(self, text, replace=False):
         """Send the specified text to the frontend to be presented at the next
         input cell."""
         payload = dict(
             source='set_next_input',
             text=text,
             replace=replace,
         )
         self.payload_manager.write_payload(payload)
 
     def set_parent(self, parent):
         """Set the parent header for associating output with its triggering input"""
         self.parent_header = parent
         self.displayhook.set_parent(parent)
         self.display_pub.set_parent(parent)
         self.data_pub.set_parent(parent)
         try:
             sys.stdout.set_parent(parent)
         except AttributeError:
             pass
         try:
             sys.stderr.set_parent(parent)
         except AttributeError:
             pass
 
     def get_parent(self):
         return self.parent_header
 
     #-------------------------------------------------------------------------
     # Things related to magics
     #-------------------------------------------------------------------------
 
     def init_magics(self):
         super(ZMQInteractiveShell, self).init_magics()
         self.register_magics(KernelMagics)
         self.magics_manager.register_alias('ed', 'edit')
 
 
 InteractiveShellABC.register(ZMQInteractiveShell)

ipython_parallel/client/magics.py

@@ -1,441 +1,436 b''
 # encoding: utf-8
 """
 =============
 parallelmagic
 =============
 
 Magic command interface for interactive parallel work.
 
 Usage
 =====
 
 ``%autopx``
 
 {AUTOPX_DOC}
 
 ``%px``
 
 {PX_DOC}
 
 ``%pxresult``
 
 {RESULT_DOC}
 
 ``%pxconfig``
 
 {CONFIG_DOC}
 
 """
 from __future__ import print_function
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008 The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import ast
 import re
 
 from IPython.core.error import UsageError
 from IPython.core.magic import Magics
 from IPython.core import magic_arguments
-from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.text import dedent
 
 #-----------------------------------------------------------------------------
 # Definitions of magic functions for use with IPython
 #-----------------------------------------------------------------------------
 
 
 NO_LAST_RESULT = "%pxresult recalls last %px result, which has not yet been used."
 
 def exec_args(f):
     """decorator for adding block/targets args for execution
     
     applied to %pxconfig and %%px
     """
     args = [
         magic_arguments.argument('-b', '--block', action="store_const",
             const=True, dest='block',
             help="use blocking (sync) execution",
         ),
         magic_arguments.argument('-a', '--noblock', action="store_const",
             const=False, dest='block',
             help="use non-blocking (async) execution",
         ),
         magic_arguments.argument('-t', '--targets', type=str,
             help="specify the targets on which to execute",
         ),
         magic_arguments.argument('--local', action="store_const",
             const=True, dest="local",
             help="also execute the cell in the local namespace",
         ),
         magic_arguments.argument('--verbose', action="store_const",
             const=True, dest="set_verbose",
             help="print a message at each execution",
         ),
         magic_arguments.argument('--no-verbose', action="store_const",
             const=False, dest="set_verbose",
             help="don't print any messages",
         ),
     ]
     for a in args:
         f = a(f)
     return f
 
 def output_args(f):
     """decorator for output-formatting args
     
     applied to %pxresult and %%px
     """
     args = [
         magic_arguments.argument('-r', action="store_const", dest='groupby',
             const='order',
             help="collate outputs in order (same as group-outputs=order)"
         ),
         magic_arguments.argument('-e', action="store_const", dest='groupby',
             const='engine',
             help="group outputs by engine (same as group-outputs=engine)"
         ),
         magic_arguments.argument('--group-outputs', dest='groupby', type=str,
             choices=['engine', 'order', 'type'], default='type',
             help="""Group the outputs in a particular way.
             
             Choices are:
             
             **type**: group outputs of all engines by type (stdout, stderr, displaypub, etc.).
             **engine**: display all output for each engine together.
             **order**: like type, but individual displaypub output from each engine is collated.
               For example, if multiple plots are generated by each engine, the first
               figure of each engine will be displayed, then the second of each, etc.
             """
         ),
         magic_arguments.argument('-o', '--out', dest='save_name', type=str,
             help="""store the AsyncResult object for this computation
                  in the global namespace under this name.
             """
         ),
     ]
     for a in args:
         f = a(f)
     return f
 
 class ParallelMagics(Magics):
     """A set of magics useful when controlling a parallel IPython cluster.
     """
     
     # magic-related
     magics = None
     registered = True
     
     # suffix for magics
     suffix = ''
     # A flag showing if autopx is activated or not
     _autopx = False
     # the current view used by the magics:
     view = None
     # last result cache for %pxresult
     last_result = None
     # verbose flag
     verbose = False
     
     def __init__(self, shell, view, suffix=''):
         self.view = view
         self.suffix = suffix
         
         # register magics
         self.magics = dict(cell={},line={})
         line_magics = self.magics['line']
         
         px = 'px' + suffix
         if not suffix:
             # keep %result for legacy compatibility
             line_magics['result'] = self.result
         
         line_magics['pxresult' + suffix] = self.result
         line_magics[px] = self.px
         line_magics['pxconfig' + suffix] = self.pxconfig
         line_magics['auto' + px] = self.autopx
         
         self.magics['cell'][px] = self.cell_px
         
         super(ParallelMagics, self).__init__(shell=shell)
     
     def _eval_target_str(self, ts):
         if ':' in ts:
             targets = eval("self.view.client.ids[%s]" % ts)
         elif 'all' in ts:
             targets = 'all'
         else:
             targets = eval(ts)
         return targets
     
     @magic_arguments.magic_arguments()
     @exec_args
     def pxconfig(self, line):
         """configure default targets/blocking for %px magics"""
         args = magic_arguments.parse_argstring(self.pxconfig, line)
         if args.targets:
             self.view.targets = self._eval_target_str(args.targets)
         if args.block is not None:
             self.view.block = args.block
         if args.set_verbose is not None:
             self.verbose = args.set_verbose
 
     @magic_arguments.magic_arguments()
     @output_args
-    @skip_doctest
     def result(self, line=''):
         """Print the result of the last asynchronous %px command.
         
         This lets you recall the results of %px computations after
         asynchronous submission (block=False).
 
         Examples
         --------
         ::
 
             In [23]: %px os.getpid()
             Async parallel execution on engine(s): all
 
             In [24]: %pxresult
             Out[8:10]: 60920
             Out[9:10]: 60921
             Out[10:10]: 60922
             Out[11:10]: 60923
         """
         args = magic_arguments.parse_argstring(self.result, line)
         
         if self.last_result is None:
             raise UsageError(NO_LAST_RESULT)
         
         self.last_result.get()
         self.last_result.display_outputs(groupby=args.groupby)
 
-    @skip_doctest
     def px(self, line=''):
         """Executes the given python command in parallel.
         
         Examples
         --------
         ::
 
             In [24]: %px a = os.getpid()
             Parallel execution on engine(s): all
             
             In [25]: %px print a
             [stdout:0] 1234
             [stdout:1] 1235
             [stdout:2] 1236
             [stdout:3] 1237
         """
         return self.parallel_execute(line)
         
     def parallel_execute(self, cell, block=None, groupby='type', save_name=None):
         """implementation used by %px and %%parallel"""
 
         # defaults:
         block = self.view.block if block is None else block
         
         base = "Parallel" if block else "Async parallel"
         
         targets = self.view.targets
         if isinstance(targets, list) and len(targets) > 10:
             str_targets = str(targets[:4])[:-1] + ', ..., ' + str(targets[-4:])[1:]
         else:
             str_targets = str(targets)
         if self.verbose:
             print(base + " execution on engine(s): %s" % str_targets)
         
         result = self.view.execute(cell, silent=False, block=False)
         self.last_result = result
         
         if save_name:
             self.shell.user_ns[save_name] = result
         
         if block:
             result.get()
             result.display_outputs(groupby)
         else:
             # return AsyncResult only on non-blocking submission
             return result
 
     @magic_arguments.magic_arguments()
     @exec_args
     @output_args
-    @skip_doctest
     def cell_px(self, line='', cell=None):
         """Executes the cell in parallel.
         
         Examples
         --------
         ::
 
             In [24]: %%px --noblock
                ....: a = os.getpid()
             Async parallel execution on engine(s): all
             
             In [25]: %%px
                ....: print a
             [stdout:0] 1234
             [stdout:1] 1235
             [stdout:2] 1236
             [stdout:3] 1237
         """
         
         args = magic_arguments.parse_argstring(self.cell_px, line)
         
         if args.targets:
             save_targets = self.view.targets
             self.view.targets = self._eval_target_str(args.targets)
         # if running local, don't block until after local has run
         block = False if args.local else args.block
         try:
             ar = self.parallel_execute(cell, block=block,
                                 groupby=args.groupby,
                                 save_name=args.save_name,
             )
         finally:
             if args.targets:
                 self.view.targets = save_targets
         
         # run locally after submitting remote
         block = self.view.block if args.block is None else args.block
         if args.local:
             self.shell.run_cell(cell)
             # now apply blocking behavor to remote execution
             if block:
                 ar.get()
                 ar.display_outputs(args.groupby)
         if not block:
             return ar
     
-    @skip_doctest
     def autopx(self, line=''):
         """Toggles auto parallel mode.
 
         Once this is called, all commands typed at the command line are send to
         the engines to be executed in parallel. To control which engine are
         used, the ``targets`` attribute of the view before
         entering ``%autopx`` mode.
 
 
         Then you can do the following::
 
             In [25]: %autopx
             %autopx to enabled
 
             In [26]: a = 10
             Parallel execution on engine(s): [0,1,2,3]
             In [27]: print a
             Parallel execution on engine(s): [0,1,2,3]
             [stdout:0] 10
             [stdout:1] 10
             [stdout:2] 10
             [stdout:3] 10
 
 
             In [27]: %autopx
             %autopx disabled
         """
         if self._autopx:
             self._disable_autopx()
         else:
             self._enable_autopx()
 
     def _enable_autopx(self):
         """Enable %autopx mode by saving the original run_cell and installing
         pxrun_cell.
         """
         # override run_cell
         self._original_run_cell = self.shell.run_cell
         self.shell.run_cell = self.pxrun_cell
 
         self._autopx = True
         print("%autopx enabled")
 
     def _disable_autopx(self):
         """Disable %autopx by restoring the original InteractiveShell.run_cell.
         """
         if self._autopx:
             self.shell.run_cell = self._original_run_cell
             self._autopx = False
             print("%autopx disabled")
 
     def pxrun_cell(self, raw_cell, store_history=False, silent=False):
         """drop-in replacement for InteractiveShell.run_cell.
 
         This executes code remotely, instead of in the local namespace.
 
         See InteractiveShell.run_cell for details.
         """
 
         if (not raw_cell) or raw_cell.isspace():
             return
 
         ipself = self.shell
 
         with ipself.builtin_trap:
             cell = ipself.prefilter_manager.prefilter_lines(raw_cell)
 
             # Store raw and processed history
             if store_history:
                 ipself.history_manager.store_inputs(ipself.execution_count,
                                                   cell, raw_cell)
 
             # ipself.logger.log(cell, raw_cell)
 
             cell_name = ipself.compile.cache(cell, ipself.execution_count)
 
             try:
                 ast.parse(cell, filename=cell_name)
             except (OverflowError, SyntaxError, ValueError, TypeError,
                     MemoryError):
                 # Case 1
                 ipself.showsyntaxerror()
                 ipself.execution_count += 1
                 return None
             except NameError:
                 # ignore name errors, because we don't know the remote keys
                 pass
 
         if store_history:
             # Write output to the database. Does nothing unless
             # history output logging is enabled.
             ipself.history_manager.store_output(ipself.execution_count)
             # Each cell is a *single* input, regardless of how many lines it has
             ipself.execution_count += 1
         if re.search(r'get_ipython\(\)\.magic\(u?["\']%?autopx', cell):
             self._disable_autopx()
             return False
         else:
             try:
                 result = self.view.execute(cell, silent=False, block=False)
             except:
                 ipself.showtraceback()
                 return True
             else:
                 if self.view.block:
                     try:
                         result.get()
                     except:
                         self.shell.showtraceback()
                         return True
                     else:
                         with ipself.builtin_trap:
                             result.display_outputs()
                 return False
 
 
 __doc__ = __doc__.format(
                 AUTOPX_DOC = dedent(ParallelMagics.autopx.__doc__),
                 PX_DOC = dedent(ParallelMagics.px.__doc__),
                 RESULT_DOC = dedent(ParallelMagics.result.__doc__),
                 CONFIG_DOC = dedent(ParallelMagics.pxconfig.__doc__),
 )

ipython_parallel/client/remotefunction.py

@@ -1,276 +1,273 b''
 """Remote Functions and decorators for Views."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import division
 
 import sys
 import warnings
 
 from decorator import decorator
-from IPython.testing.skipdoctest import skip_doctest
 
 from . import map as Map
 from .asyncresult import AsyncMapResult
 
 #-----------------------------------------------------------------------------
 # Functions and Decorators
 #-----------------------------------------------------------------------------
 
-@skip_doctest
 def remote(view, block=None, **flags):
     """Turn a function into a remote function.
 
     This method can be used for map:
 
     In [1]: @remote(view,block=True)
        ...: def func(a):
        ...:    pass
     """
 
     def remote_function(f):
         return RemoteFunction(view, f, block=block, **flags)
     return remote_function
 
-@skip_doctest
 def parallel(view, dist='b', block=None, ordered=True, **flags):
     """Turn a function into a parallel remote function.
 
     This method can be used for map:
 
     In [1]: @parallel(view, block=True)
        ...: def func(a):
        ...:    pass
     """
 
     def parallel_function(f):
         return ParallelFunction(view, f, dist=dist, block=block, ordered=ordered, **flags)
     return parallel_function
 
 def getname(f):
     """Get the name of an object.
     
     For use in case of callables that are not functions, and
     thus may not have __name__ defined.
     
     Order: f.__name__ >  f.name > str(f)
     """
     try:
         return f.__name__
     except:
         pass
     try:
         return f.name
     except:
         pass
     
     return str(f)
 
 @decorator
 def sync_view_results(f, self, *args, **kwargs):
     """sync relevant results from self.client to our results attribute.
     
     This is a clone of view.sync_results, but for remote functions
     """
     view = self.view
     if view._in_sync_results:
         return f(self, *args, **kwargs)
     view._in_sync_results = True
     try:
         ret = f(self, *args, **kwargs)
     finally:
         view._in_sync_results = False
         view._sync_results()
     return ret
     
 #--------------------------------------------------------------------------
 # Classes
 #--------------------------------------------------------------------------
 
 class RemoteFunction(object):
     """Turn an existing function into a remote function.
 
     Parameters
     ----------
 
     view : View instance
         The view to be used for execution
     f : callable
         The function to be wrapped into a remote function
     block : bool [default: None]
         Whether to wait for results or not.  The default behavior is
         to use the current `block` attribute of `view`
 
     **flags : remaining kwargs are passed to View.temp_flags
     """
 
     view = None # the remote connection
     func = None # the wrapped function
     block = None # whether to block
     flags = None # dict of extra kwargs for temp_flags
 
     def __init__(self, view, f, block=None, **flags):
         self.view = view
         self.func = f
         self.block=block
         self.flags=flags
 
     def __call__(self, *args, **kwargs):
         block = self.view.block if self.block is None else self.block
         with self.view.temp_flags(block=block, **self.flags):
             return self.view.apply(self.func, *args, **kwargs)
 
 
 class ParallelFunction(RemoteFunction):
     """Class for mapping a function to sequences.
 
     This will distribute the sequences according the a mapper, and call
     the function on each sub-sequence.  If called via map, then the function
     will be called once on each element, rather that each sub-sequence.
 
     Parameters
     ----------
 
     view : View instance
         The view to be used for execution
     f : callable
         The function to be wrapped into a remote function
     dist : str [default: 'b']
         The key for which mapObject to use to distribute sequences
         options are:
 
         * 'b' : use contiguous chunks in order
         * 'r' : use round-robin striping
 
     block : bool [default: None]
         Whether to wait for results or not.  The default behavior is
         to use the current `block` attribute of `view`
     chunksize : int or None
         The size of chunk to use when breaking up sequences in a load-balanced manner
     ordered : bool [default: True]
         Whether the result should be kept in order. If False,
         results become available as they arrive, regardless of submission order.
     **flags
         remaining kwargs are passed to View.temp_flags
     """
 
     chunksize = None
     ordered = None
     mapObject = None
     _mapping = False
 
     def __init__(self, view, f, dist='b', block=None, chunksize=None, ordered=True, **flags):
         super(ParallelFunction, self).__init__(view, f, block=block, **flags)
         self.chunksize = chunksize
         self.ordered = ordered
 
         mapClass = Map.dists[dist]
         self.mapObject = mapClass()
     
     @sync_view_results
     def __call__(self, *sequences):
         client = self.view.client
         
         lens = []
         maxlen = minlen = -1
         for i, seq in enumerate(sequences):
             try:
                 n = len(seq)
             except Exception:
                 seq = list(seq)
                 if isinstance(sequences, tuple):
                     # can't alter a tuple
                     sequences = list(sequences)
                 sequences[i] = seq
                 n = len(seq)
             if n > maxlen:
                 maxlen = n
             if minlen == -1 or n < minlen:
                 minlen = n
             lens.append(n)
         
         if maxlen == 0:
             # nothing to iterate over
             return []
         
         # check that the length of sequences match
         if not self._mapping and minlen != maxlen:
             msg = 'all sequences must have equal length, but have %s' % lens
             raise ValueError(msg)
         
         balanced = 'Balanced' in self.view.__class__.__name__
         if balanced:
             if self.chunksize:
                 nparts = maxlen // self.chunksize + int(maxlen % self.chunksize > 0)
             else:
                 nparts = maxlen
             targets = [None]*nparts
         else:
             if self.chunksize:
                 warnings.warn("`chunksize` is ignored unless load balancing", UserWarning)
             # multiplexed:
             targets = self.view.targets
             # 'all' is lazily evaluated at execution time, which is now:
             if targets == 'all':
                 targets = client._build_targets(targets)[1]
             elif isinstance(targets, int):
                 # single-engine view, targets must be iterable
                 targets = [targets]
             nparts = len(targets)
 
         msg_ids = []
         for index, t in enumerate(targets):
             args = []
             for seq in sequences:
                 part = self.mapObject.getPartition(seq, index, nparts, maxlen)
                 args.append(part)
 
             if sum([len(arg) for arg in args]) == 0:
                 continue
 
             if self._mapping:
                 if sys.version_info[0] >= 3:
                     f = lambda f, *sequences: list(map(f, *sequences))
                 else:
                     f = map
                 args = [self.func] + args
             else:
                 f=self.func
 
             view = self.view if balanced else client[t]
             with view.temp_flags(block=False, **self.flags):
                 ar = view.apply(f, *args)
 
             msg_ids.extend(ar.msg_ids)
 
         r = AsyncMapResult(self.view.client, msg_ids, self.mapObject,
                             fname=getname(self.func),
                             ordered=self.ordered
                         )
 
         if self.block:
             try:
                 return r.get()
             except KeyboardInterrupt:
                 return r
         else:
             return r
 
     def map(self, *sequences):
         """call a function on each element of one or more sequence(s) remotely.
         This should behave very much like the builtin map, but return an AsyncMapResult
         if self.block is False.
         
         That means it can take generators (will be cast to lists locally),
         and mismatched sequence lengths will be padded with None.
         """
         # set _mapping as a flag for use inside self.__call__
         self._mapping = True
         try:
             ret = self(*sequences)
         finally:
             self._mapping = False
         return ret
 
 __all__ = ['remote', 'parallel', 'RemoteFunction', 'ParallelFunction']

ipython_parallel/client/view.py

@@ -1,1125 +1,1121 b''
 """Views of remote engines."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 
 import imp
 import sys
 import warnings
 from contextlib import contextmanager
 from types import ModuleType
 
 import zmq
 
-from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import pickleutil
 from IPython.utils.traitlets import (
     HasTraits, Any, Bool, List, Dict, Set, Instance, CFloat, Integer
 )
 from decorator import decorator
 
 from ipython_parallel import util
 from ipython_parallel.controller.dependency import Dependency, dependent
 from IPython.utils.py3compat import string_types, iteritems, PY3
 
 from . import map as Map
 from .asyncresult import AsyncResult, AsyncMapResult
 from .remotefunction import ParallelFunction, parallel, remote, getname
 
 #-----------------------------------------------------------------------------
 # Decorators
 #-----------------------------------------------------------------------------
 
 @decorator
 def save_ids(f, self, *args, **kwargs):
     """Keep our history and outstanding attributes up to date after a method call."""
     n_previous = len(self.client.history)
     try:
         ret = f(self, *args, **kwargs)
     finally:
         nmsgs = len(self.client.history) - n_previous
         msg_ids = self.client.history[-nmsgs:]
         self.history.extend(msg_ids)
         self.outstanding.update(msg_ids)
     return ret
 
 @decorator
 def sync_results(f, self, *args, **kwargs):
     """sync relevant results from self.client to our results attribute."""
     if self._in_sync_results:
         return f(self, *args, **kwargs)
     self._in_sync_results = True
     try:
         ret = f(self, *args, **kwargs)
     finally:
         self._in_sync_results = False
         self._sync_results()
     return ret
 
 @decorator
 def spin_after(f, self, *args, **kwargs):
     """call spin after the method."""
     ret = f(self, *args, **kwargs)
     self.spin()
     return ret
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
-@skip_doctest
 class View(HasTraits):
     """Base View class for more convenint apply(f,*args,**kwargs) syntax via attributes.
 
     Don't use this class, use subclasses.
 
     Methods
     -------
 
     spin
         flushes incoming results and registration state changes
         control methods spin, and requesting `ids` also ensures up to date
 
     wait
         wait on one or more msg_ids
 
     execution methods
         apply
         legacy: execute, run
 
     data movement
         push, pull, scatter, gather
 
     query methods
         get_result, queue_status, purge_results, result_status
 
     control methods
         abort, shutdown
 
     """
     # flags
     block=Bool(False)
     track=Bool(True)
     targets = Any()
 
     history=List()
     outstanding = Set()
     results = Dict()
     client = Instance('ipython_parallel.Client', allow_none=True)
 
     _socket = Instance('zmq.Socket', allow_none=True)
     _flag_names = List(['targets', 'block', 'track'])
     _in_sync_results = Bool(False)
     _targets = Any()
     _idents = Any()
 
     def __init__(self, client=None, socket=None, **flags):
         super(View, self).__init__(client=client, _socket=socket)
         self.results = client.results
         self.block = client.block
 
         self.set_flags(**flags)
 
         assert not self.__class__ is View, "Don't use base View objects, use subclasses"
 
     def __repr__(self):
         strtargets = str(self.targets)
         if len(strtargets) > 16:
             strtargets = strtargets[:12]+'...]'
         return "<%s %s>"%(self.__class__.__name__, strtargets)
     
     def __len__(self):
         if isinstance(self.targets, list):
             return len(self.targets)
         elif isinstance(self.targets, int):
             return 1
         else:
             return len(self.client)
     
     def set_flags(self, **kwargs):
         """set my attribute flags by keyword.
 
         Views determine behavior with a few attributes (`block`, `track`, etc.).
         These attributes can be set all at once by name with this method.
 
         Parameters
         ----------
 
         block : bool
             whether to wait for results
         track : bool
             whether to create a MessageTracker to allow the user to
             safely edit after arrays and buffers during non-copying
             sends.
         """
         for name, value in iteritems(kwargs):
             if name not in self._flag_names:
                 raise KeyError("Invalid name: %r"%name)
             else:
                 setattr(self, name, value)
 
     @contextmanager
     def temp_flags(self, **kwargs):
         """temporarily set flags, for use in `with` statements.
 
         See set_flags for permanent setting of flags
 
         Examples
         --------
 
         >>> view.track=False
         ...
         >>> with view.temp_flags(track=True):
         ...    ar = view.apply(dostuff, my_big_array)
         ...    ar.tracker.wait() # wait for send to finish
         >>> view.track
         False
 
         """
         # preflight: save flags, and set temporaries
         saved_flags = {}
         for f in self._flag_names:
             saved_flags[f] = getattr(self, f)
         self.set_flags(**kwargs)
         # yield to the with-statement block
         try:
             yield
         finally:
             # postflight: restore saved flags
             self.set_flags(**saved_flags)
 
 
     #----------------------------------------------------------------
     # apply
     #----------------------------------------------------------------
 
     def _sync_results(self):
         """to be called by @sync_results decorator
         
         after submitting any tasks.
         """
         delta = self.outstanding.difference(self.client.outstanding)
         completed = self.outstanding.intersection(delta)
         self.outstanding = self.outstanding.difference(completed)
         
     @sync_results
     @save_ids
     def _really_apply(self, f, args, kwargs, block=None, **options):
         """wrapper for client.send_apply_request"""
         raise NotImplementedError("Implement in subclasses")
 
     def apply(self, f, *args, **kwargs):
         """calls ``f(*args, **kwargs)`` on remote engines, returning the result.
 
         This method sets all apply flags via this View's attributes.
 
         Returns :class:`~IPython.parallel.client.asyncresult.AsyncResult`
         instance if ``self.block`` is False, otherwise the return value of
         ``f(*args, **kwargs)``.
         """
         return self._really_apply(f, args, kwargs)
 
     def apply_async(self, f, *args, **kwargs):
         """calls ``f(*args, **kwargs)`` on remote engines in a nonblocking manner.
 
         Returns :class:`~IPython.parallel.client.asyncresult.AsyncResult` instance.
         """
         return self._really_apply(f, args, kwargs, block=False)
 
     @spin_after
     def apply_sync(self, f, *args, **kwargs):
         """calls ``f(*args, **kwargs)`` on remote engines in a blocking manner,
          returning the result.
         """
         return self._really_apply(f, args, kwargs, block=True)
 
     #----------------------------------------------------------------
     # wrappers for client and control methods
     #----------------------------------------------------------------
     @sync_results
     def spin(self):
         """spin the client, and sync"""
         self.client.spin()
 
     @sync_results
     def wait(self, jobs=None, timeout=-1):
         """waits on one or more `jobs`, for up to `timeout` seconds.
 
         Parameters
         ----------
 
         jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
                 ints are indices to self.history
                 strs are msg_ids
                 default: wait on all outstanding messages
         timeout : float
                 a time in seconds, after which to give up.
                 default is -1, which means no timeout
 
         Returns
         -------
 
         True : when all msg_ids are done
         False : timeout reached, some msg_ids still outstanding
         """
         if jobs is None:
             jobs = self.history
         return self.client.wait(jobs, timeout)
 
     def abort(self, jobs=None, targets=None, block=None):
         """Abort jobs on my engines.
 
         Parameters
         ----------
 
         jobs : None, str, list of strs, optional
             if None: abort all jobs.
             else: abort specific msg_id(s).
         """
         block = block if block is not None else self.block
         targets = targets if targets is not None else self.targets
         jobs = jobs if jobs is not None else list(self.outstanding)
         
         return self.client.abort(jobs=jobs, targets=targets, block=block)
 
     def queue_status(self, targets=None, verbose=False):
         """Fetch the Queue status of my engines"""
         targets = targets if targets is not None else self.targets
         return self.client.queue_status(targets=targets, verbose=verbose)
 
     def purge_results(self, jobs=[], targets=[]):
         """Instruct the controller to forget specific results."""
         if targets is None or targets == 'all':
             targets = self.targets
         return self.client.purge_results(jobs=jobs, targets=targets)
 
     def shutdown(self, targets=None, restart=False, hub=False, block=None):
         """Terminates one or more engine processes, optionally including the hub.
         """
         block = self.block if block is None else block
         if targets is None or targets == 'all':
             targets = self.targets
         return self.client.shutdown(targets=targets, restart=restart, hub=hub, block=block)
 
     @spin_after
     def get_result(self, indices_or_msg_ids=None, block=None, owner=True):
         """return one or more results, specified by history index or msg_id.
 
         See :meth:`IPython.parallel.client.client.Client.get_result` for details.
         """
 
         if indices_or_msg_ids is None:
             indices_or_msg_ids = -1
         if isinstance(indices_or_msg_ids, int):
             indices_or_msg_ids = self.history[indices_or_msg_ids]
         elif isinstance(indices_or_msg_ids, (list,tuple,set)):
             indices_or_msg_ids = list(indices_or_msg_ids)
             for i,index in enumerate(indices_or_msg_ids):
                 if isinstance(index, int):
                     indices_or_msg_ids[i] = self.history[index]
         return self.client.get_result(indices_or_msg_ids, block=block, owner=owner)
 
     #-------------------------------------------------------------------
     # Map
     #-------------------------------------------------------------------
 
     @sync_results
     def map(self, f, *sequences, **kwargs):
         """override in subclasses"""
         raise NotImplementedError
 
     def map_async(self, f, *sequences, **kwargs):
         """Parallel version of builtin :func:`python:map`, using this view's engines.
 
         This is equivalent to ``map(...block=False)``.
 
         See `self.map` for details.
         """
         if 'block' in kwargs:
             raise TypeError("map_async doesn't take a `block` keyword argument.")
         kwargs['block'] = False
         return self.map(f,*sequences,**kwargs)
 
     def map_sync(self, f, *sequences, **kwargs):
         """Parallel version of builtin :func:`python:map`, using this view's engines.
 
         This is equivalent to ``map(...block=True)``.
 
         See `self.map` for details.
         """
         if 'block' in kwargs:
             raise TypeError("map_sync doesn't take a `block` keyword argument.")
         kwargs['block'] = True
         return self.map(f,*sequences,**kwargs)
 
     def imap(self, f, *sequences, **kwargs):
         """Parallel version of :func:`itertools.imap`.
 
         See `self.map` for details.
 
         """
 
         return iter(self.map_async(f,*sequences, **kwargs))
 
     #-------------------------------------------------------------------
     # Decorators
     #-------------------------------------------------------------------
 
     def remote(self, block=None, **flags):
         """Decorator for making a RemoteFunction"""
         block = self.block if block is None else block
         return remote(self, block=block, **flags)
 
     def parallel(self, dist='b', block=None, **flags):
         """Decorator for making a ParallelFunction"""
         block = self.block if block is None else block
         return parallel(self, dist=dist, block=block, **flags)
 
-@skip_doctest
 class DirectView(View):
     """Direct Multiplexer View of one or more engines.
 
     These are created via indexed access to a client:
 
     >>> dv_1 = client[1]
     >>> dv_all = client[:]
     >>> dv_even = client[::2]
     >>> dv_some = client[1:3]
 
     This object provides dictionary access to engine namespaces:
 
     # push a=5:
     >>> dv['a'] = 5
     # pull 'foo':
     >>> dv['foo']
 
     """
 
     def __init__(self, client=None, socket=None, targets=None):
         super(DirectView, self).__init__(client=client, socket=socket, targets=targets)
 
     @property
     def importer(self):
         """sync_imports(local=True) as a property.
 
         See sync_imports for details.
 
         """
         return self.sync_imports(True)
 
     @contextmanager
     def sync_imports(self, local=True, quiet=False):
         """Context Manager for performing simultaneous local and remote imports.
 
         'import x as y' will *not* work.  The 'as y' part will simply be ignored.
         
         If `local=True`, then the package will also be imported locally.
         
         If `quiet=True`, no output will be produced when attempting remote 
         imports. 
         
         Note that remote-only (`local=False`) imports have not been implemented.
         
         >>> with view.sync_imports():
         ...    from numpy import recarray
         importing recarray from numpy on engine(s)
 
         """
         from IPython.utils.py3compat import builtin_mod
         local_import = builtin_mod.__import__
         modules = set()
         results = []
         @util.interactive
         def remote_import(name, fromlist, level):
             """the function to be passed to apply, that actually performs the import
             on the engine, and loads up the user namespace.
             """
             import sys
             user_ns = globals()
             mod = __import__(name, fromlist=fromlist, level=level)
             if fromlist:
                 for key in fromlist:
                     user_ns[key] = getattr(mod, key)
             else:
                 user_ns[name] = sys.modules[name]
 
         def view_import(name, globals={}, locals={}, fromlist=[], level=0):
             """the drop-in replacement for __import__, that optionally imports
             locally as well.
             """
             # don't override nested imports
             save_import = builtin_mod.__import__
             builtin_mod.__import__ = local_import
 
             if imp.lock_held():
                 # this is a side-effect import, don't do it remotely, or even
                 # ignore the local effects
                 return local_import(name, globals, locals, fromlist, level)
 
             imp.acquire_lock()
             if local:
                 mod = local_import(name, globals, locals, fromlist, level)
             else:
                 raise NotImplementedError("remote-only imports not yet implemented")
             imp.release_lock()
 
             key = name+':'+','.join(fromlist or [])
             if level <= 0 and key not in modules:
                 modules.add(key)
                 if not quiet:
                     if fromlist:
                         print("importing %s from %s on engine(s)"%(','.join(fromlist), name))
                     else:
                         print("importing %s on engine(s)"%name)
                 results.append(self.apply_async(remote_import, name, fromlist, level))
             # restore override
             builtin_mod.__import__ = save_import
 
             return mod
 
         # override __import__
         builtin_mod.__import__ = view_import
         try:
             # enter the block
             yield
         except ImportError:
             if local:
                 raise
             else:
                 # ignore import errors if not doing local imports
                 pass
         finally:
             # always restore __import__
             builtin_mod.__import__ = local_import
 
         for r in results:
             # raise possible remote ImportErrors here
             r.get()
     
     def use_dill(self):
         """Expand serialization support with dill
         
         adds support for closures, etc.
         
         This calls ipython_kernel.pickleutil.use_dill() here and on each engine.
         """
         pickleutil.use_dill()
         return self.apply(pickleutil.use_dill)
 
     def use_cloudpickle(self):
         """Expand serialization support with cloudpickle.
         """
         pickleutil.use_cloudpickle()
         return self.apply(pickleutil.use_cloudpickle)
 
 
     @sync_results
     @save_ids
     def _really_apply(self, f, args=None, kwargs=None, targets=None, block=None, track=None):
         """calls f(*args, **kwargs) on remote engines, returning the result.
 
         This method sets all of `apply`'s flags via this View's attributes.
 
         Parameters
         ----------
 
         f : callable
 
         args : list [default: empty]
 
         kwargs : dict [default: empty]
 
         targets : target list [default: self.targets]
             where to run
         block : bool [default: self.block]
             whether to block
         track : bool [default: self.track]
             whether to ask zmq to track the message, for safe non-copying sends
 
         Returns
         -------
 
         if self.block is False:
             returns AsyncResult
         else:
             returns actual result of f(*args, **kwargs) on the engine(s)
             This will be a list of self.targets is also a list (even length 1), or
             the single result if self.targets is an integer engine id
         """
         args = [] if args is None else args
         kwargs = {} if kwargs is None else kwargs
         block = self.block if block is None else block
         track = self.track if track is None else track
         targets = self.targets if targets is None else targets
         
         _idents, _targets = self.client._build_targets(targets)
         msg_ids = []
         trackers = []
         for ident in _idents:
             msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track,
                                     ident=ident)
             if track:
                 trackers.append(msg['tracker'])
             msg_ids.append(msg['header']['msg_id'])
         if isinstance(targets, int):
             msg_ids = msg_ids[0]
         tracker = None if track is False else zmq.MessageTracker(*trackers)
         ar = AsyncResult(self.client, msg_ids, fname=getname(f), targets=_targets,
             tracker=tracker, owner=True,
         )
         if block:
             try:
                 return ar.get()
             except KeyboardInterrupt:
                 pass
         return ar
 
 
     @sync_results
     def map(self, f, *sequences, **kwargs):
         """``view.map(f, *sequences, block=self.block)`` => list|AsyncMapResult
 
         Parallel version of builtin `map`, using this View's `targets`.
 
         There will be one task per target, so work will be chunked
         if the sequences are longer than `targets`.
 
         Results can be iterated as they are ready, but will become available in chunks.
 
         Parameters
         ----------
 
         f : callable
             function to be mapped
         *sequences: one or more sequences of matching length
             the sequences to be distributed and passed to `f`
         block : bool
             whether to wait for the result or not [default self.block]
 
         Returns
         -------
 
 
         If block=False
           An :class:`~ipython_parallel.client.asyncresult.AsyncMapResult` instance.
           An object like AsyncResult, but which reassembles the sequence of results
           into a single list. AsyncMapResults can be iterated through before all
           results are complete.
         else
             A list, the result of ``map(f,*sequences)``
         """
 
         block = kwargs.pop('block', self.block)
         for k in kwargs.keys():
             if k not in ['block', 'track']:
                 raise TypeError("invalid keyword arg, %r"%k)
 
         assert len(sequences) > 0, "must have some sequences to map onto!"
         pf = ParallelFunction(self, f, block=block, **kwargs)
         return pf.map(*sequences)
 
     @sync_results
     @save_ids
     def execute(self, code, silent=True, targets=None, block=None):
         """Executes `code` on `targets` in blocking or nonblocking manner.
 
         ``execute`` is always `bound` (affects engine namespace)
 
         Parameters
         ----------
 
         code : str
                 the code string to be executed
         block : bool
                 whether or not to wait until done to return
                 default: self.block
         """
         block = self.block if block is None else block
         targets = self.targets if targets is None else targets
 
         _idents, _targets = self.client._build_targets(targets)
         msg_ids = []
         trackers = []
         for ident in _idents:
             msg = self.client.send_execute_request(self._socket, code, silent=silent, ident=ident)
             msg_ids.append(msg['header']['msg_id'])
         if isinstance(targets, int):
             msg_ids = msg_ids[0]
         ar = AsyncResult(self.client, msg_ids, fname='execute', targets=_targets, owner=True)
         if block:
             try:
                 ar.get()
             except KeyboardInterrupt:
                 pass
         return ar
 
     def run(self, filename, targets=None, block=None):
         """Execute contents of `filename` on my engine(s).
 
         This simply reads the contents of the file and calls `execute`.
 
         Parameters
         ----------
 
         filename : str
                 The path to the file
         targets : int/str/list of ints/strs
                 the engines on which to execute
                 default : all
         block : bool
                 whether or not to wait until done
                 default: self.block
 
         """
         with open(filename, 'r') as f:
             # add newline in case of trailing indented whitespace
             # which will cause SyntaxError
             code = f.read()+'\n'
         return self.execute(code, block=block, targets=targets)
 
     def update(self, ns):
         """update remote namespace with dict `ns`
 
         See `push` for details.
         """
         return self.push(ns, block=self.block, track=self.track)
 
     def push(self, ns, targets=None, block=None, track=None):
         """update remote namespace with dict `ns`
 
         Parameters
         ----------
 
         ns : dict
             dict of keys with which to update engine namespace(s)
         block : bool [default : self.block]
             whether to wait to be notified of engine receipt
 
         """
 
         block = block if block is not None else self.block
         track = track if track is not None else self.track
         targets = targets if targets is not None else self.targets
         # applier = self.apply_sync if block else self.apply_async
         if not isinstance(ns, dict):
             raise TypeError("Must be a dict, not %s"%type(ns))
         return self._really_apply(util._push, kwargs=ns, block=block, track=track, targets=targets)
 
     def get(self, key_s):
         """get object(s) by `key_s` from remote namespace
 
         see `pull` for details.
         """
         # block = block if block is not None else self.block
         return self.pull(key_s, block=True)
 
     def pull(self, names, targets=None, block=None):
         """get object(s) by `name` from remote namespace
 
         will return one object if it is a key.
         can also take a list of keys, in which case it will return a list of objects.
         """
         block = block if block is not None else self.block
         targets = targets if targets is not None else self.targets
         applier = self.apply_sync if block else self.apply_async
         if isinstance(names, string_types):
             pass
         elif isinstance(names, (list,tuple,set)):
             for key in names:
                 if not isinstance(key, string_types):
                     raise TypeError("keys must be str, not type %r"%type(key))
         else:
             raise TypeError("names must be strs, not %r"%names)
         return self._really_apply(util._pull, (names,), block=block, targets=targets)
 
     def scatter(self, key, seq, dist='b', flatten=False, targets=None, block=None, track=None):
         """
         Partition a Python sequence and send the partitions to a set of engines.
         """
         block = block if block is not None else self.block
         track = track if track is not None else self.track
         targets = targets if targets is not None else self.targets
         
         # construct integer ID list:
         targets = self.client._build_targets(targets)[1]
 
         mapObject = Map.dists[dist]()
         nparts = len(targets)
         msg_ids = []
         trackers = []
         for index, engineid in enumerate(targets):
             partition = mapObject.getPartition(seq, index, nparts)
             if flatten and len(partition) == 1:
                 ns = {key: partition[0]}
             else:
                 ns = {key: partition}
             r = self.push(ns, block=False, track=track, targets=engineid)
             msg_ids.extend(r.msg_ids)
             if track:
                 trackers.append(r._tracker)
 
         if track:
             tracker = zmq.MessageTracker(*trackers)
         else:
             tracker = None
 
         r = AsyncResult(self.client, msg_ids, fname='scatter', targets=targets,
             tracker=tracker, owner=True,
         )
         if block:
             r.wait()
         else:
             return r
 
     @sync_results
     @save_ids
     def gather(self, key, dist='b', targets=None, block=None):
         """
         Gather a partitioned sequence on a set of engines as a single local seq.
         """
         block = block if block is not None else self.block
         targets = targets if targets is not None else self.targets
         mapObject = Map.dists[dist]()
         msg_ids = []
 
         # construct integer ID list:
         targets = self.client._build_targets(targets)[1]
         
         for index, engineid in enumerate(targets):
             msg_ids.extend(self.pull(key, block=False, targets=engineid).msg_ids)
 
         r = AsyncMapResult(self.client, msg_ids, mapObject, fname='gather')
 
         if block:
             try:
                 return r.get()
             except KeyboardInterrupt:
                 pass
         return r
 
     def __getitem__(self, key):
         return self.get(key)
 
     def __setitem__(self,key, value):
         self.update({key:value})
 
     def clear(self, targets=None, block=None):
         """Clear the remote namespaces on my engines."""
         block = block if block is not None else self.block
         targets = targets if targets is not None else self.targets
         return self.client.clear(targets=targets, block=block)
 
     #----------------------------------------
     # activate for %px, %autopx, etc. magics
     #----------------------------------------
 
     def activate(self, suffix=''):
         """Activate IPython magics associated with this View
         
         Defines the magics `%px, %autopx, %pxresult, %%px, %pxconfig`
         
         Parameters
         ----------
         
         suffix: str [default: '']
             The suffix, if any, for the magics.  This allows you to have
             multiple views associated with parallel magics at the same time.
             
             e.g. ``rc[::2].activate(suffix='_even')`` will give you
             the magics ``%px_even``, ``%pxresult_even``, etc. for running magics 
             on the even engines.
         """
         
         from IPython.parallel.client.magics import ParallelMagics
         
         try:
             # This is injected into __builtins__.
             ip = get_ipython()
         except NameError:
             print("The IPython parallel magics (%px, etc.) only work within IPython.")
             return
         
         M = ParallelMagics(ip, self, suffix)
         ip.magics_manager.register(M)
 
 
-@skip_doctest
 class LoadBalancedView(View):
     """An load-balancing View that only executes via the Task scheduler.
 
     Load-balanced views can be created with the client's `view` method:
 
     >>> v = client.load_balanced_view()
 
     or targets can be specified, to restrict the potential destinations:
 
     >>> v = client.load_balanced_view([1,3])
 
     which would restrict loadbalancing to between engines 1 and 3.
 
     """
 
     follow=Any()
     after=Any()
     timeout=CFloat()
     retries = Integer(0)
 
     _task_scheme = Any()
     _flag_names = List(['targets', 'block', 'track', 'follow', 'after', 'timeout', 'retries'])
 
     def __init__(self, client=None, socket=None, **flags):
         super(LoadBalancedView, self).__init__(client=client, socket=socket, **flags)
         self._task_scheme=client._task_scheme
 
     def _validate_dependency(self, dep):
         """validate a dependency.
 
         For use in `set_flags`.
         """
         if dep is None or isinstance(dep, string_types + (AsyncResult, Dependency)):
             return True
         elif isinstance(dep, (list,set, tuple)):
             for d in dep:
                 if not isinstance(d, string_types + (AsyncResult,)):
                     return False
         elif isinstance(dep, dict):
             if set(dep.keys()) != set(Dependency().as_dict().keys()):
                 return False
             if not isinstance(dep['msg_ids'], list):
                 return False
             for d in dep['msg_ids']:
                 if not isinstance(d, string_types):
                     return False
         else:
             return False
 
         return True
 
     def _render_dependency(self, dep):
         """helper for building jsonable dependencies from various input forms."""
         if isinstance(dep, Dependency):
             return dep.as_dict()
         elif isinstance(dep, AsyncResult):
             return dep.msg_ids
         elif dep is None:
             return []
         else:
             # pass to Dependency constructor
             return list(Dependency(dep))
 
     def set_flags(self, **kwargs):
         """set my attribute flags by keyword.
 
         A View is a wrapper for the Client's apply method, but with attributes
         that specify keyword arguments, those attributes can be set by keyword
         argument with this method.
 
         Parameters
         ----------
 
         block : bool
             whether to wait for results
         track : bool
             whether to create a MessageTracker to allow the user to
             safely edit after arrays and buffers during non-copying
             sends.
 
         after : Dependency or collection of msg_ids
             Only for load-balanced execution (targets=None)
             Specify a list of msg_ids as a time-based dependency.
             This job will only be run *after* the dependencies
             have been met.
 
         follow : Dependency or collection of msg_ids
             Only for load-balanced execution (targets=None)
             Specify a list of msg_ids as a location-based dependency.
             This job will only be run on an engine where this dependency
             is met.
 
         timeout : float/int or None
             Only for load-balanced execution (targets=None)
             Specify an amount of time (in seconds) for the scheduler to
             wait for dependencies to be met before failing with a
             DependencyTimeout.
 
         retries : int
             Number of times a task will be retried on failure.
         """
 
         super(LoadBalancedView, self).set_flags(**kwargs)
         for name in ('follow', 'after'):
             if name in kwargs:
                 value = kwargs[name]
                 if self._validate_dependency(value):
                     setattr(self, name, value)
                 else:
                     raise ValueError("Invalid dependency: %r"%value)
         if 'timeout' in kwargs:
             t = kwargs['timeout']
             if not isinstance(t, (int, float, type(None))):
                 if (not PY3) and (not isinstance(t, long)):
                     raise TypeError("Invalid type for timeout: %r"%type(t))
             if t is not None:
                 if t < 0:
                     raise ValueError("Invalid timeout: %s"%t)
             self.timeout = t
 
     @sync_results
     @save_ids
     def _really_apply(self, f, args=None, kwargs=None, block=None, track=None,
                                         after=None, follow=None, timeout=None,
                                         targets=None, retries=None):
         """calls f(*args, **kwargs) on a remote engine, returning the result.
 
         This method temporarily sets all of `apply`'s flags for a single call.
 
         Parameters
         ----------
 
         f : callable
 
         args : list [default: empty]
 
         kwargs : dict [default: empty]
 
         block : bool [default: self.block]
             whether to block
         track : bool [default: self.track]
             whether to ask zmq to track the message, for safe non-copying sends
 
         !!!!!! TODO: THE REST HERE  !!!!
 
         Returns
         -------
 
         if self.block is False:
             returns AsyncResult
         else:
             returns actual result of f(*args, **kwargs) on the engine(s)
             This will be a list of self.targets is also a list (even length 1), or
             the single result if self.targets is an integer engine id
         """
 
         # validate whether we can run
         if self._socket.closed:
             msg = "Task farming is disabled"
             if self._task_scheme == 'pure':
                 msg += " because the pure ZMQ scheduler cannot handle"
                 msg += " disappearing engines."
             raise RuntimeError(msg)
 
         if self._task_scheme == 'pure':
             # pure zmq scheme doesn't support extra features
             msg = "Pure ZMQ scheduler doesn't support the following flags:"
             "follow, after, retries, targets, timeout"
             if (follow or after or retries or targets or timeout):
                 # hard fail on Scheduler flags
                 raise RuntimeError(msg)
             if isinstance(f, dependent):
                 # soft warn on functional dependencies
                 warnings.warn(msg, RuntimeWarning)
 
         # build args
         args = [] if args is None else args
         kwargs = {} if kwargs is None else kwargs
         block = self.block if block is None else block
         track = self.track if track is None else track
         after = self.after if after is None else after
         retries = self.retries if retries is None else retries
         follow = self.follow if follow is None else follow
         timeout = self.timeout if timeout is None else timeout
         targets = self.targets if targets is None else targets
 
         if not isinstance(retries, int):
             raise TypeError('retries must be int, not %r'%type(retries))
 
         if targets is None:
             idents = []
         else:
             idents = self.client._build_targets(targets)[0]
             # ensure *not* bytes
             idents = [ ident.decode() for ident in idents ]
 
         after = self._render_dependency(after)
         follow = self._render_dependency(follow)
         metadata = dict(after=after, follow=follow, timeout=timeout, targets=idents, retries=retries)
 
         msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track,
                                 metadata=metadata)
         tracker = None if track is False else msg['tracker']
 
         ar = AsyncResult(self.client, msg['header']['msg_id'], fname=getname(f),
             targets=None, tracker=tracker, owner=True,
         )
         if block:
             try:
                 return ar.get()
             except KeyboardInterrupt:
                 pass
         return ar
 
     @sync_results
     @save_ids
     def map(self, f, *sequences, **kwargs):
         """``view.map(f, *sequences, block=self.block, chunksize=1, ordered=True)`` => list|AsyncMapResult
 
         Parallel version of builtin `map`, load-balanced by this View.
 
         `block`, and `chunksize` can be specified by keyword only.
 
         Each `chunksize` elements will be a separate task, and will be
         load-balanced. This lets individual elements be available for iteration
         as soon as they arrive.
 
         Parameters
         ----------
 
         f : callable
             function to be mapped
         *sequences: one or more sequences of matching length
             the sequences to be distributed and passed to `f`
         block : bool [default self.block]
             whether to wait for the result or not
         track : bool
             whether to create a MessageTracker to allow the user to
             safely edit after arrays and buffers during non-copying
             sends.
         chunksize : int [default 1]
             how many elements should be in each task.
         ordered : bool [default True]
             Whether the results should be gathered as they arrive, or enforce
             the order of submission.
             
             Only applies when iterating through AsyncMapResult as results arrive.
             Has no effect when block=True.
 
         Returns
         -------
 
         if block=False
           An :class:`~ipython_parallel.client.asyncresult.AsyncMapResult` instance.
           An object like AsyncResult, but which reassembles the sequence of results
           into a single list. AsyncMapResults can be iterated through before all
           results are complete.
         else
             A list, the result of ``map(f,*sequences)``
         """
 
         # default
         block = kwargs.get('block', self.block)
         chunksize = kwargs.get('chunksize', 1)
         ordered = kwargs.get('ordered', True)
 
         keyset = set(kwargs.keys())
         extra_keys = keyset.difference_update(set(['block', 'chunksize']))
         if extra_keys:
             raise TypeError("Invalid kwargs: %s"%list(extra_keys))
 
         assert len(sequences) > 0, "must have some sequences to map onto!"
 
         pf = ParallelFunction(self, f, block=block, chunksize=chunksize, ordered=ordered)
         return pf.map(*sequences)
 
 __all__ = ['LoadBalancedView', 'DirectView']

traitlets/traitlets.py

@@ -1,1878 +1,1875 b''
 # encoding: utf-8
 """
 A lightweight Traits like module.
 
 This is designed to provide a lightweight, simple, pure Python version of
 many of the capabilities of enthought.traits.  This includes:
 
 * Validation
 * Type specification with defaults
 * Static and dynamic notification
 * Basic predefined types
 * An API that is similar to enthought.traits
 
 We don't support:
 
 * Delegation
 * Automatic GUI generation
 * A full set of trait types.  Most importantly, we don't provide container
   traits (list, dict, tuple) that can trigger notifications if their
   contents change.
 * API compatibility with enthought.traits
 
 There are also some important difference in our design:
 
 * enthought.traits does not validate default values.  We do.
 
 We choose to create this module because we need these capabilities, but
 we need them to be pure Python so they work in all Python implementations,
 including Jython and IronPython.
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.utils.traitlets
    :parts: 3
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 #
 # Adapted from enthought.traits, Copyright (c) Enthought, Inc.,
 # also under the terms of the Modified BSD License.
 
 import contextlib
 import inspect
 import re
 import sys
 import types
 from types import FunctionType
 try:
     from types import ClassType, InstanceType
     ClassTypes = (ClassType, type)
 except:
     ClassTypes = (type,)
 from warnings import warn
 
 from IPython.utils import py3compat
 from IPython.utils import eventful
 from IPython.utils.getargspec import getargspec
 from IPython.utils.importstring import import_item
 from IPython.utils.py3compat import iteritems, string_types
-from IPython.testing.skipdoctest import skip_doctest
 
 from .sentinel import Sentinel
 SequenceTypes = (list, tuple, set, frozenset)
 
 #-----------------------------------------------------------------------------
 # Basic classes
 #-----------------------------------------------------------------------------
 
 
 NoDefaultSpecified = Sentinel('NoDefaultSpecified', __name__, 
 '''
 Used in Traitlets to specify that no defaults are set in kwargs
 '''
 )
 
 
 class Undefined ( object ): pass
 Undefined = Undefined()
 
 class TraitError(Exception):
     pass
 
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
 
 def class_of ( object ):
     """ Returns a string containing the class name of an object with the
     correct indefinite article ('a' or 'an') preceding it (e.g., 'an Image',
     'a PlotValue').
     """
     if isinstance( object, py3compat.string_types ):
         return add_article( object )
 
     return add_article( object.__class__.__name__ )
 
 
 def add_article ( name ):
     """ Returns a string containing the correct indefinite article ('a' or 'an')
     prefixed to the specified string.
     """
     if name[:1].lower() in 'aeiou':
        return 'an ' + name
 
     return 'a ' + name
 
 
 def repr_type(obj):
     """ Return a string representation of a value and its type for readable
     error messages.
     """
     the_type = type(obj)
     if (not py3compat.PY3) and the_type is InstanceType:
         # Old-style class.
         the_type = obj.__class__
     msg = '%r %r' % (obj, the_type)
     return msg
 
 
 def is_trait(t):
     """ Returns whether the given value is an instance or subclass of TraitType.
     """
     return (isinstance(t, TraitType) or
             (isinstance(t, type) and issubclass(t, TraitType)))
 
 
 def parse_notifier_name(name):
     """Convert the name argument to a list of names.
 
     Examples
     --------
 
     >>> parse_notifier_name('a')
     ['a']
     >>> parse_notifier_name(['a','b'])
     ['a', 'b']
     >>> parse_notifier_name(None)
     ['anytrait']
     """
     if isinstance(name, string_types):
         return [name]
     elif name is None:
         return ['anytrait']
     elif isinstance(name, (list, tuple)):
         for n in name:
             assert isinstance(n, string_types), "names must be strings"
         return name
 
 
 class _SimpleTest:
     def __init__ ( self, value ): self.value = value
     def __call__ ( self, test  ):
         return test == self.value
     def __repr__(self):
         return "<SimpleTest(%r)" % self.value
     def __str__(self):
         return self.__repr__()
 
 
 def getmembers(object, predicate=None):
     """A safe version of inspect.getmembers that handles missing attributes.
 
     This is useful when there are descriptor based attributes that for
     some reason raise AttributeError even though they exist.  This happens
     in zope.inteface with the __provides__ attribute.
     """
     results = []
     for key in dir(object):
         try:
             value = getattr(object, key)
         except AttributeError:
             pass
         else:
             if not predicate or predicate(value):
                 results.append((key, value))
     results.sort()
     return results
 
 def _validate_link(*tuples):
     """Validate arguments for traitlet link functions"""
     for t in tuples:
         if not len(t) == 2:
             raise TypeError("Each linked traitlet must be specified as (HasTraits, 'trait_name'), not %r" % t)
         obj, trait_name = t
         if not isinstance(obj, HasTraits):
             raise TypeError("Each object must be HasTraits, not %r" % type(obj))
         if not trait_name in obj.traits():
             raise TypeError("%r has no trait %r" % (obj, trait_name))
 
-@skip_doctest
 class link(object):
     """Link traits from different objects together so they remain in sync.
 
     Parameters
     ----------
     *args : pairs of objects/attributes
 
     Examples
     --------
 
     >>> c = link((obj1, 'value'), (obj2, 'value'), (obj3, 'value'))
     >>> obj1.value = 5 # updates other objects as well
     """
     updating = False
     def __init__(self, *args):
         if len(args) < 2:
             raise TypeError('At least two traitlets must be provided.')
         _validate_link(*args)
 
         self.objects = {}
 
         initial = getattr(args[0][0], args[0][1])
         for obj, attr in args:
             setattr(obj, attr, initial)
 
             callback = self._make_closure(obj, attr)
             obj.on_trait_change(callback, attr)
             self.objects[(obj, attr)] = callback
 
     @contextlib.contextmanager
     def _busy_updating(self):
         self.updating = True
         try:
             yield
         finally:
             self.updating = False
 
     def _make_closure(self, sending_obj, sending_attr):
         def update(name, old, new):
             self._update(sending_obj, sending_attr, new)
         return update
 
     def _update(self, sending_obj, sending_attr, new):
         if self.updating:
             return
         with self._busy_updating():
             for obj, attr in self.objects.keys():
                 setattr(obj, attr, new)
 
     def unlink(self):
         for key, callback in self.objects.items():
             (obj, attr) = key
             obj.on_trait_change(callback, attr, remove=True)
 
-@skip_doctest
 class directional_link(object):
     """Link the trait of a source object with traits of target objects.
 
     Parameters
     ----------
     source : pair of object, name
     targets : pairs of objects/attributes
 
     Examples
     --------
 
     >>> c = directional_link((src, 'value'), (tgt1, 'value'), (tgt2, 'value'))
     >>> src.value = 5  # updates target objects
     >>> tgt1.value = 6 # does not update other objects
     """
     updating = False
 
     def __init__(self, source, *targets):
         if len(targets) < 1:
             raise TypeError('At least two traitlets must be provided.')
         _validate_link(source, *targets)
         self.source = source
         self.targets = targets
 
         # Update current value
         src_attr_value = getattr(source[0], source[1])
         for obj, attr in targets:
             setattr(obj, attr, src_attr_value)
 
         # Wire
         self.source[0].on_trait_change(self._update, self.source[1])
 
     @contextlib.contextmanager
     def _busy_updating(self):
         self.updating = True
         try:
             yield
         finally:
             self.updating = False
 
     def _update(self, name, old, new):
         if self.updating:
             return
         with self._busy_updating():
             for obj, attr in self.targets:
                 setattr(obj, attr, new)
 
     def unlink(self):
         self.source[0].on_trait_change(self._update, self.source[1], remove=True)
         self.source = None
         self.targets = []
 
 dlink = directional_link
 
 
 #-----------------------------------------------------------------------------
 # Base TraitType for all traits
 #-----------------------------------------------------------------------------
 
 
 class TraitType(object):
     """A base class for all trait descriptors.
 
     Notes
     -----
     Our implementation of traits is based on Python's descriptor
     prototol.  This class is the base class for all such descriptors.  The
     only magic we use is a custom metaclass for the main :class:`HasTraits`
     class that does the following:
 
     1. Sets the :attr:`name` attribute of every :class:`TraitType`
        instance in the class dict to the name of the attribute.
     2. Sets the :attr:`this_class` attribute of every :class:`TraitType`
        instance in the class dict to the *class* that declared the trait.
        This is used by the :class:`This` trait to allow subclasses to
        accept superclasses for :class:`This` values.
     """
 
     metadata = {}
     default_value = Undefined
     allow_none = False
     info_text = 'any value'
 
     def __init__(self, default_value=NoDefaultSpecified, allow_none=None, **metadata):
         """Create a TraitType.
         """
         if default_value is not NoDefaultSpecified:
             self.default_value = default_value
         if allow_none is not None:
             self.allow_none = allow_none
 
         if 'default' in metadata:
             # Warn the user that they probably meant default_value.
             warn(
                 "Parameter 'default' passed to TraitType. "
                 "Did you mean 'default_value'?"
             )
 
         if len(metadata) > 0:
             if len(self.metadata) > 0:
                 self._metadata = self.metadata.copy()
                 self._metadata.update(metadata)
             else:
                 self._metadata = metadata
         else:
             self._metadata = self.metadata
 
         self.init()
 
     def init(self):
         pass
 
     def get_default_value(self):
         """Create a new instance of the default value."""
         return self.default_value
 
     def instance_init(self):
         """Part of the initialization which may depends on the underlying
         HasTraits instance.
 
         It is typically overloaded for specific trait types.
 
         This method is called by :meth:`HasTraits.__new__` and in the
         :meth:`TraitType.instance_init` method of trait types holding
         other trait types.
         """
         pass
 
     def init_default_value(self, obj):
         """Instantiate the default value for the trait type.
 
         This method is called by :meth:`TraitType.set_default_value` in the
         case a default value is provided at construction time or later when
         accessing the trait value for the first time in
         :meth:`HasTraits.__get__`.
         """
         value = self.get_default_value()
         value = self._validate(obj, value)
         obj._trait_values[self.name] = value
         return value
 
     def set_default_value(self, obj):
         """Set the default value on a per instance basis.
 
         This method is called by :meth:`HasTraits.__new__` to instantiate and
         validate the default value. The creation and validation of
         default values must be delayed until the parent :class:`HasTraits`
         class has been instantiated.
         Parameters
         ----------
         obj : :class:`HasTraits` instance
             The parent :class:`HasTraits` instance that has just been
             created.
         """
         # Check for a deferred initializer defined in the same class as the
         # trait declaration or above.
         mro = type(obj).mro()
         meth_name = '_%s_default' % self.name
         for cls in mro[:mro.index(self.this_class)+1]:
             if meth_name in cls.__dict__:
                 break
         else:
             # We didn't find one. Do static initialization.
             self.init_default_value(obj)
             return
         # Complete the dynamic initialization.
         obj._trait_dyn_inits[self.name] = meth_name
 
     def __get__(self, obj, cls=None):
         """Get the value of the trait by self.name for the instance.
 
         Default values are instantiated when :meth:`HasTraits.__new__`
         is called.  Thus by the time this method gets called either the
         default value or a user defined value (they called :meth:`__set__`)
         is in the :class:`HasTraits` instance.
         """
         if obj is None:
             return self
         else:
             try:
                 value = obj._trait_values[self.name]
             except KeyError:
                 # Check for a dynamic initializer.
                 if self.name in obj._trait_dyn_inits:
                     method = getattr(obj, obj._trait_dyn_inits[self.name])
                     value = method()
                     # FIXME: Do we really validate here?
                     value = self._validate(obj, value)
                     obj._trait_values[self.name] = value
                     return value
                 else:
                     return self.init_default_value(obj)
             except Exception:
                 # HasTraits should call set_default_value to populate
                 # this.  So this should never be reached.
                 raise TraitError('Unexpected error in TraitType: '
                                  'default value not set properly')
             else:
                 return value
 
     def __set__(self, obj, value):
         new_value = self._validate(obj, value)
         try:
             old_value = obj._trait_values[self.name]
         except KeyError:
             old_value = Undefined
 
         obj._trait_values[self.name] = new_value
         try:
             silent = bool(old_value == new_value)
         except:
             # if there is an error in comparing, default to notify
             silent = False
         if silent is not True:
             # we explicitly compare silent to True just in case the equality
             # comparison above returns something other than True/False
             obj._notify_trait(self.name, old_value, new_value)
 
     def _validate(self, obj, value):
         if value is None and self.allow_none:
             return value
         if hasattr(self, 'validate'):
             value = self.validate(obj, value)
         if obj._cross_validation_lock is False:
             value = self._cross_validate(obj, value)
         return value
 
     def _cross_validate(self, obj, value):
         if hasattr(obj, '_%s_validate' % self.name):
             cross_validate = getattr(obj, '_%s_validate' % self.name)
             value = cross_validate(value, self)
         return value
 
     def __or__(self, other):
         if isinstance(other, Union):
             return Union([self] + other.trait_types)
         else:
             return Union([self, other])
 
     def info(self):
         return self.info_text
 
     def error(self, obj, value):
         if obj is not None:
             e = "The '%s' trait of %s instance must be %s, but a value of %s was specified." \
                 % (self.name, class_of(obj),
                    self.info(), repr_type(value))
         else:
             e = "The '%s' trait must be %s, but a value of %r was specified." \
                 % (self.name, self.info(), repr_type(value))
         raise TraitError(e)
 
     def get_metadata(self, key, default=None):
         return getattr(self, '_metadata', {}).get(key, default)
 
     def set_metadata(self, key, value):
         getattr(self, '_metadata', {})[key] = value
 
 
 #-----------------------------------------------------------------------------
 # The HasTraits implementation
 #-----------------------------------------------------------------------------
 
 
 class MetaHasTraits(type):
     """A metaclass for HasTraits.
 
     This metaclass makes sure that any TraitType class attributes are
     instantiated and sets their name attribute.
     """
 
     def __new__(mcls, name, bases, classdict):
         """Create the HasTraits class.
 
         This instantiates all TraitTypes in the class dict and sets their
         :attr:`name` attribute.
         """
         # print "MetaHasTraitlets (mcls, name): ", mcls, name
         # print "MetaHasTraitlets (bases): ", bases
         # print "MetaHasTraitlets (classdict): ", classdict
         for k,v in iteritems(classdict):
             if isinstance(v, TraitType):
                 v.name = k
             elif inspect.isclass(v):
                 if issubclass(v, TraitType):
                     vinst = v()
                     vinst.name = k
                     classdict[k] = vinst
         return super(MetaHasTraits, mcls).__new__(mcls, name, bases, classdict)
 
     def __init__(cls, name, bases, classdict):
         """Finish initializing the HasTraits class.
 
         This sets the :attr:`this_class` attribute of each TraitType in the
         class dict to the newly created class ``cls``.
         """
         for k, v in iteritems(classdict):
             if isinstance(v, TraitType):
                 v.this_class = cls
         super(MetaHasTraits, cls).__init__(name, bases, classdict)
 
 
 class HasTraits(py3compat.with_metaclass(MetaHasTraits, object)):
 
     def __new__(cls, *args, **kw):
         # This is needed because object.__new__ only accepts
         # the cls argument.
         new_meth = super(HasTraits, cls).__new__
         if new_meth is object.__new__:
             inst = new_meth(cls)
         else:
             inst = new_meth(cls, **kw)
         inst._trait_values = {}
         inst._trait_notifiers = {}
         inst._trait_dyn_inits = {}
         inst._cross_validation_lock = True
         # Here we tell all the TraitType instances to set their default
         # values on the instance.
         for key in dir(cls):
             # Some descriptors raise AttributeError like zope.interface's
             # __provides__ attributes even though they exist.  This causes
             # AttributeErrors even though they are listed in dir(cls).
             try:
                 value = getattr(cls, key)
             except AttributeError:
                 pass
             else:
                 if isinstance(value, TraitType):
                     value.instance_init()
                     if key not in kw:
                         value.set_default_value(inst)
         inst._cross_validation_lock = False
         return inst
 
     def __init__(self, *args, **kw):
         # Allow trait values to be set using keyword arguments.
         # We need to use setattr for this to trigger validation and
         # notifications.
         with self.hold_trait_notifications():
             for key, value in iteritems(kw):
                 setattr(self, key, value)
 
     @contextlib.contextmanager
     def hold_trait_notifications(self):
         """Context manager for bundling trait change notifications and cross
         validation.
 
         Use this when doing multiple trait assignments (init, config), to avoid
         race conditions in trait notifiers requesting other trait values.
         All trait notifications will fire after all values have been assigned.
         """
         if self._cross_validation_lock is True:
             yield
             return
         else:
             self._cross_validation_lock = True
             cache = {}
             notifications = {}
             _notify_trait = self._notify_trait
 
             def cache_values(*a):
                 cache[a[0]] = a
 
             def hold_notifications(*a):
                 notifications[a[0]] = a
 
             self._notify_trait = cache_values
 
             try:
                 yield
             finally:
                 try:
                     self._notify_trait = hold_notifications
                     for name in cache:
                         if hasattr(self, '_%s_validate' % name):
                             cross_validate = getattr(self, '_%s_validate' % name)
                             setattr(self, name, cross_validate(getattr(self, name), self))
                 except TraitError as e:
                     self._notify_trait = lambda *x: None
                     for name in cache:
                         if cache[name][1] is not Undefined:
                             setattr(self, name, cache[name][1])
                         else:
                             delattr(self, name)
                     cache = {}
                     notifications = {}
                     raise e
                 finally:
                     self._notify_trait = _notify_trait
                     self._cross_validation_lock = False
                     if isinstance(_notify_trait, types.MethodType):
                         # FIXME: remove when support is bumped to 3.4.
                         # when original method is restored,
                         # remove the redundant value from __dict__
                         # (only used to preserve pickleability on Python < 3.4)
                         self.__dict__.pop('_notify_trait', None)
                     # trigger delayed notifications
                     for v in dict(cache, **notifications).values():
                         self._notify_trait(*v)
 
     def _notify_trait(self, name, old_value, new_value):
 
         # First dynamic ones
         callables = []
         callables.extend(self._trait_notifiers.get(name,[]))
         callables.extend(self._trait_notifiers.get('anytrait',[]))
 
         # Now static ones
         try:
             cb = getattr(self, '_%s_changed' % name)
         except:
             pass
         else:
             callables.append(cb)
 
         # Call them all now
         for c in callables:
             # Traits catches and logs errors here.  I allow them to raise
             if callable(c):
                 argspec = getargspec(c)
 
                 nargs = len(argspec[0])
                 # Bound methods have an additional 'self' argument
                 # I don't know how to treat unbound methods, but they
                 # can't really be used for callbacks.
                 if isinstance(c, types.MethodType):
                     offset = -1
                 else:
                     offset = 0
                 if nargs + offset == 0:
                     c()
                 elif nargs + offset == 1:
                     c(name)
                 elif nargs + offset == 2:
                     c(name, new_value)
                 elif nargs + offset == 3:
                     c(name, old_value, new_value)
                 else:
                     raise TraitError('a trait changed callback '
                                         'must have 0-3 arguments.')
             else:
                 raise TraitError('a trait changed callback '
                                     'must be callable.')
 
 
     def _add_notifiers(self, handler, name):
         if name not in self._trait_notifiers:
             nlist = []
             self._trait_notifiers[name] = nlist
         else:
             nlist = self._trait_notifiers[name]
         if handler not in nlist:
             nlist.append(handler)
 
     def _remove_notifiers(self, handler, name):
         if name in self._trait_notifiers:
             nlist = self._trait_notifiers[name]
             try:
                 index = nlist.index(handler)
             except ValueError:
                 pass
             else:
                 del nlist[index]
 
     def on_trait_change(self, handler, name=None, remove=False):
         """Setup a handler to be called when a trait changes.
 
         This is used to setup dynamic notifications of trait changes.
 
         Static handlers can be created by creating methods on a HasTraits
         subclass with the naming convention '_[traitname]_changed'.  Thus,
         to create static handler for the trait 'a', create the method
         _a_changed(self, name, old, new) (fewer arguments can be used, see
         below).
 
         Parameters
         ----------
         handler : callable
             A callable that is called when a trait changes.  Its
             signature can be handler(), handler(name), handler(name, new)
             or handler(name, old, new).
         name : list, str, None
             If None, the handler will apply to all traits.  If a list
             of str, handler will apply to all names in the list.  If a
             str, the handler will apply just to that name.
         remove : bool
             If False (the default), then install the handler.  If True
             then unintall it.
         """
         if remove:
             names = parse_notifier_name(name)
             for n in names:
                 self._remove_notifiers(handler, n)
         else:
             names = parse_notifier_name(name)
             for n in names:
                 self._add_notifiers(handler, n)
 
     @classmethod
     def class_trait_names(cls, **metadata):
         """Get a list of all the names of this class' traits.
 
         This method is just like the :meth:`trait_names` method,
         but is unbound.
         """
         return cls.class_traits(**metadata).keys()
 
     @classmethod
     def class_traits(cls, **metadata):
         """Get a `dict` of all the traits of this class.  The dictionary
         is keyed on the name and the values are the TraitType objects.
 
         This method is just like the :meth:`traits` method, but is unbound.
 
         The TraitTypes returned don't know anything about the values
         that the various HasTrait's instances are holding.
 
         The metadata kwargs allow functions to be passed in which
         filter traits based on metadata values.  The functions should
         take a single value as an argument and return a boolean.  If
         any function returns False, then the trait is not included in
         the output.  This does not allow for any simple way of
         testing that a metadata name exists and has any
         value because get_metadata returns None if a metadata key
         doesn't exist.
         """
         traits = dict([memb for memb in getmembers(cls) if
                      isinstance(memb[1], TraitType)])
 
         if len(metadata) == 0:
             return traits
 
         for meta_name, meta_eval in metadata.items():
             if type(meta_eval) is not FunctionType:
                 metadata[meta_name] = _SimpleTest(meta_eval)
 
         result = {}
         for name, trait in traits.items():
             for meta_name, meta_eval in metadata.items():
                 if not meta_eval(trait.get_metadata(meta_name)):
                     break
             else:
                 result[name] = trait
 
         return result
 
     def trait_names(self, **metadata):
         """Get a list of all the names of this class' traits."""
         return self.traits(**metadata).keys()
 
     def traits(self, **metadata):
         """Get a `dict` of all the traits of this class.  The dictionary
         is keyed on the name and the values are the TraitType objects.
 
         The TraitTypes returned don't know anything about the values
         that the various HasTrait's instances are holding.
 
         The metadata kwargs allow functions to be passed in which
         filter traits based on metadata values.  The functions should
         take a single value as an argument and return a boolean.  If
         any function returns False, then the trait is not included in
         the output.  This does not allow for any simple way of
         testing that a metadata name exists and has any
         value because get_metadata returns None if a metadata key
         doesn't exist.
         """
         traits = dict([memb for memb in getmembers(self.__class__) if
                      isinstance(memb[1], TraitType)])
 
         if len(metadata) == 0:
             return traits
 
         for meta_name, meta_eval in metadata.items():
             if type(meta_eval) is not FunctionType:
                 metadata[meta_name] = _SimpleTest(meta_eval)
 
         result = {}
         for name, trait in traits.items():
             for meta_name, meta_eval in metadata.items():
                 if not meta_eval(trait.get_metadata(meta_name)):
                     break
             else:
                 result[name] = trait
 
         return result
 
     def trait_metadata(self, traitname, key, default=None):
         """Get metadata values for trait by key."""
         try:
             trait = getattr(self.__class__, traitname)
         except AttributeError:
             raise TraitError("Class %s does not have a trait named %s" %
                                 (self.__class__.__name__, traitname))
         else:
             return trait.get_metadata(key, default)
 
     def add_trait(self, traitname, trait):
         """Dynamically add a trait attribute to the HasTraits instance."""
         self.__class__ = type(self.__class__.__name__, (self.__class__,),
                               {traitname: trait})
         trait.set_default_value(self)
 
 #-----------------------------------------------------------------------------
 # Actual TraitTypes implementations/subclasses
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # TraitTypes subclasses for handling classes and instances of classes
 #-----------------------------------------------------------------------------
 
 
 class ClassBasedTraitType(TraitType):
     """
     A trait with error reporting and string -> type resolution for Type,
     Instance and This.
     """
 
     def _resolve_string(self, string):
         """
         Resolve a string supplied for a type into an actual object.
         """
         return import_item(string)
 
     def error(self, obj, value):
         kind = type(value)
         if (not py3compat.PY3) and kind is InstanceType:
             msg = 'class %s' % value.__class__.__name__
         else:
             msg = '%s (i.e. %s)' % ( str( kind )[1:-1], repr( value ) )
 
         if obj is not None:
             e = "The '%s' trait of %s instance must be %s, but a value of %s was specified." \
                 % (self.name, class_of(obj),
                    self.info(), msg)
         else:
             e = "The '%s' trait must be %s, but a value of %r was specified." \
                 % (self.name, self.info(), msg)
 
         raise TraitError(e)
 
 
 class Type(ClassBasedTraitType):
     """A trait whose value must be a subclass of a specified class."""
 
     def __init__ (self, default_value=None, klass=None, allow_none=False,
                   **metadata):
         """Construct a Type trait
 
         A Type trait specifies that its values must be subclasses of
         a particular class.
 
         If only ``default_value`` is given, it is used for the ``klass`` as
         well.
 
         Parameters
         ----------
         default_value : class, str or None
             The default value must be a subclass of klass.  If an str,
             the str must be a fully specified class name, like 'foo.bar.Bah'.
             The string is resolved into real class, when the parent
             :class:`HasTraits` class is instantiated.
         klass : class, str, None
             Values of this trait must be a subclass of klass.  The klass
             may be specified in a string like: 'foo.bar.MyClass'.
             The string is resolved into real class, when the parent
             :class:`HasTraits` class is instantiated.
         allow_none : bool [ default True ]
             Indicates whether None is allowed as an assignable value. Even if
             ``False``, the default value may be ``None``.
         """
         if default_value is None:
             if klass is None:
                 klass = object
         elif klass is None:
             klass = default_value
 
         if not (inspect.isclass(klass) or isinstance(klass, py3compat.string_types)):
             raise TraitError("A Type trait must specify a class.")
 
         self.klass       = klass
 
         super(Type, self).__init__(default_value, allow_none=allow_none, **metadata)
 
     def validate(self, obj, value):
         """Validates that the value is a valid object instance."""
         if isinstance(value, py3compat.string_types):
             try:
                 value = self._resolve_string(value)
             except ImportError:
                 raise TraitError("The '%s' trait of %s instance must be a type, but "
                                  "%r could not be imported" % (self.name, obj, value))
         try:
             if issubclass(value, self.klass):
                 return value
         except:
             pass
 
         self.error(obj, value)
 
     def info(self):
         """ Returns a description of the trait."""
         if isinstance(self.klass, py3compat.string_types):
             klass = self.klass
         else:
             klass = self.klass.__name__
         result = 'a subclass of ' + klass
         if self.allow_none:
             return result + ' or None'
         return result
 
     def instance_init(self):
         self._resolve_classes()
         super(Type, self).instance_init()
 
     def _resolve_classes(self):
         if isinstance(self.klass, py3compat.string_types):
             self.klass = self._resolve_string(self.klass)
         if isinstance(self.default_value, py3compat.string_types):
             self.default_value = self._resolve_string(self.default_value)
 
     def get_default_value(self):
         return self.default_value
 
 
 class DefaultValueGenerator(object):
     """A class for generating new default value instances."""
 
     def __init__(self, *args, **kw):
         self.args = args
         self.kw = kw
 
     def generate(self, klass):
         return klass(*self.args, **self.kw)
 
 
 class Instance(ClassBasedTraitType):
     """A trait whose value must be an instance of a specified class.
 
     The value can also be an instance of a subclass of the specified class.
 
     Subclasses can declare default classes by overriding the klass attribute
     """
 
     klass = None
 
     def __init__(self, klass=None, args=None, kw=None, allow_none=False,
                  **metadata ):
         """Construct an Instance trait.
 
         This trait allows values that are instances of a particular
         class or its subclasses.  Our implementation is quite different
         from that of enthough.traits as we don't allow instances to be used
         for klass and we handle the ``args`` and ``kw`` arguments differently.
 
         Parameters
         ----------
         klass : class, str
             The class that forms the basis for the trait.  Class names
             can also be specified as strings, like 'foo.bar.Bar'.
         args : tuple
             Positional arguments for generating the default value.
         kw : dict
             Keyword arguments for generating the default value.
         allow_none : bool [default True]
             Indicates whether None is allowed as a value.
 
         Notes
         -----
         If both ``args`` and ``kw`` are None, then the default value is None.
         If ``args`` is a tuple and ``kw`` is a dict, then the default is
         created as ``klass(*args, **kw)``.  If exactly one of ``args`` or ``kw`` is
         None, the None is replaced by ``()`` or ``{}``, respectively.
         """
         if klass is None:
             klass = self.klass
         
         if (klass is not None) and (inspect.isclass(klass) or isinstance(klass, py3compat.string_types)):
             self.klass = klass
         else:
             raise TraitError('The klass attribute must be a class'
                                 ' not: %r' % klass)
 
         # self.klass is a class, so handle default_value
         if args is None and kw is None:
             default_value = None
         else:
             if args is None:
                 # kw is not None
                 args = ()
             elif kw is None:
                 # args is not None
                 kw = {}
 
             if not isinstance(kw, dict):
                 raise TraitError("The 'kw' argument must be a dict or None.")
             if not isinstance(args, tuple):
                 raise TraitError("The 'args' argument must be a tuple or None.")
 
             default_value = DefaultValueGenerator(*args, **kw)
 
         super(Instance, self).__init__(default_value, allow_none=allow_none, **metadata)
 
     def validate(self, obj, value):
         if isinstance(value, self.klass):
             return value
         else:
             self.error(obj, value)
 
     def info(self):
         if isinstance(self.klass, py3compat.string_types):
             klass = self.klass
         else:
             klass = self.klass.__name__
         result = class_of(klass)
         if self.allow_none:
             return result + ' or None'
 
         return result
 
     def instance_init(self):
         self._resolve_classes()
         super(Instance, self).instance_init()
 
     def _resolve_classes(self):
         if isinstance(self.klass, py3compat.string_types):
             self.klass = self._resolve_string(self.klass)
 
     def get_default_value(self):
         """Instantiate a default value instance.
 
         This is called when the containing HasTraits classes'
         :meth:`__new__` method is called to ensure that a unique instance
         is created for each HasTraits instance.
         """
         dv  = self.default_value
         if isinstance(dv, DefaultValueGenerator):
             return dv.generate(self.klass)
         else:
             return dv
 
 
 class ForwardDeclaredMixin(object):
     """
     Mixin for forward-declared versions of Instance and Type.
     """
     def _resolve_string(self, string):
         """
         Find the specified class name by looking for it in the module in which
         our this_class attribute was defined.
         """
         modname = self.this_class.__module__
         return import_item('.'.join([modname, string]))
 
 
 class ForwardDeclaredType(ForwardDeclaredMixin, Type):
     """
     Forward-declared version of Type.
     """
     pass
 
 
 class ForwardDeclaredInstance(ForwardDeclaredMixin, Instance):
     """
     Forward-declared version of Instance.
     """
     pass
 
 
 class This(ClassBasedTraitType):
     """A trait for instances of the class containing this trait.
 
     Because how how and when class bodies are executed, the ``This``
     trait can only have a default value of None.  This, and because we
     always validate default values, ``allow_none`` is *always* true.
     """
 
     info_text = 'an instance of the same type as the receiver or None'
 
     def __init__(self, **metadata):
         super(This, self).__init__(None, **metadata)
 
     def validate(self, obj, value):
         # What if value is a superclass of obj.__class__?  This is
         # complicated if it was the superclass that defined the This
         # trait.
         if isinstance(value, self.this_class) or (value is None):
             return value
         else:
             self.error(obj, value)
 
 
 class Union(TraitType):
     """A trait type representing a Union type."""
 
     def __init__(self, trait_types, **metadata):
         """Construct a Union  trait.
 
         This trait allows values that are allowed by at least one of the
         specified trait types. A Union traitlet cannot have metadata on
         its own, besides the metadata of the listed types.
 
         Parameters
         ----------
         trait_types: sequence
             The list of trait types of length at least 1.
 
         Notes
         -----
         Union([Float(), Bool(), Int()]) attempts to validate the provided values
         with the validation function of Float, then Bool, and finally Int.
         """
         self.trait_types = trait_types
         self.info_text = " or ".join([tt.info_text for tt in self.trait_types])
         self.default_value = self.trait_types[0].get_default_value()
         super(Union, self).__init__(**metadata)
 
     def instance_init(self):
         for trait_type in self.trait_types:
             trait_type.name = self.name
             trait_type.this_class = self.this_class
             trait_type.instance_init()
         super(Union, self).instance_init()
 
     def validate(self, obj, value):
         for trait_type in self.trait_types:
             try:
                 v = trait_type._validate(obj, value)
                 self._metadata = trait_type._metadata
                 return v
             except TraitError:
                 continue
         self.error(obj, value)
 
     def __or__(self, other):
         if isinstance(other, Union):
             return Union(self.trait_types + other.trait_types)
         else:
             return Union(self.trait_types + [other])
 
 #-----------------------------------------------------------------------------
 # Basic TraitTypes implementations/subclasses
 #-----------------------------------------------------------------------------
 
 
 class Any(TraitType):
     default_value = None
     info_text = 'any value'
 
 
 class Int(TraitType):
     """An int trait."""
 
     default_value = 0
     info_text = 'an int'
 
     def validate(self, obj, value):
         if isinstance(value, int):
             return value
         self.error(obj, value)
 
 class CInt(Int):
     """A casting version of the int trait."""
 
     def validate(self, obj, value):
         try:
             return int(value)
         except:
             self.error(obj, value)
 
 if py3compat.PY3:
     Long, CLong = Int, CInt
     Integer = Int
 else:
     class Long(TraitType):
         """A long integer trait."""
 
         default_value = 0
         info_text = 'a long'
 
         def validate(self, obj, value):
             if isinstance(value, long):
                 return value
             if isinstance(value, int):
                 return long(value)
             self.error(obj, value)
 
 
     class CLong(Long):
         """A casting version of the long integer trait."""
 
         def validate(self, obj, value):
             try:
                 return long(value)
             except:
                 self.error(obj, value)
 
     class Integer(TraitType):
         """An integer trait.
 
         Longs that are unnecessary (<= sys.maxint) are cast to ints."""
 
         default_value = 0
         info_text = 'an integer'
 
         def validate(self, obj, value):
             if isinstance(value, int):
                 return value
             if isinstance(value, long):
                 # downcast longs that fit in int:
                 # note that int(n > sys.maxint) returns a long, so
                 # we don't need a condition on this cast
                 return int(value)
             if sys.platform == "cli":
                 from System import Int64
                 if isinstance(value, Int64):
                     return int(value)
             self.error(obj, value)
 
 
 class Float(TraitType):
     """A float trait."""
 
     default_value = 0.0
     info_text = 'a float'
 
     def validate(self, obj, value):
         if isinstance(value, float):
             return value
         if isinstance(value, int):
             return float(value)
         self.error(obj, value)
 
 
 class CFloat(Float):
     """A casting version of the float trait."""
 
     def validate(self, obj, value):
         try:
             return float(value)
         except:
             self.error(obj, value)
 
 class Complex(TraitType):
     """A trait for complex numbers."""
 
     default_value = 0.0 + 0.0j
     info_text = 'a complex number'
 
     def validate(self, obj, value):
         if isinstance(value, complex):
             return value
         if isinstance(value, (float, int)):
             return complex(value)
         self.error(obj, value)
 
 
 class CComplex(Complex):
     """A casting version of the complex number trait."""
 
     def validate (self, obj, value):
         try:
             return complex(value)
         except:
             self.error(obj, value)
 
 # We should always be explicit about whether we're using bytes or unicode, both
 # for Python 3 conversion and for reliable unicode behaviour on Python 2. So
 # we don't have a Str type.
 class Bytes(TraitType):
     """A trait for byte strings."""
 
     default_value = b''
     info_text = 'a bytes object'
 
     def validate(self, obj, value):
         if isinstance(value, bytes):
             return value
         self.error(obj, value)
 
 
 class CBytes(Bytes):
     """A casting version of the byte string trait."""
 
     def validate(self, obj, value):
         try:
             return bytes(value)
         except:
             self.error(obj, value)
 
 
 class Unicode(TraitType):
     """A trait for unicode strings."""
 
     default_value = u''
     info_text = 'a unicode string'
 
     def validate(self, obj, value):
         if isinstance(value, py3compat.unicode_type):
             return value
         if isinstance(value, bytes):
             try:
                 return value.decode('ascii', 'strict')
             except UnicodeDecodeError:
                 msg = "Could not decode {!r} for unicode trait '{}' of {} instance."
                 raise TraitError(msg.format(value, self.name, class_of(obj)))
         self.error(obj, value)
 
 
 class CUnicode(Unicode):
     """A casting version of the unicode trait."""
 
     def validate(self, obj, value):
         try:
             return py3compat.unicode_type(value)
         except:
             self.error(obj, value)
 
 
 class ObjectName(TraitType):
     """A string holding a valid object name in this version of Python.
 
     This does not check that the name exists in any scope."""
     info_text = "a valid object identifier in Python"
 
     if py3compat.PY3:
         # Python 3:
         coerce_str = staticmethod(lambda _,s: s)
 
     else:
         # Python 2:
         def coerce_str(self, obj, value):
             "In Python 2, coerce ascii-only unicode to str"
             if isinstance(value, unicode):
                 try:
                     return str(value)
                 except UnicodeEncodeError:
                     self.error(obj, value)
             return value
 
     def validate(self, obj, value):
         value = self.coerce_str(obj, value)
 
         if isinstance(value, string_types) and py3compat.isidentifier(value):
             return value
         self.error(obj, value)
 
 class DottedObjectName(ObjectName):
     """A string holding a valid dotted object name in Python, such as A.b3._c"""
     def validate(self, obj, value):
         value = self.coerce_str(obj, value)
 
         if isinstance(value, string_types) and py3compat.isidentifier(value, dotted=True):
             return value
         self.error(obj, value)
 
 
 class Bool(TraitType):
     """A boolean (True, False) trait."""
 
     default_value = False
     info_text = 'a boolean'
 
     def validate(self, obj, value):
         if isinstance(value, bool):
             return value
         self.error(obj, value)
 
 
 class CBool(Bool):
     """A casting version of the boolean trait."""
 
     def validate(self, obj, value):
         try:
             return bool(value)
         except:
             self.error(obj, value)
 
 
 class Enum(TraitType):
     """An enum that whose value must be in a given sequence."""
 
     def __init__(self, values, default_value=None, **metadata):
         self.values = values
         super(Enum, self).__init__(default_value, **metadata)
 
     def validate(self, obj, value):
         if value in self.values:
                 return value
         self.error(obj, value)
 
     def info(self):
         """ Returns a description of the trait."""
         result = 'any of ' + repr(self.values)
         if self.allow_none:
             return result + ' or None'
         return result
 
 class CaselessStrEnum(Enum):
     """An enum of strings that are caseless in validate."""
 
     def validate(self, obj, value):
         if not isinstance(value, py3compat.string_types):
             self.error(obj, value)
 
         for v in self.values:
             if v.lower() == value.lower():
                 return v
         self.error(obj, value)
 
 class Container(Instance):
     """An instance of a container (list, set, etc.)
 
     To be subclassed by overriding klass.
     """
     klass = None
     _cast_types = ()
     _valid_defaults = SequenceTypes
     _trait = None
 
     def __init__(self, trait=None, default_value=None, allow_none=False,
                 **metadata):
         """Create a container trait type from a list, set, or tuple.
 
         The default value is created by doing ``List(default_value)``,
         which creates a copy of the ``default_value``.
 
         ``trait`` can be specified, which restricts the type of elements
         in the container to that TraitType.
 
         If only one arg is given and it is not a Trait, it is taken as
         ``default_value``:
 
         ``c = List([1,2,3])``
 
         Parameters
         ----------
 
         trait : TraitType [ optional ]
             the type for restricting the contents of the Container.  If unspecified,
             types are not checked.
 
         default_value : SequenceType [ optional ]
             The default value for the Trait.  Must be list/tuple/set, and
             will be cast to the container type.
 
         allow_none : bool [ default False ]
             Whether to allow the value to be None
 
         **metadata : any
             further keys for extensions to the Trait (e.g. config)
 
         """
         # allow List([values]):
         if default_value is None and not is_trait(trait):
             default_value = trait
             trait = None
 
         if default_value is None:
             args = ()
         elif isinstance(default_value, self._valid_defaults):
             args = (default_value,)
         else:
             raise TypeError('default value of %s was %s' %(self.__class__.__name__, default_value))
 
         if is_trait(trait):
             self._trait = trait() if isinstance(trait, type) else trait
             self._trait.name = 'element'
         elif trait is not None:
             raise TypeError("`trait` must be a Trait or None, got %s"%repr_type(trait))
 
         super(Container,self).__init__(klass=self.klass, args=args,
                                   allow_none=allow_none, **metadata)
 
     def element_error(self, obj, element, validator):
         e = "Element of the '%s' trait of %s instance must be %s, but a value of %s was specified." \
             % (self.name, class_of(obj), validator.info(), repr_type(element))
         raise TraitError(e)
 
     def validate(self, obj, value):
         if isinstance(value, self._cast_types):
             value = self.klass(value)
         value = super(Container, self).validate(obj, value)
         if value is None:
             return value
 
         value = self.validate_elements(obj, value)
 
         return value
 
     def validate_elements(self, obj, value):
         validated = []
         if self._trait is None or isinstance(self._trait, Any):
             return value
         for v in value:
             try:
                 v = self._trait._validate(obj, v)
             except TraitError:
                 self.element_error(obj, v, self._trait)
             else:
                 validated.append(v)
         return self.klass(validated)
 
     def instance_init(self):
         if isinstance(self._trait, TraitType):
             self._trait.this_class = self.this_class
             self._trait.instance_init()
         super(Container, self).instance_init()
 
 
 class List(Container):
     """An instance of a Python list."""
     klass = list
     _cast_types = (tuple,)
 
     def __init__(self, trait=None, default_value=None, minlen=0, maxlen=sys.maxsize, **metadata):
         """Create a List trait type from a list, set, or tuple.
 
         The default value is created by doing ``List(default_value)``,
         which creates a copy of the ``default_value``.
 
         ``trait`` can be specified, which restricts the type of elements
         in the container to that TraitType.
 
         If only one arg is given and it is not a Trait, it is taken as
         ``default_value``:
 
         ``c = List([1,2,3])``
 
         Parameters
         ----------
 
         trait : TraitType [ optional ]
             the type for restricting the contents of the Container.  If unspecified,
             types are not checked.
 
         default_value : SequenceType [ optional ]
             The default value for the Trait.  Must be list/tuple/set, and
             will be cast to the container type.
 
         minlen : Int [ default 0 ]
             The minimum length of the input list
 
         maxlen : Int [ default sys.maxsize ]
             The maximum length of the input list
 
         allow_none : bool [ default False ]
             Whether to allow the value to be None
 
         **metadata : any
             further keys for extensions to the Trait (e.g. config)
 
         """
         self._minlen = minlen
         self._maxlen = maxlen
         super(List, self).__init__(trait=trait, default_value=default_value,
                                 **metadata)
 
     def length_error(self, obj, value):
         e = "The '%s' trait of %s instance must be of length %i <= L <= %i, but a value of %s was specified." \
             % (self.name, class_of(obj), self._minlen, self._maxlen, value)
         raise TraitError(e)
 
     def validate_elements(self, obj, value):
         length = len(value)
         if length < self._minlen or length > self._maxlen:
             self.length_error(obj, value)
 
         return super(List, self).validate_elements(obj, value)
 
     def validate(self, obj, value):
         value = super(List, self).validate(obj, value)
         value = self.validate_elements(obj, value)
         return value
 
 
 class Set(List):
     """An instance of a Python set."""
     klass = set
     _cast_types = (tuple, list)
 
 
 class Tuple(Container):
     """An instance of a Python tuple."""
     klass = tuple
     _cast_types = (list,)
 
     def __init__(self, *traits, **metadata):
         """Tuple(*traits, default_value=None, **medatata)
 
         Create a tuple from a list, set, or tuple.
 
         Create a fixed-type tuple with Traits:
 
         ``t = Tuple(Int, Str, CStr)``
 
         would be length 3, with Int,Str,CStr for each element.
 
         If only one arg is given and it is not a Trait, it is taken as
         default_value:
 
         ``t = Tuple((1,2,3))``
 
         Otherwise, ``default_value`` *must* be specified by keyword.
 
         Parameters
         ----------
 
         *traits : TraitTypes [ optional ]
             the types for restricting the contents of the Tuple.  If unspecified,
             types are not checked. If specified, then each positional argument
             corresponds to an element of the tuple.  Tuples defined with traits
             are of fixed length.
 
         default_value : SequenceType [ optional ]
             The default value for the Tuple.  Must be list/tuple/set, and
             will be cast to a tuple. If `traits` are specified, the
             `default_value` must conform to the shape and type they specify.
 
         allow_none : bool [ default False ]
             Whether to allow the value to be None
 
         **metadata : any
             further keys for extensions to the Trait (e.g. config)
 
         """
         default_value = metadata.pop('default_value', None)
         allow_none = metadata.pop('allow_none', True)
 
         # allow Tuple((values,)):
         if len(traits) == 1 and default_value is None and not is_trait(traits[0]):
             default_value = traits[0]
             traits = ()
 
         if default_value is None:
             args = ()
         elif isinstance(default_value, self._valid_defaults):
             args = (default_value,)
         else:
             raise TypeError('default value of %s was %s' %(self.__class__.__name__, default_value))
 
         self._traits = []
         for trait in traits:
             t = trait() if isinstance(trait, type) else trait
             t.name = 'element'
             self._traits.append(t)
 
         if self._traits and default_value is None:
             # don't allow default to be an empty container if length is specified
             args = None
         super(Container,self).__init__(klass=self.klass, args=args, allow_none=allow_none, **metadata)
 
     def validate_elements(self, obj, value):
         if not self._traits:
             # nothing to validate
             return value
         if len(value) != len(self._traits):
             e = "The '%s' trait of %s instance requires %i elements, but a value of %s was specified." \
                 % (self.name, class_of(obj), len(self._traits), repr_type(value))
             raise TraitError(e)
 
         validated = []
         for t, v in zip(self._traits, value):
             try:
                 v = t._validate(obj, v)
             except TraitError:
                 self.element_error(obj, v, t)
             else:
                 validated.append(v)
         return tuple(validated)
 
     def instance_init(self):
         for trait in self._traits:
             if isinstance(trait, TraitType):
                 trait.this_class = self.this_class
                 trait.instance_init()
         super(Container, self).instance_init()
 
 
 class Dict(Instance):
     """An instance of a Python dict."""
     _trait = None
 
     def __init__(self, trait=None, default_value=NoDefaultSpecified, allow_none=False, **metadata):
         """Create a dict trait type from a dict.
 
         The default value is created by doing ``dict(default_value)``,
         which creates a copy of the ``default_value``.
 
         trait : TraitType [ optional ]
             the type for restricting the contents of the Container.  If unspecified,
             types are not checked.
 
         default_value : SequenceType [ optional ]
             The default value for the Dict.  Must be dict, tuple, or None, and
             will be cast to a dict if not None. If `trait` is specified, the
             `default_value` must conform to the constraints it specifies.
 
         allow_none : bool [ default False ]
             Whether to allow the value to be None
 
         """
         if default_value is NoDefaultSpecified and trait is not None:
             if not is_trait(trait):
                 default_value = trait
                 trait = None
         if default_value is NoDefaultSpecified:
             default_value = {}
         if default_value is None:
             args = None
         elif isinstance(default_value, dict):
             args = (default_value,)
         elif isinstance(default_value, SequenceTypes):
             args = (default_value,)
         else:
             raise TypeError('default value of Dict was %s' % default_value)
 
         if is_trait(trait):
             self._trait = trait() if isinstance(trait, type) else trait
             self._trait.name = 'element'
         elif trait is not None:
             raise TypeError("`trait` must be a Trait or None, got %s"%repr_type(trait))
 
         super(Dict,self).__init__(klass=dict, args=args,
                                   allow_none=allow_none, **metadata)
 
     def element_error(self, obj, element, validator):
         e = "Element of the '%s' trait of %s instance must be %s, but a value of %s was specified." \
             % (self.name, class_of(obj), validator.info(), repr_type(element))
         raise TraitError(e)
 
     def validate(self, obj, value):
         value = super(Dict, self).validate(obj, value)
         if value is None:
             return value
         value = self.validate_elements(obj, value)
         return value
 
     def validate_elements(self, obj, value):
         if self._trait is None or isinstance(self._trait, Any):
             return value
         validated = {}
         for key in value:
             v = value[key]
             try:
                 v = self._trait._validate(obj, v)
             except TraitError:
                 self.element_error(obj, v, self._trait)
             else:
                 validated[key] = v
         return self.klass(validated)
 
     def instance_init(self):
         if isinstance(self._trait, TraitType):
             self._trait.this_class = self.this_class
             self._trait.instance_init()
         super(Dict, self).instance_init()
 
 
 class EventfulDict(Instance):
     """An instance of an EventfulDict."""
 
     def __init__(self, default_value={}, allow_none=False, **metadata):
         """Create a EventfulDict trait type from a dict.
 
         The default value is created by doing
         ``eventful.EvenfulDict(default_value)``, which creates a copy of the
         ``default_value``.
         """
         if default_value is None:
             args = None
         elif isinstance(default_value, dict):
             args = (default_value,)
         elif isinstance(default_value, SequenceTypes):
             args = (default_value,)
         else:
             raise TypeError('default value of EventfulDict was %s' % default_value)
 
         super(EventfulDict, self).__init__(klass=eventful.EventfulDict, args=args,
                                   allow_none=allow_none, **metadata)
 
 
 class EventfulList(Instance):
     """An instance of an EventfulList."""
 
     def __init__(self, default_value=None, allow_none=False, **metadata):
         """Create a EventfulList trait type from a dict.
 
         The default value is created by doing 
         ``eventful.EvenfulList(default_value)``, which creates a copy of the 
         ``default_value``.
         """
         if default_value is None:
             args = ((),)
         else:
             args = (default_value,)
 
         super(EventfulList, self).__init__(klass=eventful.EventfulList, args=args,
                                   allow_none=allow_none, **metadata)
 
 
 class TCPAddress(TraitType):
     """A trait for an (ip, port) tuple.
 
     This allows for both IPv4 IP addresses as well as hostnames.
     """
 
     default_value = ('127.0.0.1', 0)
     info_text = 'an (ip, port) tuple'
 
     def validate(self, obj, value):
         if isinstance(value, tuple):
             if len(value) == 2:
                 if isinstance(value[0], py3compat.string_types) and isinstance(value[1], int):
                     port = value[1]
                     if port >= 0 and port <= 65535:
                         return value
         self.error(obj, value)
 
 class CRegExp(TraitType):
     """A casting compiled regular expression trait.
 
     Accepts both strings and compiled regular expressions. The resulting
     attribute will be a compiled regular expression."""
 
     info_text = 'a regular expression'
 
     def validate(self, obj, value):
         try:
             return re.compile(value)
         except:
             self.error(obj, value)