Fix tests in utils

IPython/testing/plugin/ipdoctest.py

@@ -1,764 +1,764
 """Nose Plugin that supports IPython doctests.
 
 Limitations:
 
 - When generating examples for use as doctests, make sure that you have
   pretty-printing OFF.  This can be done either by setting the
   ``PlainTextFormatter.pprint`` option in your configuration file to  False, or
   by interactively disabling it with  %Pprint.  This is required so that IPython
   output matches that of normal Python, which is used by doctest for internal
   execution.
 
 - Do not rely on specific prompt numbers for results (such as using
   '_34==True', for example).  For IPython tests run via an external process the
   prompt numbers may be different, and IPython tests run as normal python code
   won't even have these special _NN variables set at all.
 """
 
 #-----------------------------------------------------------------------------
 # Module imports
 
 # From the standard library
 import doctest
 import inspect
 import logging
 import os
 import re
 import sys
 import traceback
 import unittest
 
 from inspect import getmodule
 
 # We are overriding the default doctest runner, so we need to import a few
 # things from doctest directly
 from doctest import (REPORTING_FLAGS, REPORT_ONLY_FIRST_FAILURE,
                      _unittest_reportflags, DocTestRunner,
                      _extract_future_flags, pdb, _OutputRedirectingPdb,
                      _exception_traceback,
                      linecache)
 
 # Third-party modules
 import nose.core
 
 from nose.plugins import doctests, Plugin
 from nose.util import anyp, getpackage, test_address, resolve_name, tolist
 
 # Our own imports
 from IPython.utils.py3compat import builtin_mod, PY3
 
 if PY3:
     from io import StringIO
 else:
     from StringIO import StringIO
 
 #-----------------------------------------------------------------------------
 # Module globals and other constants
 #-----------------------------------------------------------------------------
 
 log = logging.getLogger(__name__)
 
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 def is_extension_module(filename):
     """Return whether the given filename is an extension module.
 
     This simply checks that the extension is either .so or .pyd.
     """
     return os.path.splitext(filename)[1].lower() in ('.so','.pyd')
 
 
 class DocTestSkip(object):
     """Object wrapper for doctests to be skipped."""
 
     ds_skip = """Doctest to skip.
     >>> 1 #doctest: +SKIP
     """
 
     def __init__(self,obj):
         self.obj = obj
 
     def __getattribute__(self,key):
         if key == '__doc__':
             return DocTestSkip.ds_skip
         else:
             return getattr(object.__getattribute__(self,'obj'),key)
 
 # Modified version of the one in the stdlib, that fixes a python bug (doctests
 # not found in extension modules, http://bugs.python.org/issue3158)
 class DocTestFinder(doctest.DocTestFinder):
 
     def _from_module(self, module, object):
         """
         Return true if the given object is defined in the given
         module.
         """
         if module is None:
             return True
         elif inspect.isfunction(object):
             return module.__dict__ is object.__globals__
         elif inspect.isbuiltin(object):
             return module.__name__ == object.__module__
         elif inspect.isclass(object):
             return module.__name__ == object.__module__
         elif inspect.ismethod(object):
             # This one may be a bug in cython that fails to correctly set the
             # __module__ attribute of methods, but since the same error is easy
             # to make by extension code writers, having this safety in place
             # isn't such a bad idea
             return module.__name__ == object.__self__.__class__.__module__
         elif inspect.getmodule(object) is not None:
             return module is inspect.getmodule(object)
         elif hasattr(object, '__module__'):
             return module.__name__ == object.__module__
         elif isinstance(object, property):
             return True # [XX] no way not be sure.
         else:
-            raise ValueError("object must be a class or function")
+            raise ValueError("object must be a class or function, got %r" % object)
 
     def _find(self, tests, obj, name, module, source_lines, globs, seen):
         """
         Find tests for the given object and any contained objects, and
         add them to `tests`.
         """
         #print '_find for:', obj, name, module  # dbg
         if hasattr(obj,"skip_doctest"):
             #print 'SKIPPING DOCTEST FOR:',obj  # dbg
             obj = DocTestSkip(obj)
 
         doctest.DocTestFinder._find(self,tests, obj, name, module,
                                     source_lines, globs, seen)
 
         # Below we re-run pieces of the above method with manual modifications,
         # because the original code is buggy and fails to correctly identify
         # doctests in extension modules.
 
         # Local shorthands
         from inspect import isroutine, isclass, ismodule
 
         # Look for tests in a module's contained objects.
         if inspect.ismodule(obj) and self._recurse:
             for valname, val in obj.__dict__.items():
                 valname1 = '%s.%s' % (name, valname)
                 if ( (isroutine(val) or isclass(val))
                      and self._from_module(module, val) ):
 
                     self._find(tests, val, valname1, module, source_lines,
                                globs, seen)
 
         # Look for tests in a class's contained objects.
         if inspect.isclass(obj) and self._recurse:
             #print 'RECURSE into class:',obj  # dbg
             for valname, val in obj.__dict__.items():
                 # Special handling for staticmethod/classmethod.
                 if isinstance(val, staticmethod):
                     val = getattr(obj, valname)
                 if isinstance(val, classmethod):
                     val = getattr(obj, valname).__func__
 
                 # Recurse to methods, properties, and nested classes.
                 if ((inspect.isfunction(val) or inspect.isclass(val) or
                      inspect.ismethod(val) or
                       isinstance(val, property)) and
                       self._from_module(module, val)):
                     valname = '%s.%s' % (name, valname)
                     self._find(tests, val, valname, module, source_lines,
                                globs, seen)
 
 
 class IPDoctestOutputChecker(doctest.OutputChecker):
     """Second-chance checker with support for random tests.
 
     If the default comparison doesn't pass, this checker looks in the expected
     output string for flags that tell us to ignore the output.
     """
 
     random_re = re.compile(r'#\s*random\s+')
 
     def check_output(self, want, got, optionflags):
         """Check output, accepting special markers embedded in the output.
 
         If the output didn't pass the default validation but the special string
         '#random' is included, we accept it."""
 
         # Let the original tester verify first, in case people have valid tests
         # that happen to have a comment saying '#random' embedded in.
         ret = doctest.OutputChecker.check_output(self, want, got,
                                                  optionflags)
         if not ret and self.random_re.search(want):
             #print >> sys.stderr, 'RANDOM OK:',want  # dbg
             return True
 
         return ret
 
 
 class DocTestCase(doctests.DocTestCase):
     """Proxy for DocTestCase: provides an address() method that
     returns the correct address for the doctest case. Otherwise
     acts as a proxy to the test case. To provide hints for address(),
     an obj may also be passed -- this will be used as the test object
     for purposes of determining the test address, if it is provided.
     """
 
     # Note: this method was taken from numpy's nosetester module.
 
     # Subclass nose.plugins.doctests.DocTestCase to work around a bug in
     # its constructor that blocks non-default arguments from being passed
     # down into doctest.DocTestCase
 
     def __init__(self, test, optionflags=0, setUp=None, tearDown=None,
                  checker=None, obj=None, result_var='_'):
         self._result_var = result_var
         doctests.DocTestCase.__init__(self, test,
                                       optionflags=optionflags,
                                       setUp=setUp, tearDown=tearDown,
                                       checker=checker)
         # Now we must actually copy the original constructor from the stdlib
         # doctest class, because we can't call it directly and a bug in nose
         # means it never gets passed the right arguments.
 
         self._dt_optionflags = optionflags
         self._dt_checker = checker
         self._dt_test = test
         self._dt_test_globs_ori = test.globs
         self._dt_setUp = setUp
         self._dt_tearDown = tearDown
 
         # XXX - store this runner once in the object!
         runner = IPDocTestRunner(optionflags=optionflags,
                                  checker=checker, verbose=False)
         self._dt_runner = runner
 
 
         # Each doctest should remember the directory it was loaded from, so
         # things like %run work without too many contortions
         self._ori_dir = os.path.dirname(test.filename)
 
     # Modified runTest from the default stdlib
     def runTest(self):
         test = self._dt_test
         runner = self._dt_runner
 
         old = sys.stdout
         new = StringIO()
         optionflags = self._dt_optionflags
 
         if not (optionflags & REPORTING_FLAGS):
             # The option flags don't include any reporting flags,
             # so add the default reporting flags
             optionflags |= _unittest_reportflags
 
         try:
             # Save our current directory and switch out to the one where the
             # test was originally created, in case another doctest did a
             # directory change.  We'll restore this in the finally clause.
             curdir = os.getcwdu()
             #print 'runTest in dir:', self._ori_dir  # dbg
             os.chdir(self._ori_dir)
 
             runner.DIVIDER = "-"*70
             failures, tries = runner.run(test,out=new.write,
                                          clear_globs=False)
         finally:
             sys.stdout = old
             os.chdir(curdir)
 
         if failures:
             raise self.failureException(self.format_failure(new.getvalue()))
 
     def setUp(self):
         """Modified test setup that syncs with ipython namespace"""
         #print "setUp test", self._dt_test.examples # dbg
         if isinstance(self._dt_test.examples[0], IPExample):
             # for IPython examples *only*, we swap the globals with the ipython
             # namespace, after updating it with the globals (which doctest
             # fills with the necessary info from the module being tested).
             self.user_ns_orig = {}
             self.user_ns_orig.update(_ip.user_ns)
             _ip.user_ns.update(self._dt_test.globs)
             # We must remove the _ key in the namespace, so that Python's
             # doctest code sets it naturally
             _ip.user_ns.pop('_', None)
             _ip.user_ns['__builtins__'] = builtin_mod
             self._dt_test.globs = _ip.user_ns
 
         super(DocTestCase, self).setUp()
 
     def tearDown(self):
 
         # Undo the test.globs reassignment we made, so that the parent class
         # teardown doesn't destroy the ipython namespace
         if isinstance(self._dt_test.examples[0], IPExample):
             self._dt_test.globs = self._dt_test_globs_ori
             _ip.user_ns.clear()
             _ip.user_ns.update(self.user_ns_orig)
 
         # XXX - fperez: I am not sure if this is truly a bug in nose 0.11, but
         # it does look like one to me: its tearDown method tries to run
         #
         # delattr(builtin_mod, self._result_var)
         #
         # without checking that the attribute really is there; it implicitly
         # assumes it should have been set via displayhook.  But if the
         # displayhook was never called, this doesn't necessarily happen.  I
         # haven't been able to find a little self-contained example outside of
         # ipython that would show the problem so I can report it to the nose
         # team, but it does happen a lot in our code.
         #
         # So here, we just protect as narrowly as possible by trapping an
         # attribute error whose message would be the name of self._result_var,
         # and letting any other error propagate.
         try:
             super(DocTestCase, self).tearDown()
         except AttributeError as exc:
             if exc.args[0] != self._result_var:
                 raise
 
 
 # A simple subclassing of the original with a different class name, so we can
 # distinguish and treat differently IPython examples from pure python ones.
 class IPExample(doctest.Example): pass
 
 
 class IPExternalExample(doctest.Example):
     """Doctest examples to be run in an external process."""
 
     def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
                  options=None):
         # Parent constructor
         doctest.Example.__init__(self,source,want,exc_msg,lineno,indent,options)
 
         # An EXTRA newline is needed to prevent pexpect hangs
         self.source += '\n'
 
 
 class IPDocTestParser(doctest.DocTestParser):
     """
     A class used to parse strings containing doctest examples.
 
     Note: This is a version modified to properly recognize IPython input and
     convert any IPython examples into valid Python ones.
     """
     # This regular expression is used to find doctest examples in a
     # string.  It defines three groups: `source` is the source code
     # (including leading indentation and prompts); `indent` is the
     # indentation of the first (PS1) line of the source code; and
     # `want` is the expected output (including leading indentation).
 
     # Classic Python prompts or default IPython ones
     _PS1_PY = r'>>>'
     _PS2_PY = r'\.\.\.'
 
     _PS1_IP = r'In\ \[\d+\]:'
     _PS2_IP = r'\ \ \ \.\.\.+:'
 
     _RE_TPL = r'''
         # Source consists of a PS1 line followed by zero or more PS2 lines.
         (?P<source>
             (?:^(?P<indent> [ ]*) (?P<ps1> %s) .*)    # PS1 line
             (?:\n           [ ]*  (?P<ps2> %s) .*)*)  # PS2 lines
         \n? # a newline
         # Want consists of any non-blank lines that do not start with PS1.
         (?P<want> (?:(?![ ]*$)    # Not a blank line
                      (?![ ]*%s)   # Not a line starting with PS1
                      (?![ ]*%s)   # Not a line starting with PS2
                      .*$\n?       # But any other line
                   )*)
                   '''
 
     _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY),
                                  re.MULTILINE | re.VERBOSE)
 
     _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP),
                                  re.MULTILINE | re.VERBOSE)
 
     # Mark a test as being fully random.  In this case, we simply append the
     # random marker ('#random') to each individual example's output.  This way
     # we don't need to modify any other code.
     _RANDOM_TEST = re.compile(r'#\s*all-random\s+')
 
     # Mark tests to be executed in an external process - currently unsupported.
     _EXTERNAL_IP = re.compile(r'#\s*ipdoctest:\s*EXTERNAL')
 
     def ip2py(self,source):
         """Convert input IPython source into valid Python."""
         block = _ip.input_transformer_manager.transform_cell(source)
         if len(block.splitlines()) == 1:
             return _ip.prefilter(block)
         else:
             return block
 
     def parse(self, string, name='<string>'):
         """
         Divide the given string into examples and intervening text,
         and return them as a list of alternating Examples and strings.
         Line numbers for the Examples are 0-based.  The optional
         argument `name` is a name identifying this string, and is only
         used for error messages.
         """
 
         #print 'Parse string:\n',string # dbg
 
         string = string.expandtabs()
         # If all lines begin with the same indentation, then strip it.
         min_indent = self._min_indent(string)
         if min_indent > 0:
             string = '\n'.join([l[min_indent:] for l in string.split('\n')])
 
         output = []
         charno, lineno = 0, 0
 
         # We make 'all random' tests by adding the '# random' mark to every
         # block of output in the test.
         if self._RANDOM_TEST.search(string):
             random_marker = '\n# random'
         else:
             random_marker = ''
 
         # Whether to convert the input from ipython to python syntax
         ip2py = False
         # Find all doctest examples in the string.  First, try them as Python
         # examples, then as IPython ones
         terms = list(self._EXAMPLE_RE_PY.finditer(string))
         if terms:
             # Normal Python example
             #print '-'*70  # dbg
             #print 'PyExample, Source:\n',string  # dbg
             #print '-'*70  # dbg
             Example = doctest.Example
         else:
             # It's an ipython example.  Note that IPExamples are run
             # in-process, so their syntax must be turned into valid python.
             # IPExternalExamples are run out-of-process (via pexpect) so they
             # don't need any filtering (a real ipython will be executing them).
             terms = list(self._EXAMPLE_RE_IP.finditer(string))
             if self._EXTERNAL_IP.search(string):
                 #print '-'*70  # dbg
                 #print 'IPExternalExample, Source:\n',string  # dbg
                 #print '-'*70  # dbg
                 Example = IPExternalExample
             else:
                 #print '-'*70  # dbg
                 #print 'IPExample, Source:\n',string  # dbg
                 #print '-'*70  # dbg
                 Example = IPExample
                 ip2py = True
 
         for m in terms:
             # Add the pre-example text to `output`.
             output.append(string[charno:m.start()])
             # Update lineno (lines before this example)
             lineno += string.count('\n', charno, m.start())
             # Extract info from the regexp match.
             (source, options, want, exc_msg) = \
                      self._parse_example(m, name, lineno,ip2py)
 
             # Append the random-output marker (it defaults to empty in most
             # cases, it's only non-empty for 'all-random' tests):
             want += random_marker
 
             if Example is IPExternalExample:
                 options[doctest.NORMALIZE_WHITESPACE] = True
                 want += '\n'
 
             # Create an Example, and add it to the list.
             if not self._IS_BLANK_OR_COMMENT(source):
                 output.append(Example(source, want, exc_msg,
                                       lineno=lineno,
                                       indent=min_indent+len(m.group('indent')),
                                       options=options))
             # Update lineno (lines inside this example)
             lineno += string.count('\n', m.start(), m.end())
             # Update charno.
             charno = m.end()
         # Add any remaining post-example text to `output`.
         output.append(string[charno:])
         return output
 
     def _parse_example(self, m, name, lineno,ip2py=False):
         """
         Given a regular expression match from `_EXAMPLE_RE` (`m`),
         return a pair `(source, want)`, where `source` is the matched
         example's source code (with prompts and indentation stripped);
         and `want` is the example's expected output (with indentation
         stripped).
 
         `name` is the string's name, and `lineno` is the line number
         where the example starts; both are used for error messages.
 
         Optional:
         `ip2py`: if true, filter the input via IPython to convert the syntax
         into valid python.
         """
 
         # Get the example's indentation level.
         indent = len(m.group('indent'))
 
         # Divide source into lines; check that they're properly
         # indented; and then strip their indentation & prompts.
         source_lines = m.group('source').split('\n')
 
         # We're using variable-length input prompts
         ps1 = m.group('ps1')
         ps2 = m.group('ps2')
         ps1_len = len(ps1)
 
         self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len)
         if ps2:
             self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno)
 
         source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines])
 
         if ip2py:
             # Convert source input from IPython into valid Python syntax
             source = self.ip2py(source)
 
         # Divide want into lines; check that it's properly indented; and
         # then strip the indentation.  Spaces before the last newline should
         # be preserved, so plain rstrip() isn't good enough.
         want = m.group('want')
         want_lines = want.split('\n')
         if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
             del want_lines[-1]  # forget final newline & spaces after it
         self._check_prefix(want_lines, ' '*indent, name,
                            lineno + len(source_lines))
 
         # Remove ipython output prompt that might be present in the first line
         want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0])
 
         want = '\n'.join([wl[indent:] for wl in want_lines])
 
         # If `want` contains a traceback message, then extract it.
         m = self._EXCEPTION_RE.match(want)
         if m:
             exc_msg = m.group('msg')
         else:
             exc_msg = None
 
         # Extract options from the source.
         options = self._find_options(source, name, lineno)
 
         return source, options, want, exc_msg
 
     def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len):
         """
         Given the lines of a source string (including prompts and
         leading indentation), check to make sure that every prompt is
         followed by a space character.  If any line is not followed by
         a space character, then raise ValueError.
 
         Note: IPython-modified version which takes the input prompt length as a
         parameter, so that prompts of variable length can be dealt with.
         """
         space_idx = indent+ps1_len
         min_len = space_idx+1
         for i, line in enumerate(lines):
             if len(line) >=  min_len and line[space_idx] != ' ':
                 raise ValueError('line %r of the docstring for %s '
                                  'lacks blank after %s: %r' %
                                  (lineno+i+1, name,
                                   line[indent:space_idx], line))
 
 
 SKIP = doctest.register_optionflag('SKIP')
 
 
 class IPDocTestRunner(doctest.DocTestRunner,object):
     """Test runner that synchronizes the IPython namespace with test globals.
     """
 
     def run(self, test, compileflags=None, out=None, clear_globs=True):
 
         # Hack: ipython needs access to the execution context of the example,
         # so that it can propagate user variables loaded by %run into
         # test.globs.  We put them here into our modified %run as a function
         # attribute.  Our new %run will then only make the namespace update
         # when called (rather than unconconditionally updating test.globs here
         # for all examples, most of which won't be calling %run anyway).
         #_ip._ipdoctest_test_globs = test.globs
         #_ip._ipdoctest_test_filename = test.filename
 
         test.globs.update(_ip.user_ns)
 
         return super(IPDocTestRunner,self).run(test,
                                                compileflags,out,clear_globs)
 
 
 class DocFileCase(doctest.DocFileCase):
     """Overrides to provide filename
     """
     def address(self):
         return (self._dt_test.filename, None, None)
 
 
 class ExtensionDoctest(doctests.Doctest):
     """Nose Plugin that supports doctests in extension modules.
     """
     name = 'extdoctest'   # call nosetests with --with-extdoctest
     enabled = True
 
     def options(self, parser, env=os.environ):
         Plugin.options(self, parser, env)
         parser.add_option('--doctest-tests', action='store_true',
                           dest='doctest_tests',
                           default=env.get('NOSE_DOCTEST_TESTS',True),
                           help="Also look for doctests in test modules. "
                           "Note that classes, methods and functions should "
                           "have either doctests or non-doctest tests, "
                           "not both. [NOSE_DOCTEST_TESTS]")
         parser.add_option('--doctest-extension', action="append",
                           dest="doctestExtension",
                           help="Also look for doctests in files with "
                           "this extension [NOSE_DOCTEST_EXTENSION]")
         # Set the default as a list, if given in env; otherwise
         # an additional value set on the command line will cause
         # an error.
         env_setting = env.get('NOSE_DOCTEST_EXTENSION')
         if env_setting is not None:
             parser.set_defaults(doctestExtension=tolist(env_setting))
 
 
     def configure(self, options, config):
         Plugin.configure(self, options, config)
         # Pull standard doctest plugin out of config; we will do doctesting
         config.plugins.plugins = [p for p in config.plugins.plugins
                                   if p.name != 'doctest']
         self.doctest_tests = options.doctest_tests
         self.extension = tolist(options.doctestExtension)
 
         self.parser = doctest.DocTestParser()
         self.finder = DocTestFinder()
         self.checker = IPDoctestOutputChecker()
         self.globs = None
         self.extraglobs = None
 
 
     def loadTestsFromExtensionModule(self,filename):
         bpath,mod = os.path.split(filename)
         modname = os.path.splitext(mod)[0]
         try:
             sys.path.append(bpath)
             module = __import__(modname)
             tests = list(self.loadTestsFromModule(module))
         finally:
             sys.path.pop()
         return tests
 
     # NOTE: the method below is almost a copy of the original one in nose, with
     # a  few modifications to control output checking.
 
     def loadTestsFromModule(self, module):
         #print '*** ipdoctest - lTM',module  # dbg
 
         if not self.matches(module.__name__):
             log.debug("Doctest doesn't want module %s", module)
             return
 
         tests = self.finder.find(module,globs=self.globs,
                                  extraglobs=self.extraglobs)
         if not tests:
             return
 
         # always use whitespace and ellipsis options
         optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
 
         tests.sort()
         module_file = module.__file__
         if module_file[-4:] in ('.pyc', '.pyo'):
             module_file = module_file[:-1]
         for test in tests:
             if not test.examples:
                 continue
             if not test.filename:
                 test.filename = module_file
 
             yield DocTestCase(test,
                               optionflags=optionflags,
                               checker=self.checker)
 
 
     def loadTestsFromFile(self, filename):
         #print "ipdoctest - from file", filename # dbg
         if is_extension_module(filename):
             for t in self.loadTestsFromExtensionModule(filename):
                 yield t
         else:
             if self.extension and anyp(filename.endswith, self.extension):
                 name = os.path.basename(filename)
                 dh = open(filename)
                 try:
                     doc = dh.read()
                 finally:
                     dh.close()
                 test = self.parser.get_doctest(
                     doc, globs={'__file__': filename}, name=name,
                     filename=filename, lineno=0)
                 if test.examples:
                     #print 'FileCase:',test.examples  # dbg
                     yield DocFileCase(test)
                 else:
                     yield False # no tests to load
 
 
 class IPythonDoctest(ExtensionDoctest):
     """Nose Plugin that supports doctests in extension modules.
     """
     name = 'ipdoctest'   # call nosetests with --with-ipdoctest
     enabled = True
 
     def makeTest(self, obj, parent):
         """Look for doctests in the given object, which will be a
         function, method or class.
         """
         #print 'Plugin analyzing:', obj, parent  # dbg
         # always use whitespace and ellipsis options
         optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
 
         doctests = self.finder.find(obj, module=getmodule(parent))
         if doctests:
             for test in doctests:
                 if len(test.examples) == 0:
                     continue
 
                 yield DocTestCase(test, obj=obj,
                                   optionflags=optionflags,
                                   checker=self.checker)
 
     def options(self, parser, env=os.environ):
         #print "Options for nose plugin:", self.name # dbg
         Plugin.options(self, parser, env)
         parser.add_option('--ipdoctest-tests', action='store_true',
                           dest='ipdoctest_tests',
                           default=env.get('NOSE_IPDOCTEST_TESTS',True),
                           help="Also look for doctests in test modules. "
                           "Note that classes, methods and functions should "
                           "have either doctests or non-doctest tests, "
                           "not both. [NOSE_IPDOCTEST_TESTS]")
         parser.add_option('--ipdoctest-extension', action="append",
                           dest="ipdoctest_extension",
                           help="Also look for doctests in files with "
                           "this extension [NOSE_IPDOCTEST_EXTENSION]")
         # Set the default as a list, if given in env; otherwise
         # an additional value set on the command line will cause
         # an error.
         env_setting = env.get('NOSE_IPDOCTEST_EXTENSION')
         if env_setting is not None:
             parser.set_defaults(ipdoctest_extension=tolist(env_setting))
 
     def configure(self, options, config):
         #print "Configuring nose plugin:", self.name # dbg
         Plugin.configure(self, options, config)
         # Pull standard doctest plugin out of config; we will do doctesting
         config.plugins.plugins = [p for p in config.plugins.plugins
                                   if p.name != 'doctest']
         self.doctest_tests = options.ipdoctest_tests
         self.extension = tolist(options.ipdoctest_extension)
 
         self.parser = IPDocTestParser()
         self.finder = DocTestFinder(parser=self.parser)
         self.checker = IPDoctestOutputChecker()
         self.globs = None
         self.extraglobs = None

IPython/utils/frame.py

@@ -1,95 +1,95
 # encoding: utf-8
 """
 Utilities for working with stack frames.
 """
 from __future__ import print_function
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import sys
 from IPython.utils import py3compat
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 @py3compat.doctest_refactor_print
 def extract_vars(*names,**kw):
     """Extract a set of variables by name from another frame.
 
     :Parameters:
       - `*names`: strings
         One or more variable names which will be extracted from the caller's
     frame.
 
     :Keywords:
       - `depth`: integer (0)
         How many frames in the stack to walk when looking for your variables.
 
 
     Examples:
 
         In [2]: def func(x):
            ...:     y = 1
-           ...:     print sorted(extract_vars('x','y').items())
+           ...:     print(sorted(extract_vars('x','y').items()))
            ...:
 
         In [3]: func('hello')
         [('x', 'hello'), ('y', 1)]
     """
 
     depth = kw.get('depth',0)
     
     callerNS = sys._getframe(depth+1).f_locals
     return dict((k,callerNS[k]) for k in names)
 
 
 def extract_vars_above(*names):
     """Extract a set of variables by name from another frame.
 
     Similar to extractVars(), but with a specified depth of 1, so that names
     are exctracted exactly from above the caller.
 
     This is simply a convenience function so that the very common case (for us)
     of skipping exactly 1 frame doesn't have to construct a special dict for
     keyword passing."""
 
     callerNS = sys._getframe(2).f_locals
     return dict((k,callerNS[k]) for k in names)
 
 
 def debugx(expr,pre_msg=''):
     """Print the value of an expression from the caller's frame.
 
     Takes an expression, evaluates it in the caller's frame and prints both
     the given expression and the resulting value (as well as a debug mark
     indicating the name of the calling function.  The input must be of a form
     suitable for eval().
 
     An optional message can be passed, which will be prepended to the printed
     expr->value pair."""
 
     cf = sys._getframe(1)
     print('[DBG:%s] %s%s -> %r' % (cf.f_code.co_name,pre_msg,expr,
                                    eval(expr,cf.f_globals,cf.f_locals)))
 
 
 # deactivate it by uncommenting the following line, which makes it a no-op
 #def debugx(expr,pre_msg=''): pass
 
 def extract_module_locals(depth=0):
     """Returns (module, locals) of the funciton `depth` frames away from the caller"""
     f = sys._getframe(depth + 1)
     global_ns = f.f_globals
     module = sys.modules[global_ns['__name__']]
     return (module, f.f_locals)
 

IPython/utils/jsonutil.py

@@ -1,229 +1,229
 """Utilities to manipulate JSON objects.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING.txt, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 # stdlib
 import math
 import re
 import types
 from datetime import datetime
 
 try:
     # base64.encodestring is deprecated in Python 3.x
     from base64 import encodebytes
 except ImportError:
     # Python 2.x
     from base64 import encodestring as encodebytes
 
 from IPython.utils import py3compat
 from IPython.utils.py3compat import string_types, unicode_type, iteritems
 from IPython.utils.encoding import DEFAULT_ENCODING
 next_attr_name = '__next__' if py3compat.PY3 else 'next'
 
 #-----------------------------------------------------------------------------
 # Globals and constants
 #-----------------------------------------------------------------------------
 
 # timestamp formats
 ISO8601="%Y-%m-%dT%H:%M:%S.%f"
 ISO8601_PAT=re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+)Z?([\+\-]\d{2}:?\d{2})?$")
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 def rekey(dikt):
     """Rekey a dict that has been forced to use str keys where there should be
     ints by json."""
     for k in dikt:
         if isinstance(k, string_types):
             ik=fk=None
             try:
                 ik = int(k)
             except ValueError:
                 try:
                     fk = float(k)
                 except ValueError:
                     continue
             if ik is not None:
                 nk = ik
             else:
                 nk = fk
             if nk in dikt:
                 raise KeyError("already have key %r"%nk)
             dikt[nk] = dikt.pop(k)
     return dikt
 
 
 def extract_dates(obj):
     """extract ISO8601 dates from unpacked JSON"""
     if isinstance(obj, dict):
         obj = dict(obj) # don't clobber
         for k,v in iteritems(obj):
             obj[k] = extract_dates(v)
     elif isinstance(obj, (list, tuple)):
         obj = [ extract_dates(o) for o in obj ]
     elif isinstance(obj, string_types):
         m = ISO8601_PAT.match(obj)
         if m:
             # FIXME: add actual timezone support
             # this just drops the timezone info
             notz = m.groups()[0]
             obj = datetime.strptime(notz, ISO8601)
     return obj
 
 def squash_dates(obj):
     """squash datetime objects into ISO8601 strings"""
     if isinstance(obj, dict):
         obj = dict(obj) # don't clobber
         for k,v in iteritems(obj):
             obj[k] = squash_dates(v)
     elif isinstance(obj, (list, tuple)):
         obj = [ squash_dates(o) for o in obj ]
     elif isinstance(obj, datetime):
         obj = obj.isoformat()
     return obj
 
 def date_default(obj):
     """default function for packing datetime objects in JSON."""
     if isinstance(obj, datetime):
         return obj.isoformat()
     else:
         raise TypeError("%r is not JSON serializable"%obj)
 
 
 # constants for identifying png/jpeg data
 PNG = b'\x89PNG\r\n\x1a\n'
 # front of PNG base64-encoded
 PNG64 = b'iVBORw0KG'
 JPEG = b'\xff\xd8'
 # front of JPEG base64-encoded
 JPEG64 = b'/9'
 
 def encode_images(format_dict):
     """b64-encodes images in a displaypub format dict
 
     Perhaps this should be handled in json_clean itself?
 
     Parameters
     ----------
 
     format_dict : dict
         A dictionary of display data keyed by mime-type
 
     Returns
     -------
 
     format_dict : dict
         A copy of the same dictionary,
         but binary image data ('image/png' or 'image/jpeg')
         is base64-encoded.
 
     """
     encoded = format_dict.copy()
 
     pngdata = format_dict.get('image/png')
     if isinstance(pngdata, bytes):
         # make sure we don't double-encode
         if not pngdata.startswith(PNG64):
             pngdata = encodebytes(pngdata)
         encoded['image/png'] = pngdata.decode('ascii')
 
     jpegdata = format_dict.get('image/jpeg')
     if isinstance(jpegdata, bytes):
         # make sure we don't double-encode
         if not jpegdata.startswith(JPEG64):
             jpegdata = encodebytes(jpegdata)
         encoded['image/jpeg'] = jpegdata.decode('ascii')
 
     return encoded
 
 
 def json_clean(obj):
     """Clean an object to ensure it's safe to encode in JSON.
 
     Atomic, immutable objects are returned unmodified.  Sets and tuples are
     converted to lists, lists are copied and dicts are also copied.
 
     Note: dicts whose keys could cause collisions upon encoding (such as a dict
     with both the number 1 and the string '1' as keys) will cause a ValueError
     to be raised.
 
     Parameters
     ----------
     obj : any python object
 
     Returns
     -------
     out : object
 
       A version of the input which will not cause an encoding error when
       encoded as JSON.  Note that this function does not *encode* its inputs,
       it simply sanitizes it so that there will be no encoding errors later.
 
     Examples
     --------
     >>> json_clean(4)
     4
-    >>> json_clean(range(10))
+    >>> json_clean(list(range(10)))
     [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
     >>> sorted(json_clean(dict(x=1, y=2)).items())
     [('x', 1), ('y', 2)]
     >>> sorted(json_clean(dict(x=1, y=2, z=[1,2,3])).items())
     [('x', 1), ('y', 2), ('z', [1, 2, 3])]
     >>> json_clean(True)
     True
     """
     # types that are 'atomic' and ok in json as-is.  bool doesn't need to be
     # listed explicitly because bools pass as int instances
     atomic_ok = (unicode_type, int, type(None))
 
     # containers that we need to convert into lists
     container_to_list = (tuple, set, types.GeneratorType)
 
     if isinstance(obj, float):
         # cast out-of-range floats to their reprs
         if math.isnan(obj) or math.isinf(obj):
             return repr(obj)
         return obj
 
     if isinstance(obj, atomic_ok):
         return obj
 
     if isinstance(obj, bytes):
         return obj.decode(DEFAULT_ENCODING, 'replace')
 
     if isinstance(obj, container_to_list) or (
         hasattr(obj, '__iter__') and hasattr(obj, next_attr_name)):
         obj = list(obj)
 
     if isinstance(obj, list):
         return [json_clean(x) for x in obj]
 
     if isinstance(obj, dict):
         # First, validate that the dict won't lose data in conversion due to
         # key collisions after stringification.  This can happen with keys like
         # True and 'true' or 1 and '1', which collide in JSON.
         nkeys = len(obj)
         nkeys_collapsed = len(set(map(str, obj)))
         if nkeys != nkeys_collapsed:
             raise ValueError('dict can not be safely converted to JSON: '
                              'key collision would lead to dropped values')
         # If all OK, proceed by making the new dict that will be json-safe
         out = {}
         for k,v in iteritems(obj):
             out[str(k)] = json_clean(v)
         return out
 
     # If we get here, we don't know how to handle the object, so we just get
     # its repr and return that.  This will catch lambdas, open sockets, class
     # objects, and any other complicated contraption that json can't encode
     return repr(obj)

IPython/utils/py3compat.py

@@ -1,239 +1,239
 # coding: utf-8
 """Compatibility tricks for Python 3. Mainly to do with unicode."""
 import functools
 import sys
 import re
 import types
 
 from .encoding import DEFAULT_ENCODING
 
 orig_open = open
 
 def no_code(x, encoding=None):
     return x
 
 def decode(s, encoding=None):
     encoding = encoding or DEFAULT_ENCODING
     return s.decode(encoding, "replace")
 
 def encode(u, encoding=None):
     encoding = encoding or DEFAULT_ENCODING
     return u.encode(encoding, "replace")
 
 
 def cast_unicode(s, encoding=None):
     if isinstance(s, bytes):
         return decode(s, encoding)
     return s
 
 def cast_bytes(s, encoding=None):
     if not isinstance(s, bytes):
         return encode(s, encoding)
     return s
 
 def _modify_str_or_docstring(str_change_func):
     @functools.wraps(str_change_func)
     def wrapper(func_or_str):
         if isinstance(func_or_str, string_types):
             func = None
             doc = func_or_str
         else:
             func = func_or_str
             doc = func.__doc__
         
         doc = str_change_func(doc)
         
         if func:
             func.__doc__ = doc
             return func
         return doc
     return wrapper
 
 def safe_unicode(e):
     """unicode(e) with various fallbacks. Used for exceptions, which may not be
     safe to call unicode() on.
     """
     try:
         return unicode_type(e)
     except UnicodeError:
         pass
 
     try:
         return str_to_unicode(str(e))
     except UnicodeError:
         pass
 
     try:
         return str_to_unicode(repr(e))
     except UnicodeError:
         pass
 
     return u'Unrecoverably corrupt evalue'
 
 if sys.version_info[0] >= 3:
     PY3 = True
     
     input = input
     builtin_mod_name = "builtins"
     import builtins as builtin_mod
     
     str_to_unicode = no_code
     unicode_to_str = no_code
     str_to_bytes = encode
     bytes_to_str = decode
     cast_bytes_py2 = no_code
     
     string_types = (str,)
     unicode_type = str
     
     def isidentifier(s, dotted=False):
         if dotted:
             return all(isidentifier(a) for a in s.split("."))
         return s.isidentifier()
     
     open = orig_open
     xrange = range
-    iteritems = dict.items
-    itervalues = dict.values
+    def iteritems(d): return iter(d.items())
+    def itervalues(d): return iter(d.values())
     
     MethodType = types.MethodType
     
     def execfile(fname, glob, loc=None):
         loc = loc if (loc is not None) else glob
         with open(fname, 'rb') as f:
             exec(compile(f.read(), fname, 'exec'), glob, loc)
     
     # Refactor print statements in doctests.
     _print_statement_re = re.compile(r"\bprint (?P<expr>.*)$", re.MULTILINE)
     def _print_statement_sub(match):
         expr = match.groups('expr')
         return "print(%s)" % expr
     
     @_modify_str_or_docstring
     def doctest_refactor_print(doc):
         """Refactor 'print x' statements in a doctest to print(x) style. 2to3
         unfortunately doesn't pick up on our doctests.
         
         Can accept a string or a function, so it can be used as a decorator."""
         return _print_statement_re.sub(_print_statement_sub, doc)
     
     # Abstract u'abc' syntax:
     @_modify_str_or_docstring
     def u_format(s):
         """"{u}'abc'" --> "'abc'" (Python 3)
         
         Accepts a string or a function, so it can be used as a decorator."""
         return s.format(u='')
 
 else:
     PY3 = False
     
     input = raw_input
     builtin_mod_name = "__builtin__"
     import __builtin__ as builtin_mod
     
     str_to_unicode = decode
     unicode_to_str = encode
     str_to_bytes = no_code
     bytes_to_str = no_code
     cast_bytes_py2 = cast_bytes
     
     string_types = (str, unicode)
     unicode_type = unicode
     
     import re
     _name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$")
     def isidentifier(s, dotted=False):
         if dotted:
             return all(isidentifier(a) for a in s.split("."))
         return bool(_name_re.match(s))
     
     class open(object):
         """Wrapper providing key part of Python 3 open() interface."""
         def __init__(self, fname, mode="r", encoding="utf-8"):
             self.f = orig_open(fname, mode)
             self.enc = encoding
         
         def write(self, s):
             return self.f.write(s.encode(self.enc))
         
         def read(self, size=-1):
             return self.f.read(size).decode(self.enc)
         
         def close(self):
             return self.f.close()
         
         def __enter__(self):
             return self
         
         def __exit__(self, etype, value, traceback):
             self.f.close()
     
     xrange = xrange
-    iteritems = dict.iteritems
-    itervalues = dict.itervalues
+    def iteritems(d): return d.iteritems()
+    def itervalues(d): return d.itervalues()
 
     def MethodType(func, instance):
         return types.MethodType(func, instance, type(instance))
     
     # don't override system execfile on 2.x:
     execfile = execfile
     
     def doctest_refactor_print(func_or_str):
         return func_or_str
 
 
     # Abstract u'abc' syntax:
     @_modify_str_or_docstring
     def u_format(s):
         """"{u}'abc'" --> "u'abc'" (Python 2)
         
         Accepts a string or a function, so it can be used as a decorator."""
         return s.format(u='u')
 
     if sys.platform == 'win32':
         def execfile(fname, glob=None, loc=None):
             loc = loc if (loc is not None) else glob
             # The rstrip() is necessary b/c trailing whitespace in files will
             # cause an IndentationError in Python 2.6 (this was fixed in 2.7,
             # but we still support 2.6).  See issue 1027.
             scripttext = builtin_mod.open(fname).read().rstrip() + '\n'
             # compile converts unicode filename to str assuming
             # ascii. Let's do the conversion before calling compile
             if isinstance(fname, unicode):
                 filename = unicode_to_str(fname)
             else:
                 filename = fname
             exec(compile(scripttext, filename, 'exec'), glob, loc)
     else:
         def execfile(fname, *where):
             if isinstance(fname, unicode):
                 filename = fname.encode(sys.getfilesystemencoding())
             else:
                 filename = fname
             builtin_mod.execfile(filename, *where)
 
 # Parts below taken from six:
 # Copyright (c) 2010-2013 Benjamin Peterson
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in all
 # copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
 def with_metaclass(meta, *bases):
     """Create a base class with a metaclass."""
     return meta("NewBase", bases, {})

IPython/utils/tests/test_path.py

@@ -1,643 +1,643
 # encoding: utf-8
 """Tests for IPython.utils.path.py"""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
 
 import os
 import shutil
 import sys
 import tempfile
 from contextlib import contextmanager
 
 from os.path import join, abspath, split
 
 import nose.tools as nt
 
 from nose import with_setup
 
 import IPython
 from IPython.testing import decorators as dec
 from IPython.testing.decorators import (skip_if_not_win32, skip_win32,
                                         onlyif_unicode_paths,)
 from IPython.testing.tools import make_tempfile, AssertPrints
 from IPython.utils import path
 from IPython.utils import py3compat
 from IPython.utils.tempdir import TemporaryDirectory
 
 # Platform-dependent imports
 try:
     import winreg as wreg  # Py 3
 except ImportError:
     try:
         import _winreg as wreg  # Py 2
     except ImportError:
         #Fake _winreg module on none windows platforms
         import types
         wr_name = "winreg" if py3compat.PY3 else "_winreg"
         sys.modules[wr_name] = types.ModuleType(wr_name)
         try:
             import winreg as wreg
         except ImportError:
             import _winreg as wreg
         #Add entries that needs to be stubbed by the testing code
         (wreg.OpenKey, wreg.QueryValueEx,) = (None, None)
 
 try:
     reload
 except NameError:   # Python 3
     from imp import reload
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 env = os.environ
 TEST_FILE_PATH = split(abspath(__file__))[0]
 TMP_TEST_DIR = tempfile.mkdtemp()
 HOME_TEST_DIR = join(TMP_TEST_DIR, "home_test_dir")
 XDG_TEST_DIR = join(HOME_TEST_DIR, "xdg_test_dir")
 XDG_CACHE_DIR = join(HOME_TEST_DIR, "xdg_cache_dir")
 IP_TEST_DIR = join(HOME_TEST_DIR,'.ipython')
 #
 # Setup/teardown functions/decorators
 #
 
 def setup():
     """Setup testenvironment for the module:
 
             - Adds dummy home dir tree
     """
     # Do not mask exceptions here.  In particular, catching WindowsError is a
     # problem because that exception is only defined on Windows...
     os.makedirs(IP_TEST_DIR)
     os.makedirs(os.path.join(XDG_TEST_DIR, 'ipython'))
     os.makedirs(os.path.join(XDG_CACHE_DIR, 'ipython'))
 
 
 def teardown():
     """Teardown testenvironment for the module:
 
             - Remove dummy home dir tree
     """
     # Note: we remove the parent test dir, which is the root of all test
     # subdirs we may have created.  Use shutil instead of os.removedirs, so
     # that non-empty directories are all recursively removed.
     shutil.rmtree(TMP_TEST_DIR)
 
 
 def setup_environment():
     """Setup testenvironment for some functions that are tested
     in this module. In particular this functions stores attributes
     and other things that we need to stub in some test functions.
     This needs to be done on a function level and not module level because
     each testfunction needs a pristine environment.
     """
     global oldstuff, platformstuff
     oldstuff = (env.copy(), os.name, sys.platform, path.get_home_dir, IPython.__file__, os.getcwd())
 
     if os.name == 'nt':
         platformstuff = (wreg.OpenKey, wreg.QueryValueEx,)
 
 
 def teardown_environment():
     """Restore things that were remembered by the setup_environment function
     """
     (oldenv, os.name, sys.platform, path.get_home_dir, IPython.__file__, old_wd) = oldstuff
     os.chdir(old_wd)
     reload(path)
 
-    for key in env.keys():
+    for key in list(env):
         if key not in oldenv:
             del env[key]
     env.update(oldenv)
     if hasattr(sys, 'frozen'):
         del sys.frozen
     if os.name == 'nt':
         (wreg.OpenKey, wreg.QueryValueEx,) = platformstuff
 
 # Build decorator that uses the setup_environment/setup_environment
 with_environment = with_setup(setup_environment, teardown_environment)
 
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_1():
     """Testcase for py2exe logic, un-compressed lib
     """
     unfrozen = path.get_home_dir()
     sys.frozen = True
 
     #fake filename for IPython.__init__
     IPython.__file__ = abspath(join(HOME_TEST_DIR, "Lib/IPython/__init__.py"))
 
     home_dir = path.get_home_dir()
     nt.assert_equal(home_dir, unfrozen)
 
 
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_2():
     """Testcase for py2exe logic, compressed lib
     """
     unfrozen = path.get_home_dir()
     sys.frozen = True
     #fake filename for IPython.__init__
     IPython.__file__ = abspath(join(HOME_TEST_DIR, "Library.zip/IPython/__init__.py")).lower()
 
     home_dir = path.get_home_dir(True)
     nt.assert_equal(home_dir, unfrozen)
 
 
 @with_environment
 def test_get_home_dir_3():
     """get_home_dir() uses $HOME if set"""
     env["HOME"] = HOME_TEST_DIR
     home_dir = path.get_home_dir(True)
     # get_home_dir expands symlinks
     nt.assert_equal(home_dir, os.path.realpath(env["HOME"]))
 
 
 @with_environment
 def test_get_home_dir_4():
     """get_home_dir() still works if $HOME is not set"""
 
     if 'HOME' in env: del env['HOME']
     # this should still succeed, but we don't care what the answer is
     home = path.get_home_dir(False)
 
 @with_environment
 def test_get_home_dir_5():
     """raise HomeDirError if $HOME is specified, but not a writable dir"""
     env['HOME'] = abspath(HOME_TEST_DIR+'garbage')
     # set os.name = posix, to prevent My Documents fallback on Windows
     os.name = 'posix'
     nt.assert_raises(path.HomeDirError, path.get_home_dir, True)
 
 
 # Should we stub wreg fully so we can run the test on all platforms?
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_8():
     """Using registry hack for 'My Documents', os=='nt'
 
     HOMESHARE, HOMEDRIVE, HOMEPATH, USERPROFILE and others are missing.
     """
     os.name = 'nt'
     # Remove from stub environment all keys that may be set
     for key in ['HOME', 'HOMESHARE', 'HOMEDRIVE', 'HOMEPATH', 'USERPROFILE']:
         env.pop(key, None)
 
     #Stub windows registry functions
     def OpenKey(x, y):
         class key:
             def Close(self):
                 pass
         return key()
     def QueryValueEx(x, y):
         return [abspath(HOME_TEST_DIR)]
 
     wreg.OpenKey = OpenKey
     wreg.QueryValueEx = QueryValueEx
 
     home_dir = path.get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
 
 
 @with_environment
 def test_get_ipython_dir_1():
     """test_get_ipython_dir_1, Testcase to see if we can call get_ipython_dir without Exceptions."""
     env_ipdir = os.path.join("someplace", ".ipython")
     path._writable_dir = lambda path: True
     env['IPYTHONDIR'] = env_ipdir
     ipdir = path.get_ipython_dir()
     nt.assert_equal(ipdir, env_ipdir)
 
 
 @with_environment
 def test_get_ipython_dir_2():
     """test_get_ipython_dir_2, Testcase to see if we can call get_ipython_dir without Exceptions."""
     path.get_home_dir = lambda : "someplace"
     path.get_xdg_dir = lambda : None
     path._writable_dir = lambda path: True
     os.name = "posix"
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
     env.pop('XDG_CONFIG_HOME', None)
     ipdir = path.get_ipython_dir()
     nt.assert_equal(ipdir, os.path.join("someplace", ".ipython"))
 
 @with_environment
 def test_get_ipython_dir_3():
     """test_get_ipython_dir_3, use XDG if defined, and .ipython doesn't exist."""
     path.get_home_dir = lambda : "someplace"
     path._writable_dir = lambda path: True
     os.name = "posix"
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
     env['XDG_CONFIG_HOME'] = XDG_TEST_DIR
     ipdir = path.get_ipython_dir()
     if sys.platform == "darwin":
         expected = os.path.join("someplace", ".ipython")
     else:
         expected = os.path.join(XDG_TEST_DIR, "ipython")
     nt.assert_equal(ipdir, expected)
 
 @with_environment
 def test_get_ipython_dir_4():
     """test_get_ipython_dir_4, use XDG if both exist."""
     path.get_home_dir = lambda : HOME_TEST_DIR
     os.name = "posix"
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
     env['XDG_CONFIG_HOME'] = XDG_TEST_DIR
     ipdir = path.get_ipython_dir()
     if sys.platform == "darwin":
         expected = os.path.join(HOME_TEST_DIR, ".ipython")
     else:
         expected = os.path.join(XDG_TEST_DIR, "ipython")
     nt.assert_equal(ipdir, expected)
 
 @with_environment
 def test_get_ipython_dir_5():
     """test_get_ipython_dir_5, use .ipython if exists and XDG defined, but doesn't exist."""
     path.get_home_dir = lambda : HOME_TEST_DIR
     os.name = "posix"
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
     env['XDG_CONFIG_HOME'] = XDG_TEST_DIR
     os.rmdir(os.path.join(XDG_TEST_DIR, 'ipython'))
     ipdir = path.get_ipython_dir()
     nt.assert_equal(ipdir, IP_TEST_DIR)
 
 @with_environment
 def test_get_ipython_dir_6():
     """test_get_ipython_dir_6, use XDG if defined and neither exist."""
     xdg = os.path.join(HOME_TEST_DIR, 'somexdg')
     os.mkdir(xdg)
     shutil.rmtree(os.path.join(HOME_TEST_DIR, '.ipython'))
     path.get_home_dir = lambda : HOME_TEST_DIR
     path.get_xdg_dir = lambda : xdg
     os.name = "posix"
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
     env.pop('XDG_CONFIG_HOME', None)
     xdg_ipdir = os.path.join(xdg, "ipython")
     ipdir = path.get_ipython_dir()
     nt.assert_equal(ipdir, xdg_ipdir)
 
 @with_environment
 def test_get_ipython_dir_7():
     """test_get_ipython_dir_7, test home directory expansion on IPYTHONDIR"""
     path._writable_dir = lambda path: True
     home_dir = os.path.normpath(os.path.expanduser('~'))
     env['IPYTHONDIR'] = os.path.join('~', 'somewhere')
     ipdir = path.get_ipython_dir()
     nt.assert_equal(ipdir, os.path.join(home_dir, 'somewhere'))
 
 @skip_win32
 @with_environment
 def test_get_ipython_dir_8():
     """test_get_ipython_dir_8, test / home directory"""
     old = path._writable_dir, path.get_xdg_dir
     try:
         path._writable_dir = lambda path: bool(path)
         path.get_xdg_dir = lambda: None
         env.pop('IPYTHON_DIR', None)
         env.pop('IPYTHONDIR', None)
         env['HOME'] = '/'
         nt.assert_equal(path.get_ipython_dir(), '/.ipython')
     finally:
         path._writable_dir, path.get_xdg_dir = old
 
 @with_environment
 def test_get_xdg_dir_0():
     """test_get_xdg_dir_0, check xdg_dir"""
     reload(path)
     path._writable_dir = lambda path: True
     path.get_home_dir = lambda : 'somewhere'
     os.name = "posix"
     sys.platform = "linux2"
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
     env.pop('XDG_CONFIG_HOME', None)
 
     nt.assert_equal(path.get_xdg_dir(), os.path.join('somewhere', '.config'))
 
 
 @with_environment
 def test_get_xdg_dir_1():
     """test_get_xdg_dir_1, check nonexistant xdg_dir"""
     reload(path)
     path.get_home_dir = lambda : HOME_TEST_DIR
     os.name = "posix"
     sys.platform = "linux2"
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
     env.pop('XDG_CONFIG_HOME', None)
     nt.assert_equal(path.get_xdg_dir(), None)
 
 @with_environment
 def test_get_xdg_dir_2():
     """test_get_xdg_dir_2, check xdg_dir default to ~/.config"""
     reload(path)
     path.get_home_dir = lambda : HOME_TEST_DIR
     os.name = "posix"
     sys.platform = "linux2"
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
     env.pop('XDG_CONFIG_HOME', None)
     cfgdir=os.path.join(path.get_home_dir(), '.config')
     if not os.path.exists(cfgdir):
         os.makedirs(cfgdir)
 
     nt.assert_equal(path.get_xdg_dir(), cfgdir)
 
 @with_environment
 def test_get_xdg_dir_3():
     """test_get_xdg_dir_3, check xdg_dir not used on OS X"""
     reload(path)
     path.get_home_dir = lambda : HOME_TEST_DIR
     os.name = "posix"
     sys.platform = "darwin"
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
     env.pop('XDG_CONFIG_HOME', None)
     cfgdir=os.path.join(path.get_home_dir(), '.config')
     if not os.path.exists(cfgdir):
         os.makedirs(cfgdir)
 
     nt.assert_equal(path.get_xdg_dir(), None)
 
 def test_filefind():
     """Various tests for filefind"""
     f = tempfile.NamedTemporaryFile()
     # print 'fname:',f.name
     alt_dirs = path.get_ipython_dir()
     t = path.filefind(f.name, alt_dirs)
     # print 'found:',t
 
 @with_environment
 def test_get_ipython_cache_dir():
     os.environ["HOME"] = HOME_TEST_DIR
     if os.name == 'posix' and sys.platform != 'darwin':
         # test default
         os.makedirs(os.path.join(HOME_TEST_DIR, ".cache"))
         os.environ.pop("XDG_CACHE_HOME", None)
         ipdir = path.get_ipython_cache_dir()
         nt.assert_equal(os.path.join(HOME_TEST_DIR, ".cache", "ipython"),
                         ipdir)
         nt.assert_true(os.path.isdir(ipdir))
 
         # test env override
         os.environ["XDG_CACHE_HOME"] = XDG_CACHE_DIR
         ipdir = path.get_ipython_cache_dir()
         nt.assert_true(os.path.isdir(ipdir))
         nt.assert_equal(ipdir, os.path.join(XDG_CACHE_DIR, "ipython"))
     else:
         nt.assert_equal(path.get_ipython_cache_dir(),
                         path.get_ipython_dir())
 
 def test_get_ipython_package_dir():
     ipdir = path.get_ipython_package_dir()
     nt.assert_true(os.path.isdir(ipdir))
 
 
 def test_get_ipython_module_path():
     ipapp_path = path.get_ipython_module_path('IPython.terminal.ipapp')
     nt.assert_true(os.path.isfile(ipapp_path))
 
 
 @dec.skip_if_not_win32
 def test_get_long_path_name_win32():
     with TemporaryDirectory() as tmpdir:
 
         # Make a long path.
         long_path = os.path.join(tmpdir, u'this is my long path name')
         os.makedirs(long_path)
 
         # Test to see if the short path evaluates correctly.
         short_path = os.path.join(tmpdir, u'THISIS~1')
         evaluated_path = path.get_long_path_name(short_path)
         nt.assert_equal(evaluated_path.lower(), long_path.lower())
 
 
 @dec.skip_win32
 def test_get_long_path_name():
     p = path.get_long_path_name('/usr/local')
     nt.assert_equal(p,'/usr/local')
 
 @dec.skip_win32 # can't create not-user-writable dir on win
 @with_environment
 def test_not_writable_ipdir():
     tmpdir = tempfile.mkdtemp()
     os.name = "posix"
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
     env.pop('XDG_CONFIG_HOME', None)
     env['HOME'] = tmpdir
     ipdir = os.path.join(tmpdir, '.ipython')
     os.mkdir(ipdir)
     os.chmod(ipdir, 600)
     with AssertPrints('is not a writable location', channel='stderr'):
         ipdir = path.get_ipython_dir()
     env.pop('IPYTHON_DIR', None)
 
 def test_unquote_filename():
     for win32 in (True, False):
         nt.assert_equal(path.unquote_filename('foo.py', win32=win32), 'foo.py')
         nt.assert_equal(path.unquote_filename('foo bar.py', win32=win32), 'foo bar.py')
     nt.assert_equal(path.unquote_filename('"foo.py"', win32=True), 'foo.py')
     nt.assert_equal(path.unquote_filename('"foo bar.py"', win32=True), 'foo bar.py')
     nt.assert_equal(path.unquote_filename("'foo.py'", win32=True), 'foo.py')
     nt.assert_equal(path.unquote_filename("'foo bar.py'", win32=True), 'foo bar.py')
     nt.assert_equal(path.unquote_filename('"foo.py"', win32=False), '"foo.py"')
     nt.assert_equal(path.unquote_filename('"foo bar.py"', win32=False), '"foo bar.py"')
     nt.assert_equal(path.unquote_filename("'foo.py'", win32=False), "'foo.py'")
     nt.assert_equal(path.unquote_filename("'foo bar.py'", win32=False), "'foo bar.py'")
 
 @with_environment
 def test_get_py_filename():
     os.chdir(TMP_TEST_DIR)
     for win32 in (True, False):
         with make_tempfile('foo.py'):
             nt.assert_equal(path.get_py_filename('foo.py', force_win32=win32), 'foo.py')
             nt.assert_equal(path.get_py_filename('foo', force_win32=win32), 'foo.py')
         with make_tempfile('foo'):
             nt.assert_equal(path.get_py_filename('foo', force_win32=win32), 'foo')
             nt.assert_raises(IOError, path.get_py_filename, 'foo.py', force_win32=win32)
         nt.assert_raises(IOError, path.get_py_filename, 'foo', force_win32=win32)
         nt.assert_raises(IOError, path.get_py_filename, 'foo.py', force_win32=win32)
         true_fn = 'foo with spaces.py'
         with make_tempfile(true_fn):
             nt.assert_equal(path.get_py_filename('foo with spaces', force_win32=win32), true_fn)
             nt.assert_equal(path.get_py_filename('foo with spaces.py', force_win32=win32), true_fn)
             if win32:
                 nt.assert_equal(path.get_py_filename('"foo with spaces.py"', force_win32=True), true_fn)
                 nt.assert_equal(path.get_py_filename("'foo with spaces.py'", force_win32=True), true_fn)
             else:
                 nt.assert_raises(IOError, path.get_py_filename, '"foo with spaces.py"', force_win32=False)
                 nt.assert_raises(IOError, path.get_py_filename, "'foo with spaces.py'", force_win32=False)
 
 @onlyif_unicode_paths
 def test_unicode_in_filename():
     """When a file doesn't exist, the exception raised should be safe to call
     str() on - i.e. in Python 2 it must only have ASCII characters.
     
     https://github.com/ipython/ipython/issues/875
     """
     try:
         # these calls should not throw unicode encode exceptions
         path.get_py_filename(u'fooéè.py',  force_win32=False)
     except IOError as ex:
         str(ex)
 
 
 class TestShellGlob(object):
 
     @classmethod
     def setUpClass(cls):
-        cls.filenames_start_with_a = map('a{0}'.format, range(3))
-        cls.filenames_end_with_b = map('{0}b'.format, range(3))
+        cls.filenames_start_with_a = ['a0', 'a1', 'a2']
+        cls.filenames_end_with_b = ['0b', '1b', '2b']
         cls.filenames = cls.filenames_start_with_a + cls.filenames_end_with_b
         cls.tempdir = TemporaryDirectory()
         td = cls.tempdir.name
 
         with cls.in_tempdir():
             # Create empty files
             for fname in cls.filenames:
                 open(os.path.join(td, fname), 'w').close()
 
     @classmethod
     def tearDownClass(cls):
         cls.tempdir.cleanup()
 
     @classmethod
     @contextmanager
     def in_tempdir(cls):
         save = os.getcwdu()
         try:
             os.chdir(cls.tempdir.name)
             yield
         finally:
             os.chdir(save)
 
     def check_match(self, patterns, matches):
         with self.in_tempdir():
             # glob returns unordered list. that's why sorted is required.
             nt.assert_equals(sorted(path.shellglob(patterns)),
                              sorted(matches))
 
     def common_cases(self):
         return [
             (['*'], self.filenames),
             (['a*'], self.filenames_start_with_a),
             (['*c'], ['*c']),
             (['*', 'a*', '*b', '*c'], self.filenames
                                       + self.filenames_start_with_a
                                       + self.filenames_end_with_b
                                       + ['*c']),
             (['a[012]'], self.filenames_start_with_a),
         ]
 
     @skip_win32
     def test_match_posix(self):
         for (patterns, matches) in self.common_cases() + [
                 ([r'\*'], ['*']),
                 ([r'a\*', 'a*'], ['a*'] + self.filenames_start_with_a),
                 ([r'a\[012]'], ['a[012]']),
                 ]:
             yield (self.check_match, patterns, matches)
 
     @skip_if_not_win32
     def test_match_windows(self):
         for (patterns, matches) in self.common_cases() + [
                 # In windows, backslash is interpreted as path
                 # separator.  Therefore, you can't escape glob
                 # using it.
                 ([r'a\*', 'a*'], [r'a\*'] + self.filenames_start_with_a),
                 ([r'a\[012]'], [r'a\[012]']),
                 ]:
             yield (self.check_match, patterns, matches)
 
 
 def test_unescape_glob():
     nt.assert_equals(path.unescape_glob(r'\*\[\!\]\?'), '*[!]?')
     nt.assert_equals(path.unescape_glob(r'\\*'), r'\*')
     nt.assert_equals(path.unescape_glob(r'\\\*'), r'\*')
     nt.assert_equals(path.unescape_glob(r'\\a'), r'\a')
     nt.assert_equals(path.unescape_glob(r'\a'), r'\a')
 
 
 class TestLinkOrCopy(object):
     def setUp(self):
         self.tempdir = TemporaryDirectory()
         self.src = self.dst("src")
         with open(self.src, "w") as f:
             f.write("Hello, world!")
 
     def tearDown(self):
         self.tempdir.cleanup()
 
     def dst(self, *args):
         return os.path.join(self.tempdir.name, *args)
 
     def assert_inode_not_equal(self, a, b):
         nt.assert_not_equals(os.stat(a).st_ino, os.stat(b).st_ino,
                              "%r and %r do reference the same indoes" %(a, b))
 
     def assert_inode_equal(self, a, b):
         nt.assert_equals(os.stat(a).st_ino, os.stat(b).st_ino,
                          "%r and %r do not reference the same indoes" %(a, b))
 
     def assert_content_equal(self, a, b):
         with open(a) as a_f:
             with open(b) as b_f:
                 nt.assert_equals(a_f.read(), b_f.read())
 
     @skip_win32
     def test_link_successful(self):
         dst = self.dst("target")
         path.link_or_copy(self.src, dst)
         self.assert_inode_equal(self.src, dst)
 
     @skip_win32
     def test_link_into_dir(self):
         dst = self.dst("some_dir")
         os.mkdir(dst)
         path.link_or_copy(self.src, dst)
         expected_dst = self.dst("some_dir", os.path.basename(self.src))
         self.assert_inode_equal(self.src, expected_dst)
 
     @skip_win32
     def test_target_exists(self):
         dst = self.dst("target")
         open(dst, "w").close()
         path.link_or_copy(self.src, dst)
         self.assert_inode_equal(self.src, dst)
 
     @skip_win32
     def test_no_link(self):
         real_link = os.link
         try:
             del os.link
             dst = self.dst("target")
             path.link_or_copy(self.src, dst)
             self.assert_content_equal(self.src, dst)
             self.assert_inode_not_equal(self.src, dst)
         finally:
             os.link = real_link
 
     @skip_if_not_win32
     def test_windows(self):
         dst = self.dst("target")
         path.link_or_copy(self.src, dst)
         self.assert_content_equal(self.src, dst)

IPython/utils/tests/test_traitlets.py

@@ -1,975 +1,975
 # encoding: utf-8
 """
 Tests for IPython.utils.traitlets.
 
 Authors:
 
 * Brian Granger
 * Enthought, Inc.  Some of the code in this file comes from enthought.traits
   and is licensed under the BSD license.  Also, many of the ideas also come
   from enthought.traits even though our implementation is very different.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import re
 import sys
 from unittest import TestCase
 
 import nose.tools as nt
 from nose import SkipTest
 
 from IPython.utils.traitlets import (
     HasTraits, MetaHasTraits, TraitType, Any, CBytes, Dict,
     Int, Long, Integer, Float, Complex, Bytes, Unicode, TraitError,
     Undefined, Type, This, Instance, TCPAddress, List, Tuple,
     ObjectName, DottedObjectName, CRegExp
 )
 from IPython.utils import py3compat
 from IPython.testing.decorators import skipif
 
 #-----------------------------------------------------------------------------
 # Helper classes for testing
 #-----------------------------------------------------------------------------
 
 
 class HasTraitsStub(HasTraits):
 
     def _notify_trait(self, name, old, new):
         self._notify_name = name
         self._notify_old = old
         self._notify_new = new
 
 
 #-----------------------------------------------------------------------------
 # Test classes
 #-----------------------------------------------------------------------------
 
 
 class TestTraitType(TestCase):
 
     def test_get_undefined(self):
         class A(HasTraits):
             a = TraitType
         a = A()
         self.assertEqual(a.a, Undefined)
 
     def test_set(self):
         class A(HasTraitsStub):
             a = TraitType
 
         a = A()
         a.a = 10
         self.assertEqual(a.a, 10)
         self.assertEqual(a._notify_name, 'a')
         self.assertEqual(a._notify_old, Undefined)
         self.assertEqual(a._notify_new, 10)
 
     def test_validate(self):
         class MyTT(TraitType):
             def validate(self, inst, value):
                 return -1
         class A(HasTraitsStub):
             tt = MyTT
 
         a = A()
         a.tt = 10
         self.assertEqual(a.tt, -1)
 
     def test_default_validate(self):
         class MyIntTT(TraitType):
             def validate(self, obj, value):
                 if isinstance(value, int):
                     return value
                 self.error(obj, value)
         class A(HasTraits):
             tt = MyIntTT(10)
         a = A()
         self.assertEqual(a.tt, 10)
 
         # Defaults are validated when the HasTraits is instantiated
         class B(HasTraits):
             tt = MyIntTT('bad default')
         self.assertRaises(TraitError, B)
 
     def test_is_valid_for(self):
         class MyTT(TraitType):
             def is_valid_for(self, value):
                 return True
         class A(HasTraits):
             tt = MyTT
 
         a = A()
         a.tt = 10
         self.assertEqual(a.tt, 10)
 
     def test_value_for(self):
         class MyTT(TraitType):
             def value_for(self, value):
                 return 20
         class A(HasTraits):
             tt = MyTT
 
         a = A()
         a.tt = 10
         self.assertEqual(a.tt, 20)
 
     def test_info(self):
         class A(HasTraits):
             tt = TraitType
         a = A()
         self.assertEqual(A.tt.info(), 'any value')
 
     def test_error(self):
         class A(HasTraits):
             tt = TraitType
         a = A()
         self.assertRaises(TraitError, A.tt.error, a, 10)
 
     def test_dynamic_initializer(self):
         class A(HasTraits):
             x = Int(10)
             def _x_default(self):
                 return 11
         class B(A):
             x = Int(20)
         class C(A):
             def _x_default(self):
                 return 21
 
         a = A()
         self.assertEqual(a._trait_values, {})
-        self.assertEqual(a._trait_dyn_inits.keys(), ['x'])
+        self.assertEqual(list(a._trait_dyn_inits.keys()), ['x'])
         self.assertEqual(a.x, 11)
         self.assertEqual(a._trait_values, {'x': 11})
         b = B()
         self.assertEqual(b._trait_values, {'x': 20})
-        self.assertEqual(a._trait_dyn_inits.keys(), ['x'])
+        self.assertEqual(list(a._trait_dyn_inits.keys()), ['x'])
         self.assertEqual(b.x, 20)
         c = C()
         self.assertEqual(c._trait_values, {})
-        self.assertEqual(a._trait_dyn_inits.keys(), ['x'])
+        self.assertEqual(list(a._trait_dyn_inits.keys()), ['x'])
         self.assertEqual(c.x, 21)
         self.assertEqual(c._trait_values, {'x': 21})
         # Ensure that the base class remains unmolested when the _default
         # initializer gets overridden in a subclass.
         a = A()
         c = C()
         self.assertEqual(a._trait_values, {})
-        self.assertEqual(a._trait_dyn_inits.keys(), ['x'])
+        self.assertEqual(list(a._trait_dyn_inits.keys()), ['x'])
         self.assertEqual(a.x, 11)
         self.assertEqual(a._trait_values, {'x': 11})
 
 
 
 class TestHasTraitsMeta(TestCase):
 
     def test_metaclass(self):
         self.assertEqual(type(HasTraits), MetaHasTraits)
 
         class A(HasTraits):
             a = Int
 
         a = A()
         self.assertEqual(type(a.__class__), MetaHasTraits)
         self.assertEqual(a.a,0)
         a.a = 10
         self.assertEqual(a.a,10)
 
         class B(HasTraits):
             b = Int()
 
         b = B()
         self.assertEqual(b.b,0)
         b.b = 10
         self.assertEqual(b.b,10)
 
         class C(HasTraits):
             c = Int(30)
 
         c = C()
         self.assertEqual(c.c,30)
         c.c = 10
         self.assertEqual(c.c,10)
 
     def test_this_class(self):
         class A(HasTraits):
             t = This()
             tt = This()
         class B(A):
             tt = This()
             ttt = This()
         self.assertEqual(A.t.this_class, A)
         self.assertEqual(B.t.this_class, A)
         self.assertEqual(B.tt.this_class, B)
         self.assertEqual(B.ttt.this_class, B)
 
 class TestHasTraitsNotify(TestCase):
 
     def setUp(self):
         self._notify1 = []
         self._notify2 = []
 
     def notify1(self, name, old, new):
         self._notify1.append((name, old, new))
 
     def notify2(self, name, old, new):
         self._notify2.append((name, old, new))
 
     def test_notify_all(self):
 
         class A(HasTraits):
             a = Int
             b = Float
 
         a = A()
         a.on_trait_change(self.notify1)
         a.a = 0
         self.assertEqual(len(self._notify1),0)
         a.b = 0.0
         self.assertEqual(len(self._notify1),0)
         a.a = 10
         self.assertTrue(('a',0,10) in self._notify1)
         a.b = 10.0
         self.assertTrue(('b',0.0,10.0) in self._notify1)
         self.assertRaises(TraitError,setattr,a,'a','bad string')
         self.assertRaises(TraitError,setattr,a,'b','bad string')
         self._notify1 = []
         a.on_trait_change(self.notify1,remove=True)
         a.a = 20
         a.b = 20.0
         self.assertEqual(len(self._notify1),0)
 
     def test_notify_one(self):
 
         class A(HasTraits):
             a = Int
             b = Float
 
         a = A()
         a.on_trait_change(self.notify1, 'a')
         a.a = 0
         self.assertEqual(len(self._notify1),0)
         a.a = 10
         self.assertTrue(('a',0,10) in self._notify1)
         self.assertRaises(TraitError,setattr,a,'a','bad string')
 
     def test_subclass(self):
 
         class A(HasTraits):
             a = Int
 
         class B(A):
             b = Float
 
         b = B()
         self.assertEqual(b.a,0)
         self.assertEqual(b.b,0.0)
         b.a = 100
         b.b = 100.0
         self.assertEqual(b.a,100)
         self.assertEqual(b.b,100.0)
 
     def test_notify_subclass(self):
 
         class A(HasTraits):
             a = Int
 
         class B(A):
             b = Float
 
         b = B()
         b.on_trait_change(self.notify1, 'a')
         b.on_trait_change(self.notify2, 'b')
         b.a = 0
         b.b = 0.0
         self.assertEqual(len(self._notify1),0)
         self.assertEqual(len(self._notify2),0)
         b.a = 10
         b.b = 10.0
         self.assertTrue(('a',0,10) in self._notify1)
         self.assertTrue(('b',0.0,10.0) in self._notify2)
 
     def test_static_notify(self):
 
         class A(HasTraits):
             a = Int
             _notify1 = []
             def _a_changed(self, name, old, new):
                 self._notify1.append((name, old, new))
 
         a = A()
         a.a = 0
         # This is broken!!!
         self.assertEqual(len(a._notify1),0)
         a.a = 10
         self.assertTrue(('a',0,10) in a._notify1)
 
         class B(A):
             b = Float
             _notify2 = []
             def _b_changed(self, name, old, new):
                 self._notify2.append((name, old, new))
 
         b = B()
         b.a = 10
         b.b = 10.0
         self.assertTrue(('a',0,10) in b._notify1)
         self.assertTrue(('b',0.0,10.0) in b._notify2)
 
     def test_notify_args(self):
 
         def callback0():
             self.cb = ()
         def callback1(name):
             self.cb = (name,)
         def callback2(name, new):
             self.cb = (name, new)
         def callback3(name, old, new):
             self.cb = (name, old, new)
 
         class A(HasTraits):
             a = Int
 
         a = A()
         a.on_trait_change(callback0, 'a')
         a.a = 10
         self.assertEqual(self.cb,())
         a.on_trait_change(callback0, 'a', remove=True)
 
         a.on_trait_change(callback1, 'a')
         a.a = 100
         self.assertEqual(self.cb,('a',))
         a.on_trait_change(callback1, 'a', remove=True)
 
         a.on_trait_change(callback2, 'a')
         a.a = 1000
         self.assertEqual(self.cb,('a',1000))
         a.on_trait_change(callback2, 'a', remove=True)
 
         a.on_trait_change(callback3, 'a')
         a.a = 10000
         self.assertEqual(self.cb,('a',1000,10000))
         a.on_trait_change(callback3, 'a', remove=True)
 
         self.assertEqual(len(a._trait_notifiers['a']),0)
 
     def test_notify_only_once(self):
 
         class A(HasTraits):
             listen_to = ['a']
             
             a = Int(0)
             b = 0
             
             def __init__(self, **kwargs):
                 super(A, self).__init__(**kwargs)
                 self.on_trait_change(self.listener1, ['a'])
             
             def listener1(self, name, old, new):
                 self.b += 1
 
         class B(A):
                     
             c = 0
             d = 0
             
             def __init__(self, **kwargs):
                 super(B, self).__init__(**kwargs)
                 self.on_trait_change(self.listener2)
             
             def listener2(self, name, old, new):
                 self.c += 1
             
             def _a_changed(self, name, old, new):
                 self.d += 1
 
         b = B()
         b.a += 1
         self.assertEqual(b.b, b.c)
         self.assertEqual(b.b, b.d)
         b.a += 1
         self.assertEqual(b.b, b.c)
         self.assertEqual(b.b, b.d)
 
 
 class TestHasTraits(TestCase):
 
     def test_trait_names(self):
         class A(HasTraits):
             i = Int
             f = Float
         a = A()
         self.assertEqual(sorted(a.trait_names()),['f','i'])
         self.assertEqual(sorted(A.class_trait_names()),['f','i'])
 
     def test_trait_metadata(self):
         class A(HasTraits):
             i = Int(config_key='MY_VALUE')
         a = A()
         self.assertEqual(a.trait_metadata('i','config_key'), 'MY_VALUE')
 
     def test_traits(self):
         class A(HasTraits):
             i = Int
             f = Float
         a = A()
         self.assertEqual(a.traits(), dict(i=A.i, f=A.f))
         self.assertEqual(A.class_traits(), dict(i=A.i, f=A.f))
 
     def test_traits_metadata(self):
         class A(HasTraits):
             i = Int(config_key='VALUE1', other_thing='VALUE2')
             f = Float(config_key='VALUE3', other_thing='VALUE2')
             j = Int(0)
         a = A()
         self.assertEqual(a.traits(), dict(i=A.i, f=A.f, j=A.j))
         traits = a.traits(config_key='VALUE1', other_thing='VALUE2')
         self.assertEqual(traits, dict(i=A.i))
 
         # This passes, but it shouldn't because I am replicating a bug in
         # traits.
         traits = a.traits(config_key=lambda v: True)
         self.assertEqual(traits, dict(i=A.i, f=A.f, j=A.j))
 
     def test_init(self):
         class A(HasTraits):
             i = Int()
             x = Float()
         a = A(i=1, x=10.0)
         self.assertEqual(a.i, 1)
         self.assertEqual(a.x, 10.0)
 
     def test_positional_args(self):
         class A(HasTraits):
             i = Int(0)
             def __init__(self, i):
                 super(A, self).__init__()
                 self.i = i
         
         a = A(5)
         self.assertEqual(a.i, 5)
         # should raise TypeError if no positional arg given
         self.assertRaises(TypeError, A)
 
 #-----------------------------------------------------------------------------
 # Tests for specific trait types
 #-----------------------------------------------------------------------------
 
 
 class TestType(TestCase):
 
     def test_default(self):
 
         class B(object): pass
         class A(HasTraits):
             klass = Type
 
         a = A()
         self.assertEqual(a.klass, None)
 
         a.klass = B
         self.assertEqual(a.klass, B)
         self.assertRaises(TraitError, setattr, a, 'klass', 10)
 
     def test_value(self):
 
         class B(object): pass
         class C(object): pass
         class A(HasTraits):
             klass = Type(B)
 
         a = A()
         self.assertEqual(a.klass, B)
         self.assertRaises(TraitError, setattr, a, 'klass', C)
         self.assertRaises(TraitError, setattr, a, 'klass', object)
         a.klass = B
 
     def test_allow_none(self):
 
         class B(object): pass
         class C(B): pass
         class A(HasTraits):
             klass = Type(B, allow_none=False)
 
         a = A()
         self.assertEqual(a.klass, B)
         self.assertRaises(TraitError, setattr, a, 'klass', None)
         a.klass = C
         self.assertEqual(a.klass, C)
 
     def test_validate_klass(self):
 
         class A(HasTraits):
             klass = Type('no strings allowed')
 
         self.assertRaises(ImportError, A)
 
         class A(HasTraits):
             klass = Type('rub.adub.Duck')
 
         self.assertRaises(ImportError, A)
 
     def test_validate_default(self):
 
         class B(object): pass
         class A(HasTraits):
             klass = Type('bad default', B)
 
         self.assertRaises(ImportError, A)
 
         class C(HasTraits):
             klass = Type(None, B, allow_none=False)
 
         self.assertRaises(TraitError, C)
 
     def test_str_klass(self):
 
         class A(HasTraits):
             klass = Type('IPython.utils.ipstruct.Struct')
 
         from IPython.utils.ipstruct import Struct
         a = A()
         a.klass = Struct
         self.assertEqual(a.klass, Struct)
 
         self.assertRaises(TraitError, setattr, a, 'klass', 10)
 
 class TestInstance(TestCase):
 
     def test_basic(self):
         class Foo(object): pass
         class Bar(Foo): pass
         class Bah(object): pass
 
         class A(HasTraits):
             inst = Instance(Foo)
 
         a = A()
         self.assertTrue(a.inst is None)
         a.inst = Foo()
         self.assertTrue(isinstance(a.inst, Foo))
         a.inst = Bar()
         self.assertTrue(isinstance(a.inst, Foo))
         self.assertRaises(TraitError, setattr, a, 'inst', Foo)
         self.assertRaises(TraitError, setattr, a, 'inst', Bar)
         self.assertRaises(TraitError, setattr, a, 'inst', Bah())
 
     def test_unique_default_value(self):
         class Foo(object): pass
         class A(HasTraits):
             inst = Instance(Foo,(),{})
 
         a = A()
         b = A()
         self.assertTrue(a.inst is not b.inst)
 
     def test_args_kw(self):
         class Foo(object):
             def __init__(self, c): self.c = c
         class Bar(object): pass
         class Bah(object):
             def __init__(self, c, d):
                 self.c = c; self.d = d
 
         class A(HasTraits):
             inst = Instance(Foo, (10,))
         a = A()
         self.assertEqual(a.inst.c, 10)
 
         class B(HasTraits):
             inst = Instance(Bah, args=(10,), kw=dict(d=20))
         b = B()
         self.assertEqual(b.inst.c, 10)
         self.assertEqual(b.inst.d, 20)
 
         class C(HasTraits):
             inst = Instance(Foo)
         c = C()
         self.assertTrue(c.inst is None)
 
     def test_bad_default(self):
         class Foo(object): pass
 
         class A(HasTraits):
             inst = Instance(Foo, allow_none=False)
 
         self.assertRaises(TraitError, A)
 
     def test_instance(self):
         class Foo(object): pass
 
         def inner():
             class A(HasTraits):
                 inst = Instance(Foo())
 
         self.assertRaises(TraitError, inner)
 
 
 class TestThis(TestCase):
 
     def test_this_class(self):
         class Foo(HasTraits):
             this = This
 
         f = Foo()
         self.assertEqual(f.this, None)
         g = Foo()
         f.this = g
         self.assertEqual(f.this, g)
         self.assertRaises(TraitError, setattr, f, 'this', 10)
 
     def test_this_inst(self):
         class Foo(HasTraits):
             this = This()
 
         f = Foo()
         f.this = Foo()
         self.assertTrue(isinstance(f.this, Foo))
 
     def test_subclass(self):
         class Foo(HasTraits):
             t = This()
         class Bar(Foo):
             pass
         f = Foo()
         b = Bar()
         f.t = b
         b.t = f
         self.assertEqual(f.t, b)
         self.assertEqual(b.t, f)
 
     def test_subclass_override(self):
         class Foo(HasTraits):
             t = This()
         class Bar(Foo):
             t = This()
         f = Foo()
         b = Bar()
         f.t = b
         self.assertEqual(f.t, b)
         self.assertRaises(TraitError, setattr, b, 't', f)
 
 class TraitTestBase(TestCase):
     """A best testing class for basic trait types."""
 
     def assign(self, value):
         self.obj.value = value
 
     def coerce(self, value):
         return value
 
     def test_good_values(self):
         if hasattr(self, '_good_values'):
             for value in self._good_values:
                 self.assign(value)
                 self.assertEqual(self.obj.value, self.coerce(value))
 
     def test_bad_values(self):
         if hasattr(self, '_bad_values'):
             for value in self._bad_values:
                 try:
                     self.assertRaises(TraitError, self.assign, value)
                 except AssertionError:
                     assert False, value
 
     def test_default_value(self):
         if hasattr(self, '_default_value'):
             self.assertEqual(self._default_value, self.obj.value)
 
     def tearDown(self):
         # restore default value after tests, if set
         if hasattr(self, '_default_value'):
             self.obj.value = self._default_value
 
 
 class AnyTrait(HasTraits):
 
     value = Any
 
 class AnyTraitTest(TraitTestBase):
 
     obj = AnyTrait()
 
     _default_value = None
     _good_values   = [10.0, 'ten', u'ten', [10], {'ten': 10},(10,), None, 1j]
     _bad_values    = []
 
 
 class IntTrait(HasTraits):
 
     value = Int(99)
 
 class TestInt(TraitTestBase):
 
     obj = IntTrait()
     _default_value = 99
     _good_values   = [10, -10]
     _bad_values    = ['ten', u'ten', [10], {'ten': 10},(10,), None, 1j,
                       10.1, -10.1, '10L', '-10L', '10.1', '-10.1', u'10L',
                       u'-10L', u'10.1', u'-10.1',  '10', '-10', u'10', u'-10']
     if not py3compat.PY3:
         _bad_values.extend([long(10), long(-10), 10*sys.maxint, -10*sys.maxint])
 
 
 class LongTrait(HasTraits):
 
     value = Long(99 if py3compat.PY3 else long(99))
 
 class TestLong(TraitTestBase):
 
     obj = LongTrait()
 
     _default_value = 99 if py3compat.PY3 else long(99)
     _good_values   = [10, -10]
     _bad_values    = ['ten', u'ten', [10], {'ten': 10},(10,),
                       None, 1j, 10.1, -10.1, '10', '-10', '10L', '-10L', '10.1',
                       '-10.1', u'10', u'-10', u'10L', u'-10L', u'10.1',
                       u'-10.1']
     if not py3compat.PY3:
         # maxint undefined on py3, because int == long
         _good_values.extend([long(10), long(-10), 10*sys.maxint, -10*sys.maxint])
         _bad_values.extend([[long(10)], (long(10),)])
 
     @skipif(py3compat.PY3, "not relevant on py3")
     def test_cast_small(self):
         """Long casts ints to long"""
         self.obj.value = 10
         self.assertEqual(type(self.obj.value), long)
 
 
 class IntegerTrait(HasTraits):
     value = Integer(1)
 
 class TestInteger(TestLong):
     obj = IntegerTrait()
     _default_value = 1
 
     def coerce(self, n):
         return int(n)
 
     @skipif(py3compat.PY3, "not relevant on py3")
     def test_cast_small(self):
         """Integer casts small longs to int"""
         if py3compat.PY3:
             raise SkipTest("not relevant on py3")
 
         self.obj.value = long(100)
         self.assertEqual(type(self.obj.value), int)
 
 
 class FloatTrait(HasTraits):
 
     value = Float(99.0)
 
 class TestFloat(TraitTestBase):
 
     obj = FloatTrait()
 
     _default_value = 99.0
     _good_values   = [10, -10, 10.1, -10.1]
     _bad_values    = ['ten', u'ten', [10], {'ten': 10},(10,), None,
                       1j, '10', '-10', '10L', '-10L', '10.1', '-10.1', u'10',
                       u'-10', u'10L', u'-10L', u'10.1', u'-10.1']
     if not py3compat.PY3:
         _bad_values.extend([long(10), long(-10)])
 
 
 class ComplexTrait(HasTraits):
 
     value = Complex(99.0-99.0j)
 
 class TestComplex(TraitTestBase):
 
     obj = ComplexTrait()
 
     _default_value = 99.0-99.0j
     _good_values   = [10, -10, 10.1, -10.1, 10j, 10+10j, 10-10j,
                       10.1j, 10.1+10.1j, 10.1-10.1j]
     _bad_values    = [u'10L', u'-10L', 'ten', [10], {'ten': 10},(10,), None]
     if not py3compat.PY3:
         _bad_values.extend([long(10), long(-10)])
 
 
 class BytesTrait(HasTraits):
 
     value = Bytes(b'string')
 
 class TestBytes(TraitTestBase):
 
     obj = BytesTrait()
 
     _default_value = b'string'
     _good_values   = [b'10', b'-10', b'10L',
                       b'-10L', b'10.1', b'-10.1', b'string']
     _bad_values    = [10, -10, 10.1, -10.1, 1j, [10],
                       ['ten'],{'ten': 10},(10,), None,  u'string']
     if not py3compat.PY3:
         _bad_values.extend([long(10), long(-10)])
 
 
 class UnicodeTrait(HasTraits):
 
     value = Unicode(u'unicode')
 
 class TestUnicode(TraitTestBase):
 
     obj = UnicodeTrait()
 
     _default_value = u'unicode'
     _good_values   = ['10', '-10', '10L', '-10L', '10.1',
                       '-10.1', '', u'', 'string', u'string', u"€"]
     _bad_values    = [10, -10, 10.1, -10.1, 1j,
                       [10], ['ten'], [u'ten'], {'ten': 10},(10,), None]
     if not py3compat.PY3:
         _bad_values.extend([long(10), long(-10)])
 
 
 class ObjectNameTrait(HasTraits):
     value = ObjectName("abc")
 
 class TestObjectName(TraitTestBase):
     obj = ObjectNameTrait()
 
     _default_value = "abc"
     _good_values = ["a", "gh", "g9", "g_", "_G", u"a345_"]
     _bad_values = [1, "", u"€", "9g", "!", "#abc", "aj@", "a.b", "a()", "a[0]",
                                                             object(), object]
     if sys.version_info[0] < 3:
         _bad_values.append(u"þ")
     else:
         _good_values.append(u"þ")  # þ=1 is valid in Python 3 (PEP 3131).
 
 
 class DottedObjectNameTrait(HasTraits):
     value = DottedObjectName("a.b")
 
 class TestDottedObjectName(TraitTestBase):
     obj = DottedObjectNameTrait()
 
     _default_value = "a.b"
     _good_values = ["A", "y.t", "y765.__repr__", "os.path.join", u"os.path.join"]
     _bad_values = [1, u"abc.€", "_.@", ".", ".abc", "abc.", ".abc."]
     if sys.version_info[0] < 3:
         _bad_values.append(u"t.þ")
     else:
         _good_values.append(u"t.þ")
 
 
 class TCPAddressTrait(HasTraits):
 
     value = TCPAddress()
 
 class TestTCPAddress(TraitTestBase):
 
     obj = TCPAddressTrait()
 
     _default_value = ('127.0.0.1',0)
     _good_values = [('localhost',0),('192.168.0.1',1000),('www.google.com',80)]
     _bad_values = [(0,0),('localhost',10.0),('localhost',-1)]
 
 class ListTrait(HasTraits):
 
     value = List(Int)
 
 class TestList(TraitTestBase):
 
     obj = ListTrait()
 
     _default_value = []
-    _good_values = [[], [1], range(10)]
+    _good_values = [[], [1], list(range(10))]
     _bad_values = [10, [1,'a'], 'a', (1,2)]
 
 class LenListTrait(HasTraits):
 
     value = List(Int, [0], minlen=1, maxlen=2)
 
 class TestLenList(TraitTestBase):
 
     obj = LenListTrait()
 
     _default_value = [0]
-    _good_values = [[1], range(2)]
-    _bad_values = [10, [1,'a'], 'a', (1,2), [], range(3)]
+    _good_values = [[1], list(range(2))]
+    _bad_values = [10, [1,'a'], 'a', (1,2), [], list(range(3))]
 
 class TupleTrait(HasTraits):
 
     value = Tuple(Int)
 
 class TestTupleTrait(TraitTestBase):
 
     obj = TupleTrait()
 
     _default_value = None
     _good_values = [(1,), None,(0,)]
     _bad_values = [10, (1,2), [1],('a'), ()]
 
     def test_invalid_args(self):
         self.assertRaises(TypeError, Tuple, 5)
         self.assertRaises(TypeError, Tuple, default_value='hello')
         t = Tuple(Int, CBytes, default_value=(1,5))
 
 class LooseTupleTrait(HasTraits):
 
     value = Tuple((1,2,3))
 
 class TestLooseTupleTrait(TraitTestBase):
 
     obj = LooseTupleTrait()
 
     _default_value = (1,2,3)
     _good_values = [(1,), None, (0,), tuple(range(5)), tuple('hello'), ('a',5), ()]
     _bad_values = [10, 'hello', [1], []]
 
     def test_invalid_args(self):
         self.assertRaises(TypeError, Tuple, 5)
         self.assertRaises(TypeError, Tuple, default_value='hello')
         t = Tuple(Int, CBytes, default_value=(1,5))
 
 
 class MultiTupleTrait(HasTraits):
 
     value = Tuple(Int, Bytes, default_value=[99,b'bottles'])
 
 class TestMultiTuple(TraitTestBase):
 
     obj = MultiTupleTrait()
 
     _default_value = (99,b'bottles')
     _good_values = [(1,b'a'), (2,b'b')]
     _bad_values = ((),10, b'a', (1,b'a',3), (b'a',1), (1, u'a'))
 
 class CRegExpTrait(HasTraits):
 
     value = CRegExp(r'')
 
 class TestCRegExp(TraitTestBase):
 
     def coerce(self, value):
         return re.compile(value)
 
     obj = CRegExpTrait()
 
     _default_value = re.compile(r'')
     _good_values = [r'\d+', re.compile(r'\d+')]
     _bad_values = [r'(', None, ()]
 
 class DictTrait(HasTraits):
     value = Dict()
 
 def test_dict_assignment():
     d = dict()
     c = DictTrait()
     c.value = d
     d['a'] = 5
     nt.assert_equal(d, c.value)
     nt.assert_true(c.value is d)

IPython/utils/text.py

@@ -1,758 +1,758
 # encoding: utf-8
 """
 Utilities for working with strings and text.
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.utils.text
    :parts: 3
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import re
 import sys
 import textwrap
 from string import Formatter
 
 from IPython.external.path import path
 from IPython.testing.skipdoctest import skip_doctest_py3, skip_doctest
 from IPython.utils import py3compat
 
 #-----------------------------------------------------------------------------
 # Declarations
 #-----------------------------------------------------------------------------
 
 # datetime.strftime date format for ipython
 if sys.platform == 'win32':
     date_format = "%B %d, %Y"
 else:
     date_format = "%B %-d, %Y"
 
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 class LSString(str):
     """String derivative with a special access attributes.
 
     These are normal strings, but with the special attributes:
 
         .l (or .list) : value as list (split on newlines).
         .n (or .nlstr): original value (the string itself).
         .s (or .spstr): value as whitespace-separated string.
         .p (or .paths): list of path objects
 
     Any values which require transformations are computed only once and
     cached.
 
     Such strings are very useful to efficiently interact with the shell, which
     typically only understands whitespace-separated options for commands."""
 
     def get_list(self):
         try:
             return self.__list
         except AttributeError:
             self.__list = self.split('\n')
             return self.__list
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = self.replace('\n',' ')
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         return self
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self.split('\n') if os.path.exists(p)]
             return self.__paths
 
     p = paths = property(get_paths)
 
 # FIXME: We need to reimplement type specific displayhook and then add this
 # back as a custom printer. This should also be moved outside utils into the
 # core.
 
 # def print_lsstring(arg):
 #     """ Prettier (non-repr-like) and more informative printer for LSString """
 #     print "LSString (.p, .n, .l, .s available). Value:"
 #     print arg
 #
 #
 # print_lsstring = result_display.when_type(LSString)(print_lsstring)
 
 
 class SList(list):
     """List derivative with a special access attributes.
 
     These are normal lists, but with the special attributes:
 
         .l (or .list) : value as list (the list itself).
         .n (or .nlstr): value as a string, joined on newlines.
         .s (or .spstr): value as a string, joined on spaces.
         .p (or .paths): list of path objects
 
     Any values which require transformations are computed only once and
     cached."""
 
     def get_list(self):
         return self
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = ' '.join(self)
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         try:
             return self.__nlstr
         except AttributeError:
             self.__nlstr = '\n'.join(self)
             return self.__nlstr
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self if os.path.exists(p)]
             return self.__paths
 
     p = paths = property(get_paths)
 
     def grep(self, pattern, prune = False, field = None):
         """ Return all strings matching 'pattern' (a regex or callable)
 
         This is case-insensitive. If prune is true, return all items
         NOT matching the pattern.
 
         If field is specified, the match must occur in the specified
         whitespace-separated field.
 
         Examples::
 
             a.grep( lambda x: x.startswith('C') )
             a.grep('Cha.*log', prune=1)
             a.grep('chm', field=-1)
         """
 
         def match_target(s):
             if field is None:
                 return s
             parts = s.split()
             try:
                 tgt = parts[field]
                 return tgt
             except IndexError:
                 return ""
 
         if isinstance(pattern, py3compat.string_types):
             pred = lambda x : re.search(pattern, x, re.IGNORECASE)
         else:
             pred = pattern
         if not prune:
             return SList([el for el in self if pred(match_target(el))])
         else:
             return SList([el for el in self if not pred(match_target(el))])
 
     def fields(self, *fields):
         """ Collect whitespace-separated fields from string list
 
         Allows quick awk-like usage of string lists.
 
         Example data (in var a, created by 'a = !ls -l')::
             -rwxrwxrwx  1 ville None      18 Dec 14  2006 ChangeLog
             drwxrwxrwx+ 6 ville None       0 Oct 24 18:05 IPython
 
         a.fields(0) is ['-rwxrwxrwx', 'drwxrwxrwx+']
         a.fields(1,0) is ['1 -rwxrwxrwx', '6 drwxrwxrwx+']
         (note the joining by space).
         a.fields(-1) is ['ChangeLog', 'IPython']
 
         IndexErrors are ignored.
 
         Without args, fields() just split()'s the strings.
         """
         if len(fields) == 0:
             return [el.split() for el in self]
 
         res = SList()
         for el in [f.split() for f in self]:
             lineparts = []
 
             for fd in fields:
                 try:
                     lineparts.append(el[fd])
                 except IndexError:
                     pass
             if lineparts:
                 res.append(" ".join(lineparts))
 
         return res
 
     def sort(self,field= None,  nums = False):
         """ sort by specified fields (see fields())
 
         Example::
             a.sort(1, nums = True)
 
         Sorts a by second field, in numerical order (so that 21 > 3)
 
         """
 
         #decorate, sort, undecorate
         if field is not None:
             dsu = [[SList([line]).fields(field),  line] for line in self]
         else:
             dsu = [[line,  line] for line in self]
         if nums:
             for i in range(len(dsu)):
                 numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
                 try:
                     n = int(numstr)
                 except ValueError:
                     n = 0;
                 dsu[i][0] = n
 
 
         dsu.sort()
         return SList([t[1] for t in dsu])
 
 
 # FIXME: We need to reimplement type specific displayhook and then add this
 # back as a custom printer. This should also be moved outside utils into the
 # core.
 
 # def print_slist(arg):
 #     """ Prettier (non-repr-like) and more informative printer for SList """
 #     print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
 #     if hasattr(arg,  'hideonce') and arg.hideonce:
 #         arg.hideonce = False
 #         return
 #
 #     nlprint(arg)   # This was a nested list printer, now removed.
 #
 # print_slist = result_display.when_type(SList)(print_slist)
 
 
 def indent(instr,nspaces=4, ntabs=0, flatten=False):
     """Indent a string a given number of spaces or tabstops.
 
     indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
 
     Parameters
     ----------
 
     instr : basestring
         The string to be indented.
     nspaces : int (default: 4)
         The number of spaces to be indented.
     ntabs : int (default: 0)
         The number of tabs to be indented.
     flatten : bool (default: False)
         Whether to scrub existing indentation.  If True, all lines will be
         aligned to the same indentation.  If False, existing indentation will
         be strictly increased.
 
     Returns
     -------
 
     str|unicode : string indented by ntabs and nspaces.
 
     """
     if instr is None:
         return
     ind = '\t'*ntabs+' '*nspaces
     if flatten:
         pat = re.compile(r'^\s*', re.MULTILINE)
     else:
         pat = re.compile(r'^', re.MULTILINE)
     outstr = re.sub(pat, ind, instr)
     if outstr.endswith(os.linesep+ind):
         return outstr[:-len(ind)]
     else:
         return outstr
 
 
 def list_strings(arg):
     """Always return a list of strings, given a string or list of strings
     as input.
 
     :Examples:
 
         In [7]: list_strings('A single string')
         Out[7]: ['A single string']
 
         In [8]: list_strings(['A single string in a list'])
         Out[8]: ['A single string in a list']
 
         In [9]: list_strings(['A','list','of','strings'])
         Out[9]: ['A', 'list', 'of', 'strings']
     """
 
     if isinstance(arg, py3compat.string_types): return [arg]
     else: return arg
 
 
 def marquee(txt='',width=78,mark='*'):
     """Return the input string centered in a 'marquee'.
 
     :Examples:
 
         In [16]: marquee('A test',40)
         Out[16]: '**************** A test ****************'
 
         In [17]: marquee('A test',40,'-')
         Out[17]: '---------------- A test ----------------'
 
         In [18]: marquee('A test',40,' ')
         Out[18]: '                 A test                 '
 
     """
     if not txt:
         return (mark*width)[:width]
     nmark = (width-len(txt)-2)//len(mark)//2
     if nmark < 0: nmark =0
     marks = mark*nmark
     return '%s %s %s' % (marks,txt,marks)
 
 
 ini_spaces_re = re.compile(r'^(\s+)')
 
 def num_ini_spaces(strng):
     """Return the number of initial spaces in a string"""
 
     ini_spaces = ini_spaces_re.match(strng)
     if ini_spaces:
         return ini_spaces.end()
     else:
         return 0
 
 
 def format_screen(strng):
     """Format a string for screen printing.
 
     This removes some latex-type format codes."""
     # Paragraph continue
     par_re = re.compile(r'\\$',re.MULTILINE)
     strng = par_re.sub('',strng)
     return strng
 
 
 def dedent(text):
     """Equivalent of textwrap.dedent that ignores unindented first line.
 
     This means it will still dedent strings like:
     '''foo
     is a bar
     '''
 
     For use in wrap_paragraphs.
     """
 
     if text.startswith('\n'):
         # text starts with blank line, don't ignore the first line
         return textwrap.dedent(text)
 
     # split first line
     splits = text.split('\n',1)
     if len(splits) == 1:
         # only one line
         return textwrap.dedent(text)
 
     first, rest = splits
     # dedent everything but the first line
     rest = textwrap.dedent(rest)
     return '\n'.join([first, rest])
 
 
 def wrap_paragraphs(text, ncols=80):
     """Wrap multiple paragraphs to fit a specified width.
 
     This is equivalent to textwrap.wrap, but with support for multiple
     paragraphs, as separated by empty lines.
 
     Returns
     -------
 
     list of complete paragraphs, wrapped to fill `ncols` columns.
     """
     paragraph_re = re.compile(r'\n(\s*\n)+', re.MULTILINE)
     text = dedent(text).strip()
     paragraphs = paragraph_re.split(text)[::2] # every other entry is space
     out_ps = []
     indent_re = re.compile(r'\n\s+', re.MULTILINE)
     for p in paragraphs:
         # presume indentation that survives dedent is meaningful formatting,
         # so don't fill unless text is flush.
         if indent_re.search(p) is None:
             # wrap paragraph
             p = textwrap.fill(p, ncols)
         out_ps.append(p)
     return out_ps
 
 
 def long_substr(data):
     """Return the longest common substring in a list of strings.
     
     Credit: http://stackoverflow.com/questions/2892931/longest-common-substring-from-more-than-two-strings-python
     """
     substr = ''
     if len(data) > 1 and len(data[0]) > 0:
         for i in range(len(data[0])):
             for j in range(len(data[0])-i+1):
                 if j > len(substr) and all(data[0][i:i+j] in x for x in data):
                     substr = data[0][i:i+j]
     elif len(data) == 1:
         substr = data[0]
     return substr
 
 
 def strip_email_quotes(text):
     """Strip leading email quotation characters ('>').
 
     Removes any combination of leading '>' interspersed with whitespace that
     appears *identically* in all lines of the input text.
 
     Parameters
     ----------
     text : str
 
     Examples
     --------
 
     Simple uses::
 
         In [2]: strip_email_quotes('> > text')
         Out[2]: 'text'
 
         In [3]: strip_email_quotes('> > text\\n> > more')
         Out[3]: 'text\\nmore'
 
     Note how only the common prefix that appears in all lines is stripped::
 
         In [4]: strip_email_quotes('> > text\\n> > more\\n> more...')
         Out[4]: '> text\\n> more\\nmore...'
 
     So if any line has no quote marks ('>') , then none are stripped from any
     of them ::
     
         In [5]: strip_email_quotes('> > text\\n> > more\\nlast different')
         Out[5]: '> > text\\n> > more\\nlast different'
     """
     lines = text.splitlines()
     matches = set()
     for line in lines:
         prefix = re.match(r'^(\s*>[ >]*)', line)
         if prefix:
             matches.add(prefix.group(1))
         else:
             break
     else:
         prefix = long_substr(list(matches))
         if prefix:
             strip = len(prefix)
             text = '\n'.join([ ln[strip:] for ln in lines])
     return text
 
 
 class EvalFormatter(Formatter):
     """A String Formatter that allows evaluation of simple expressions.
     
     Note that this version interprets a : as specifying a format string (as per
     standard string formatting), so if slicing is required, you must explicitly
     create a slice.
     
     This is to be used in templating cases, such as the parallel batch
     script templates, where simple arithmetic on arguments is useful.
 
     Examples
     --------
     
     In  [1]: f = EvalFormatter()
     In  [2]: f.format('{n//4}', n=8)
     Out [2]: '2'
     
     In  [3]: f.format("{greeting[slice(2,4)]}", greeting="Hello")
     Out [3]: 'll'
     """
     def get_field(self, name, args, kwargs):
         v = eval(name, kwargs)
         return v, name
 
 
 @skip_doctest_py3
 class FullEvalFormatter(Formatter):
     """A String Formatter that allows evaluation of simple expressions.
     
     Any time a format key is not found in the kwargs,
     it will be tried as an expression in the kwargs namespace.
     
     Note that this version allows slicing using [1:2], so you cannot specify
     a format string. Use :class:`EvalFormatter` to permit format strings.
     
     Examples
     --------
     
     In [1]: f = FullEvalFormatter()
     In [2]: f.format('{n//4}', n=8)
     Out[2]: u'2'
     
     In [3]: f.format('{list(range(5))[2:4]}')
     Out[3]: u'[2, 3]'
 
     In [4]: f.format('{3*2}')
     Out[4]: u'6'
     """
     # copied from Formatter._vformat with minor changes to allow eval
     # and replace the format_spec code with slicing
     def _vformat(self, format_string, args, kwargs, used_args, recursion_depth):
         if recursion_depth < 0:
             raise ValueError('Max string recursion exceeded')
         result = []
         for literal_text, field_name, format_spec, conversion in \
                 self.parse(format_string):
 
             # output the literal text
             if literal_text:
                 result.append(literal_text)
 
             # if there's a field, output it
             if field_name is not None:
                 # this is some markup, find the object and do
                 # the formatting
 
                 if format_spec:
                     # override format spec, to allow slicing:
                     field_name = ':'.join([field_name, format_spec])
 
                 # eval the contents of the field for the object
                 # to be formatted
                 obj = eval(field_name, kwargs)
 
                 # do any conversion on the resulting object
                 obj = self.convert_field(obj, conversion)
 
                 # format the object and append to the result
                 result.append(self.format_field(obj, ''))
 
         return u''.join(py3compat.cast_unicode(s) for s in result)
 
 
 @skip_doctest_py3
 class DollarFormatter(FullEvalFormatter):
     """Formatter allowing Itpl style $foo replacement, for names and attribute
     access only. Standard {foo} replacement also works, and allows full
     evaluation of its arguments.
 
     Examples
     --------
     In [1]: f = DollarFormatter()
     In [2]: f.format('{n//4}', n=8)
     Out[2]: u'2'
     
     In [3]: f.format('23 * 76 is $result', result=23*76)
     Out[3]: u'23 * 76 is 1748'
     
     In [4]: f.format('$a or {b}', a=1, b=2)
     Out[4]: u'1 or 2'
     """
     _dollar_pattern = re.compile("(.*?)\$(\$?[\w\.]+)")
     def parse(self, fmt_string):
         for literal_txt, field_name, format_spec, conversion \
                     in Formatter.parse(self, fmt_string):
             
             # Find $foo patterns in the literal text.
             continue_from = 0
             txt = ""
             for m in self._dollar_pattern.finditer(literal_txt):
                 new_txt, new_field = m.group(1,2)
                 # $$foo --> $foo
                 if new_field.startswith("$"):
                     txt += new_txt + new_field
                 else:
                     yield (txt + new_txt, new_field, "", None)
                     txt = ""
                 continue_from = m.end()
             
             # Re-yield the {foo} style pattern
             yield (txt + literal_txt[continue_from:], field_name, format_spec, conversion)
 
 #-----------------------------------------------------------------------------
 # Utils to columnize a list of string
 #-----------------------------------------------------------------------------
 
 def _chunks(l, n):
     """Yield successive n-sized chunks from l."""
     for i in py3compat.xrange(0, len(l), n):
         yield l[i:i+n]
 
 
 def _find_optimal(rlist , separator_size=2 , displaywidth=80):
     """Calculate optimal info to columnize a list of string"""
     for nrow in range(1, len(rlist)+1) :
-        chk = map(max,_chunks(rlist, nrow))
+        chk = list(map(max,_chunks(rlist, nrow)))
         sumlength = sum(chk)
         ncols = len(chk)
         if sumlength+separator_size*(ncols-1) <= displaywidth :
             break;
     return {'columns_numbers' : ncols,
             'optimal_separator_width':(displaywidth - sumlength)/(ncols-1) if (ncols -1) else 0,
             'rows_numbers' : nrow,
             'columns_width' : chk
            }
 
 
 def _get_or_default(mylist, i, default=None):
     """return list item number, or default if don't exist"""
     if i >= len(mylist):
         return default
     else :
         return mylist[i]
 
 
 @skip_doctest
 def compute_item_matrix(items, empty=None, *args, **kwargs) :
     """Returns a nested list, and info to columnize items
 
     Parameters
     ----------
 
     items :
         list of strings to columize
     empty : (default None)
         default value to fill list if needed
     separator_size : int (default=2)
         How much caracters will be used as a separation between each columns.
     displaywidth : int (default=80)
         The width of the area onto wich the columns should enter
 
     Returns
     -------
 
     Returns a tuple of (strings_matrix, dict_info)
 
     strings_matrix :
 
         nested list of string, the outer most list contains as many list as
         rows, the innermost lists have each as many element as colums. If the
         total number of elements in `items` does not equal the product of
         rows*columns, the last element of some lists are filled with `None`.
 
     dict_info :
         some info to make columnize easier:
 
         columns_numbers : number of columns
         rows_numbers    : number of rows
         columns_width   : list of with of each columns
         optimal_separator_width : best separator width between columns
 
     Examples
     --------
 
     In [1]: l = ['aaa','b','cc','d','eeeee','f','g','h','i','j','k','l']
        ...: compute_item_matrix(l,displaywidth=12)
     Out[1]:
         ([['aaa', 'f', 'k'],
         ['b', 'g', 'l'],
         ['cc', 'h', None],
         ['d', 'i', None],
         ['eeeee', 'j', None]],
         {'columns_numbers': 3,
         'columns_width': [5, 1, 1],
         'optimal_separator_width': 2,
         'rows_numbers': 5})
 
     """
-    info = _find_optimal(map(len, items), *args, **kwargs)
+    info = _find_optimal(list(map(len, items)), *args, **kwargs)
     nrow, ncol = info['rows_numbers'], info['columns_numbers']
     return ([[ _get_or_default(items, c*nrow+i, default=empty) for c in range(ncol) ] for i in range(nrow) ], info)
 
 
 def columnize(items, separator='  ', displaywidth=80):
     """ Transform a list of strings into a single string with columns.
 
     Parameters
     ----------
     items : sequence of strings
         The strings to process.
 
     separator : str, optional [default is two spaces]
         The string that separates columns.
 
     displaywidth : int, optional [default is 80]
         Width of the display in number of characters.
 
     Returns
     -------
     The formatted string.
     """
     if not items :
         return '\n'
     matrix, info = compute_item_matrix(items, separator_size=len(separator), displaywidth=displaywidth)
     fmatrix = [filter(None, x) for x in matrix]
     sjoin = lambda x : separator.join([ y.ljust(w, ' ') for y, w in zip(x, info['columns_width'])])
     return '\n'.join(map(sjoin, fmatrix))+'\n'
 
 
 def get_text_list(list_, last_sep=' and ', sep=", ", wrap_item_with=""):
     """
     Return a string with a natural enumeration of items
 
     >>> get_text_list(['a', 'b', 'c', 'd'])
     'a, b, c and d'
     >>> get_text_list(['a', 'b', 'c'], ' or ')
     'a, b or c'
     >>> get_text_list(['a', 'b', 'c'], ', ')
     'a, b, c'
     >>> get_text_list(['a', 'b'], ' or ')
     'a or b'
     >>> get_text_list(['a'])
     'a'
     >>> get_text_list([])
     ''
     >>> get_text_list(['a', 'b'], wrap_item_with="`")
     '`a` and `b`'
     >>> get_text_list(['a', 'b', 'c', 'd'], " = ", sep=" + ")
     'a + b + c = d'
     """
     if len(list_) == 0:
         return ''
     if wrap_item_with:
         list_ = ['%s%s%s' % (wrap_item_with, item, wrap_item_with) for
                  item in list_]
     if len(list_) == 1:
         return list_[0]
     return '%s%s%s' % (
         sep.join(i for i in list_[:-1]),
         last_sep, list_[-1])
  No newline at end of file