Merge branch 'takowl-ipy3-preparation' into trunk

This branch cleans up our trunk to make a 2to3 run be as painless as
possible, while keeping our code fully 2.6-compatible.

Closes gh-159 (pull request).

.gitignore

@@ -1,6 +1,8 b''
 docs/dist
 docs/build/*
 docs/source/api/generated
 *.pyc
 build
 *.egg-info
+*.py~
+*.bak

IPython/__init__.py

@@ -1,54 +1,54 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 IPython.
 
 IPython is a set of tools for interactive and exploratory computing in Python.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import absolute_import
 
 import os
 import sys
 
 #-----------------------------------------------------------------------------
 # Setup everything
 #-----------------------------------------------------------------------------
 
 if sys.version[0:3] < '2.6':
     raise ImportError('Python Version 2.6 or above is required for IPython.')
 
 
 # Make it easy to import extensions - they are always directly on pythonpath.
 # Therefore, non-IPython modules can be added to extensions directory.
 # This should probably be in ipapp.py.
 sys.path.append(os.path.join(os.path.dirname(__file__), "extensions"))
 
 #-----------------------------------------------------------------------------
 # Setup the top level names
 #-----------------------------------------------------------------------------
 
 from .config.loader import Config
 from .core import release
 from .core.application import Application
 from .frontend.terminal.embed import embed
 from .core.error import TryNext
 from .core.interactiveshell import InteractiveShell
 from .testing import test
 
 # Release data
 __author__ = ''
-for author, email in release.authors.values():
+for author, email in release.authors.itervalues():
     __author__ += author + ' <' + email + '>\n'
 __license__  = release.license
 __version__  = release.version
 __revision__ = release.revision

IPython/config/configurable.py

@@ -1,139 +1,139 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 A base class for objects that are configurable.
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from copy import deepcopy
 import datetime
 from weakref import WeakValueDictionary
 
 from IPython.utils.importstring import import_item
 from loader import Config
 from IPython.utils.traitlets import HasTraits, Instance
 
 
 #-----------------------------------------------------------------------------
 # Helper classes for Configurables
 #-----------------------------------------------------------------------------
 
 
 class ConfigurableError(Exception):
     pass
 
 
 #-----------------------------------------------------------------------------
 # Configurable implementation
 #-----------------------------------------------------------------------------
 
 
 class Configurable(HasTraits):
 
     config = Instance(Config,(),{})
     created = None
 
     def __init__(self, **kwargs):
         """Create a conigurable given a config config.
 
         Parameters
         ----------
         config : Config
             If this is empty, default values are used. If config is a 
             :class:`Config` instance, it will be used to configure the
             instance.
         
         Notes
         -----
         Subclasses of Configurable must call the :meth:`__init__` method of
         :class:`Configurable` *before* doing anything else and using 
         :func:`super`::
         
             class MyConfigurable(Configurable):
                 def __init__(self, config=None):
                     super(MyConfigurable, self).__init__(config)
                     # Then any other code you need to finish initialization.
 
         This ensures that instances will be configured properly.
         """
         config = kwargs.pop('config', None)
         if config is not None:
             # We used to deepcopy, but for now we are trying to just save
             # by reference.  This *could* have side effects as all components
             # will share config. In fact, I did find such a side effect in
             # _config_changed below. If a config attribute value was a mutable type
             # all instances of a component were getting the same copy, effectively
             # making that a class attribute.
             # self.config = deepcopy(config)
             self.config = config
         # This should go second so individual keyword arguments override 
         # the values in config.
         super(Configurable, self).__init__(**kwargs)
         self.created = datetime.datetime.now()
 
     #-------------------------------------------------------------------------
     # Static trait notifiations
     #-------------------------------------------------------------------------
 
     def _config_changed(self, name, old, new):
         """Update all the class traits having ``config=True`` as metadata.
 
         For any class trait with a ``config`` metadata attribute that is
         ``True``, we update the trait with the value of the corresponding
         config entry.
         """
         # Get all traits with a config metadata entry that is True
         traits = self.traits(config=True)
 
         # We auto-load config section for this class as well as any parent
         # classes that are Configurable subclasses.  This starts with Configurable
         # and works down the mro loading the config for each section.
         section_names = [cls.__name__ for cls in \
             reversed(self.__class__.__mro__) if 
             issubclass(cls, Configurable) and issubclass(self.__class__, cls)]
 
         for sname in section_names:
             # Don't do a blind getattr as that would cause the config to 
             # dynamically create the section with name self.__class__.__name__.
             if new._has_section(sname):
                 my_config = new[sname]
-                for k, v in traits.items():
+                for k, v in traits.iteritems():
                     # Don't allow traitlets with config=True to start with
                     # uppercase.  Otherwise, they are confused with Config
                     # subsections.  But, developers shouldn't have uppercase
                     # attributes anyways! (PEP 6)
                     if k[0].upper()==k[0] and not k.startswith('_'):
                         raise ConfigurableError('Configurable traitlets with '
                         'config=True must start with a lowercase so they are '
                         'not confused with Config subsections: %s.%s' % \
                         (self.__class__.__name__, k))
                     try:
                         # Here we grab the value from the config
                         # If k has the naming convention of a config
                         # section, it will be auto created.
                         config_value = my_config[k]
                     except KeyError:
                         pass
                     else:
                         # print "Setting %s.%s from %s.%s=%r" % \
                         #     (self.__class__.__name__,k,sname,k,config_value)
                         # We have to do a deepcopy here if we don't deepcopy the entire
                         # config object. If we don't, a mutable config_value will be
                         # shared by all instances, effectively making it a class attribute.
                         setattr(self, k, deepcopy(config_value))
 

IPython/config/loader.py

@@ -1,370 +1,373 b''
+# -*- coding: utf-8 -*-
 # coding: utf-8
 """A simple configuration system.
 
 Authors
 -------
 * Brian Granger
 * Fernando Perez
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import __builtin__
 import os
 import sys
 
 from IPython.external import argparse
 from IPython.utils.path import filefind
 
 #-----------------------------------------------------------------------------
 # Exceptions
 #-----------------------------------------------------------------------------
 
 
 class ConfigError(Exception):
     pass
 
 
 class ConfigLoaderError(ConfigError):
     pass
 
 #-----------------------------------------------------------------------------
 # Argparse fix
 #-----------------------------------------------------------------------------
 
 # Unfortunately argparse by default prints help messages to stderr instead of
 # stdout.  This makes it annoying to capture long help screens at the command
 # line, since one must know how to pipe stderr, which many users don't know how
 # to do.  So we override the print_help method with one that defaults to
 # stdout and use our class instead.
 
 class ArgumentParser(argparse.ArgumentParser):
     """Simple argparse subclass that prints help to stdout by default."""
     
     def print_help(self, file=None):
         if file is None:
             file = sys.stdout
         return super(ArgumentParser, self).print_help(file)
     
     print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
     
 #-----------------------------------------------------------------------------
 # Config class for holding config information
 #-----------------------------------------------------------------------------
 
 
 class Config(dict):
     """An attribute based dict that can do smart merges."""
 
     def __init__(self, *args, **kwds):
         dict.__init__(self, *args, **kwds)
         # This sets self.__dict__ = self, but it has to be done this way
         # because we are also overriding __setattr__.
         dict.__setattr__(self, '__dict__', self)
 
     def _merge(self, other):
         to_update = {}
-        for k, v in other.items():
+        for k, v in other.iteritems():
             if not self.has_key(k):
                 to_update[k] = v
             else: # I have this key
                 if isinstance(v, Config):
                     # Recursively merge common sub Configs
                     self[k]._merge(v)
                 else:
                     # Plain updates for non-Configs
                     to_update[k] = v
 
         self.update(to_update)
 
     def _is_section_key(self, key):
         if key[0].upper()==key[0] and not key.startswith('_'):
             return True
         else:
             return False
 
-    def has_key(self, key):
+    def __contains__(self, key):
         if self._is_section_key(key):
             return True
         else:
-            return dict.has_key(self, key)
+            return super(Config, self).__contains__(key)
+    # .has_key is deprecated for dictionaries.
+    has_key = __contains__
 
     def _has_section(self, key):
         if self._is_section_key(key):
-            if dict.has_key(self, key):
+            if super(Config, self).__contains__(key):
                 return True
         return False
 
     def copy(self):
         return type(self)(dict.copy(self))
 
     def __copy__(self):
         return self.copy()
 
     def __deepcopy__(self, memo):
         import copy
         return type(self)(copy.deepcopy(self.items()))
 
     def __getitem__(self, key):
         # Because we use this for an exec namespace, we need to delegate
         # the lookup of names in __builtin__ to itself.  This means
         # that you can't have section or attribute names that are 
         # builtins.
         try:
             return getattr(__builtin__, key)
         except AttributeError:
             pass
         if self._is_section_key(key):
             try:
                 return dict.__getitem__(self, key)
             except KeyError:
                 c = Config()
                 dict.__setitem__(self, key, c)
                 return c
         else:
             return dict.__getitem__(self, key)
 
     def __setitem__(self, key, value):
         # Don't allow names in __builtin__ to be modified.
         if hasattr(__builtin__, key):
             raise ConfigError('Config variable names cannot have the same name '
                               'as a Python builtin: %s' % key)
         if self._is_section_key(key):
             if not isinstance(value, Config):
                 raise ValueError('values whose keys begin with an uppercase '
                                  'char must be Config instances: %r, %r' % (key, value))
         else:
             dict.__setitem__(self, key, value)
 
     def __getattr__(self, key):
         try:
             return self.__getitem__(key)
         except KeyError, e:
             raise AttributeError(e)
 
     def __setattr__(self, key, value):
         try:
             self.__setitem__(key, value)
         except KeyError, e:
             raise AttributeError(e)
 
     def __delattr__(self, key):
         try:
             dict.__delitem__(self, key)
         except KeyError, e:
             raise AttributeError(e)
 
 
 #-----------------------------------------------------------------------------
 # Config loading classes
 #-----------------------------------------------------------------------------
 
 
 class ConfigLoader(object):
     """A object for loading configurations from just about anywhere.
     
     The resulting configuration is packaged as a :class:`Struct`.
     
     Notes
     -----
     A :class:`ConfigLoader` does one thing: load a config from a source 
     (file, command line arguments) and returns the data as a :class:`Struct`.
     There are lots of things that :class:`ConfigLoader` does not do.  It does
     not implement complex logic for finding config files.  It does not handle
     default values or merge multiple configs.  These things need to be 
     handled elsewhere.
     """
 
     def __init__(self):
         """A base class for config loaders.
         
         Examples
         --------
         
         >>> cl = ConfigLoader()
         >>> config = cl.load_config()
         >>> config
         {}
         """
         self.clear()
 
     def clear(self):
         self.config = Config()
 
     def load_config(self):
         """Load a config from somewhere, return a :class:`Config` instance.
         
         Usually, this will cause self.config to be set and then returned.
         However, in most cases, :meth:`ConfigLoader.clear` should be called
         to erase any previous state.
         """
         self.clear()
         return self.config
 
 
 class FileConfigLoader(ConfigLoader):
     """A base class for file based configurations.
 
     As we add more file based config loaders, the common logic should go
     here.
     """
     pass
 
 
 class PyFileConfigLoader(FileConfigLoader):
     """A config loader for pure python files.
     
     This calls execfile on a plain python file and looks for attributes
     that are all caps.  These attribute are added to the config Struct.
     """
 
     def __init__(self, filename, path=None):
         """Build a config loader for a filename and path.
 
         Parameters
         ----------
         filename : str
             The file name of the config file.
         path : str, list, tuple
             The path to search for the config file on, or a sequence of
             paths to try in order.
         """
         super(PyFileConfigLoader, self).__init__()
         self.filename = filename
         self.path = path
         self.full_filename = ''
         self.data = None
 
     def load_config(self):
         """Load the config from a file and return it as a Struct."""
         self.clear()
         self._find_file()
         self._read_file_as_dict()
         self._convert_to_config()
         return self.config
 
     def _find_file(self):
         """Try to find the file by searching the paths."""
         self.full_filename = filefind(self.filename, self.path)
 
     def _read_file_as_dict(self):
         """Load the config file into self.config, with recursive loading."""
         # This closure is made available in the namespace that is used
         # to exec the config file.  This allows users to call
         # load_subconfig('myconfig.py') to load config files recursively.
         # It needs to be a closure because it has references to self.path
         # and self.config.  The sub-config is loaded with the same path
         # as the parent, but it uses an empty config which is then merged
         # with the parents.
         def load_subconfig(fname):
             loader = PyFileConfigLoader(fname, self.path)
             try:
                 sub_config = loader.load_config()
             except IOError:
                 # Pass silently if the sub config is not there. This happens
                 # when a user us using a profile, but not the default config.
                 pass
             else:
                 self.config._merge(sub_config)
 
         # Again, this needs to be a closure and should be used in config
         # files to get the config being loaded.
         def get_config():
             return self.config
 
         namespace = dict(load_subconfig=load_subconfig, get_config=get_config)
         execfile(self.full_filename, namespace)
 
     def _convert_to_config(self):
         if self.data is None:
             ConfigLoaderError('self.data does not exist')
 
 
 class CommandLineConfigLoader(ConfigLoader):
     """A config loader for command line arguments.
 
     As we add more command line based loaders, the common logic should go
     here.
     """
 
 
 class ArgParseConfigLoader(CommandLineConfigLoader):
 
     def __init__(self, argv=None, *parser_args, **parser_kw):
         """Create a config loader for use with argparse.
 
         Parameters
         ----------
 
         argv : optional, list
           If given, used to read command-line arguments from, otherwise
           sys.argv[1:] is used.
 
         parser_args : tuple
           A tuple of positional arguments that will be passed to the
           constructor of :class:`argparse.ArgumentParser`.
 
         parser_kw : dict
           A tuple of keyword arguments that will be passed to the
           constructor of :class:`argparse.ArgumentParser`.
         """
         super(CommandLineConfigLoader, self).__init__()
         if argv == None:
             argv = sys.argv[1:]
         self.argv = argv
         self.parser_args = parser_args
         kwargs = dict(argument_default=argparse.SUPPRESS)
         kwargs.update(parser_kw)
         self.parser_kw = kwargs
 
     def load_config(self, args=None):
         """Parse command line arguments and return as a Struct.
 
         Parameters
         ----------
 
         args : optional, list
           If given, a list with the structure of sys.argv[1:] to parse
           arguments from. If not given, the instance's self.argv attribute
           (given at construction time) is used."""
         self.clear()
         if args is None:
             args = self.argv
         self._create_parser()
         self._parse_args(args)
         self._convert_to_config()
         return self.config
 
     def get_extra_args(self):
         if hasattr(self, 'extra_args'):
             return self.extra_args
         else:
             return []
 
     def _create_parser(self):
         self.parser = ArgumentParser(*self.parser_args, **self.parser_kw)
         self._add_arguments()
 
     def _add_arguments(self):
         raise NotImplementedError("subclasses must implement _add_arguments")
 
     def _parse_args(self, args):
         """self.parser->self.parsed_data""" 
         self.parsed_data, self.extra_args = self.parser.parse_known_args(args)
 
     def _convert_to_config(self):
         """self.parsed_data->self.config"""
-        for k, v in vars(self.parsed_data).items():
+        for k, v in vars(self.parsed_data).iteritems():
             exec_str = 'self.config.' + k + '= v'
             exec exec_str in locals(), globals()
 
 

IPython/core/completerlib.py

@@ -1,346 +1,346 b''
 """Implementations for various useful completers.
 
 These are all loaded by default by IPython.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010 The IPython Development Team.
 #
 #  Distributed under the terms of the BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Stdlib imports
 import glob
 import inspect
 import os
 import re
 import shlex
 import sys
 
 # Third-party imports
 from time import time
 from zipimport import zipimporter
 
 # Our own imports
 from IPython.core.completer import expand_user, compress_user
 from IPython.core.error import TryNext
 
 # FIXME: this should be pulled in with the right call via the component system
 from IPython.core.ipapi import get as get_ipython
 
 #-----------------------------------------------------------------------------
 # Globals and constants
 #-----------------------------------------------------------------------------
 
 # Time in seconds after which the rootmodules will be stored permanently in the
 # ipython ip.db database (kept in the user's .ipython dir).
 TIMEOUT_STORAGE = 2
 
 # Time in seconds after which we give up
 TIMEOUT_GIVEUP = 20
 
 # Regular expression for the python import statement
 import_re = re.compile(r'.*(\.so|\.py[cod]?)$')
 
 # RE for the ipython %run command (python + ipython scripts)
 magic_run_re = re.compile(r'.*(\.ipy|\.py[w]?)$')
 
 #-----------------------------------------------------------------------------
 # Local utilities
 #-----------------------------------------------------------------------------
 
 def shlex_split(x):
     """Helper function to split lines into segments.
     """
     # shlex.split raises an exception if there is a syntax error in sh syntax
     # for example if no closing " is found. This function keeps dropping the
     # last character of the line until shlex.split does not raise
     # an exception. It adds end of the line to the result of shlex.split
     #
     # Example:
     # %run "c:/python -> ['%run','"c:/python']
 
     # shlex.split has unicode bugs, so encode first to str
     if isinstance(x, unicode):
         x = x.encode(sys.stdin.encoding)
 
     endofline = []
     while x != '':
         try:
             comps = shlex.split(x)
             if len(endofline) >= 1:
                 comps.append(''.join(endofline))
             return comps
         
         except ValueError:
             endofline = [x[-1:]]+endofline
             x = x[:-1]
 
     return [''.join(endofline)]
 
 def module_list(path):
     """
     Return the list containing the names of the modules available in the given
     folder.
     """
 
     if os.path.isdir(path):
         folder_list = os.listdir(path)
     elif path.endswith('.egg'):
         try:
             folder_list = [f for f in zipimporter(path)._files]
         except:
             folder_list = []
     else:
         folder_list = []
 
     if not folder_list:
         return []
 
     # A few local constants to be used in loops below
     isfile = os.path.isfile
     pjoin = os.path.join
     basename = os.path.basename
     
     # Now find actual path matches for packages or modules
     folder_list = [p for p in folder_list
                    if isfile(pjoin(path, p,'__init__.py'))
                    or import_re.match(p) ]
 
     return [basename(p).split('.')[0] for p in folder_list]
 
 def get_root_modules():
     """
     Returns a list containing the names of all the modules available in the
     folders of the pythonpath.
     """
     ip = get_ipython()
 
     if 'rootmodules' in ip.db:
         return ip.db['rootmodules']
     
     t = time()
     store = False
     modules = list(sys.builtin_module_names)
     for path in sys.path:
         modules += module_list(path)        
         if time() - t >= TIMEOUT_STORAGE and not store:
             store = True
             print("\nCaching the list of root modules, please wait!")
             print("(This will only be done once - type '%rehashx' to "
                   "reset cache!)\n")
             sys.stdout.flush()
         if time() - t > TIMEOUT_GIVEUP:
             print("This is taking too long, we give up.\n")
             ip.db['rootmodules'] = []
             return []
     
     modules = set(modules)
     if '__init__' in modules:
         modules.remove('__init__')
     modules = list(modules)
     if store:
         ip.db['rootmodules'] = modules
     return modules
 
 
 def is_importable(module, attr, only_modules):
     if only_modules:
         return inspect.ismodule(getattr(module, attr))
     else:
         return not(attr[:2] == '__' and attr[-2:] == '__')
 
 
 def try_import(mod, only_modules=False):
     try:
         m = __import__(mod)
     except:
         return []
     mods = mod.split('.')
     for module in mods[1:]:
         m = getattr(m, module)
 
     m_is_init = hasattr(m, '__file__') and '__init__' in m.__file__
 
     completions = []
     if (not hasattr(m, '__file__')) or (not only_modules) or m_is_init:
         completions.extend( [attr for attr in dir(m) if
                              is_importable(m, attr, only_modules)])
         
     completions.extend(getattr(m, '__all__', []))
     if m_is_init:
         completions.extend(module_list(os.path.dirname(m.__file__)))
     completions = set(completions)
     if '__init__' in completions:
         completions.remove('__init__')
     return list(completions)
 
 
 #-----------------------------------------------------------------------------
 # Completion-related functions.
 #-----------------------------------------------------------------------------
 
 def quick_completer(cmd, completions):
     """ Easily create a trivial completer for a command.
 
     Takes either a list of completions, or all completions in string (that will
     be split on whitespace).
     
     Example::
     
         [d:\ipython]|1> import ipy_completers
         [d:\ipython]|2> ipy_completers.quick_completer('foo', ['bar','baz'])
         [d:\ipython]|3> foo b<TAB>
         bar baz
         [d:\ipython]|3> foo ba
     """
     
     if isinstance(completions, basestring):
         completions = completions.split()
 
     def do_complete(self, event):
         return completions
     
     get_ipython().set_hook('complete_command',do_complete, str_key = cmd)
 
 
 def module_completion(line):
     """
     Returns a list containing the completion possibilities for an import line.
 
     The line looks like this :
     'import xml.d'
     'from xml.dom import'
     """
 
     words = line.split(' ')
     nwords = len(words)
 
     # from whatever <tab> -> 'import '
     if nwords == 3 and words[0] == 'from':
         return ['import ']
     
     # 'from xy<tab>' or 'import xy<tab>'
     if nwords < 3 and (words[0] in ['import','from']) :
         if nwords == 1:
             return get_root_modules()
         mod = words[1].split('.')
         if len(mod) < 2:
             return get_root_modules()
         completion_list = try_import('.'.join(mod[:-1]), True)
         return ['.'.join(mod[:-1] + [el]) for el in completion_list]
     
     # 'from xyz import abc<tab>'
     if nwords >= 3 and words[0] == 'from':
         mod = words[1]
         return try_import(mod)
 
 #-----------------------------------------------------------------------------
 # Completers
 #-----------------------------------------------------------------------------
 # These all have the func(self, event) signature to be used as custom
 # completers
 
 def module_completer(self,event):
     """Give completions after user has typed 'import ...' or 'from ...'"""
 
     # This works in all versions of python.  While 2.5 has
     # pkgutil.walk_packages(), that particular routine is fairly dangerous,
     # since it imports *EVERYTHING* on sys.path.  That is: a) very slow b) full
     # of possibly problematic side effects.
     # This search the folders in the sys.path for available modules.
 
     return module_completion(event.line)
 
 # FIXME: there's a lot of logic common to the run, cd and builtin file
 # completers, that is currently reimplemented in each.
 
 def magic_run_completer(self, event):
     """Complete files that end in .py or .ipy for the %run command.
     """
     comps = shlex_split(event.line)
     relpath = (len(comps) > 1 and comps[-1] or '').strip("'\"")
 
     #print("\nev=", event)  # dbg
     #print("rp=", relpath)  # dbg
     #print('comps=', comps)  # dbg
 
     lglob = glob.glob
     isdir = os.path.isdir
     relpath, tilde_expand, tilde_val = expand_user(relpath)
         
     dirs = [f.replace('\\','/') + "/" for f in lglob(relpath+'*') if isdir(f)]
 
     # Find if the user has already typed the first filename, after which we
     # should complete on all files, since after the first one other files may
     # be arguments to the input script.
 
     if filter(magic_run_re.match, comps):
         pys =  [f.replace('\\','/') for f in lglob('*')]
     else:
         pys =  [f.replace('\\','/')
                 for f in lglob(relpath+'*.py') + lglob(relpath+'*.ipy') +
                 lglob(relpath + '*.pyw')]
     #print('run comp:', dirs+pys) # dbg
     return [compress_user(p, tilde_expand, tilde_val) for p in dirs+pys]
 
 
 def cd_completer(self, event):
     """Completer function for cd, which only returns directories."""
     ip = get_ipython()
     relpath = event.symbol
 
     #print(event) # dbg
     if event.line.endswith('-b') or ' -b ' in event.line:
         # return only bookmark completions
         bkms = self.db.get('bookmarks', None)
         if bkms:
             return bkms.keys()
         else:
             return []
     
     if event.symbol == '-':
         width_dh = str(len(str(len(ip.user_ns['_dh']) + 1)))
         # jump in directory history by number
         fmt = '-%0' + width_dh +'d [%s]'
         ents = [ fmt % (i,s) for i,s in enumerate(ip.user_ns['_dh'])]
         if len(ents) > 1:
             return ents
         return []
 
     if event.symbol.startswith('--'):
         return ["--" + os.path.basename(d) for d in ip.user_ns['_dh']]
 
     # Expand ~ in path and normalize directory separators.
     relpath, tilde_expand, tilde_val = expand_user(relpath)
     relpath = relpath.replace('\\','/')
 
     found = []
     for d in [f.replace('\\','/') + '/' for f in glob.glob(relpath+'*')
               if os.path.isdir(f)]:
         if ' ' in d:
             # we don't want to deal with any of that, complex code
             # for this is elsewhere
             raise TryNext
         
         found.append(d)
 
     if not found:
         if os.path.isdir(relpath):
             return [compress_user(relpath, tilde_expand, tilde_val)]
 
         # if no completions so far, try bookmarks
-        bks = self.db.get('bookmarks',{}).keys()
+        bks = self.db.get('bookmarks',{}).iterkeys()
         bkmatches = [s for s in bks if s.startswith(event.symbol)]
         if bkmatches:
             return bkmatches
         
         raise TryNext
 
     return [compress_user(p, tilde_expand, tilde_val) for p in found]

IPython/core/history.py

@@ -1,283 +1,283 b''
 # -*- coding: utf-8 -*-
 """ History related magics and functionality """
 
 # Stdlib imports
 import fnmatch
 import os
 
 import IPython.utils.io
 from IPython.utils.io import ask_yes_no
 from IPython.utils.warn import warn
 from IPython.core import ipapi
 
 def magic_history(self, parameter_s = ''):
     """Print input history (_i<n> variables), with most recent last.
     
     %history       -> print at most 40 inputs (some may be multi-line)\\
     %history n     -> print at most n inputs\\
     %history n1 n2 -> print inputs between n1 and n2 (n2 not included)\\
 
     By default, input history is printed without line numbers so it can be
     directly pasted into an editor.
 
     With -n, each input's number <n> is shown, and is accessible as the
     automatically generated variable _i<n> as well as In[<n>].  Multi-line
     statements are printed starting at a new line for easy copy/paste.
 
     Options:
 
       -n: print line numbers for each input.
       This feature is only available if numbered prompts are in use.
 
       -o: also print outputs for each input.
 
       -p: print classic '>>>' python prompts before each input.  This is useful
        for making documentation, and in conjunction with -o, for producing
        doctest-ready output.
 
       -r: (default) print the 'raw' history, i.e. the actual commands you typed.
       
       -t: print the 'translated' history, as IPython understands it.  IPython
       filters your input and converts it all into valid Python source before
       executing it (things like magics or aliases are turned into function
       calls, for example). With this option, you'll see the native history
       instead of the user-entered version: '%cd /' will be seen as
       'get_ipython().magic("%cd /")' instead of '%cd /'.
       
       -g: treat the arg as a pattern to grep for in (full) history.
       This includes the "shadow history" (almost all commands ever written).
       Use '%hist -g' to show full shadow history (may be very long).
       In shadow history, every index nuwber starts with 0.
 
       -f FILENAME: instead of printing the output to the screen, redirect it to
        the given file.  The file is always overwritten, though IPython asks for
        confirmation first if it already exists.
     """
 
     if not self.shell.displayhook.do_full_cache:
         print 'This feature is only available if numbered prompts are in use.'
         return
     opts,args = self.parse_options(parameter_s,'gnoptsrf:',mode='list')
 
     # Check if output to specific file was requested.
     try:
         outfname = opts['f']
     except KeyError:
         outfile = IPython.utils.io.Term.cout  # default
         # We don't want to close stdout at the end!
         close_at_end = False
     else:
         if os.path.exists(outfname):
             if not ask_yes_no("File %r exists. Overwrite?" % outfname): 
                 print 'Aborting.'
                 return
 
         outfile = open(outfname,'w')
         close_at_end = True
 
     if 't' in opts:
         input_hist = self.shell.input_hist
     elif 'r' in opts:
         input_hist = self.shell.input_hist_raw
     else:
         # Raw history is the default
         input_hist = self.shell.input_hist_raw
             
     default_length = 40
     pattern = None
     if 'g' in opts:
         init = 1
         final = len(input_hist)
         parts = parameter_s.split(None, 1)
         if len(parts) == 1:
             parts += '*'
         head, pattern = parts
         pattern = "*" + pattern + "*"
     elif len(args) == 0:
         final = len(input_hist)-1
         init = max(1,final-default_length)
     elif len(args) == 1:
         final = len(input_hist)
         init = max(1, final-int(args[0]))
     elif len(args) == 2:
         init, final = map(int, args)
     else:
         warn('%hist takes 0, 1 or 2 arguments separated by spaces.')
         print >> IPython.utils.io.Term.cout, self.magic_hist.__doc__
         return
     
     width = len(str(final))
     line_sep = ['','\n']
     print_nums = 'n' in opts
     print_outputs = 'o' in opts
     pyprompts = 'p' in opts
     
     found = False
     if pattern is not None:
         sh = self.shell.shadowhist.all()
         for idx, s in sh:
             if fnmatch.fnmatch(s, pattern):
                 print >> outfile, "0%d: %s" %(idx, s.expandtabs(4))
                 found = True
     
     if found:
         print >> outfile, "==="
         print >> outfile, \
               "shadow history ends, fetch by %rep <number> (must start with 0)"
         print >> outfile, "=== start of normal history ==="
         
     for in_num in range(init, final):
         # Print user history with tabs expanded to 4 spaces.  The GUI clients
         # use hard tabs for easier usability in auto-indented code, but we want
         # to produce PEP-8 compliant history for safe pasting into an editor.
         inline = input_hist[in_num].expandtabs(4)
 
         if pattern is not None and not fnmatch.fnmatch(inline, pattern):
             continue
             
         multiline = int(inline.count('\n') > 1)
         if print_nums:
             print >> outfile, \
                   '%s:%s' % (str(in_num).ljust(width), line_sep[multiline]),
         if pyprompts:
             print >> outfile, '>>>',
             if multiline:
                 lines = inline.splitlines()
                 print >> outfile, '\n... '.join(lines)
                 print >> outfile, '... '
             else:
                 print >> outfile, inline,
         else:
             print >> outfile, inline,
         if print_outputs:
             output = self.shell.output_hist.get(in_num)
             if output is not None:
                 print >> outfile, repr(output)
 
     if close_at_end:
         outfile.close()
 
 
 def magic_hist(self, parameter_s=''):
     """Alternate name for %history."""
     return self.magic_history(parameter_s)
 
 
 def rep_f(self, arg):
     r""" Repeat a command, or get command to input line for editing
 
     - %rep (no arguments):
     
     Place a string version of last computation result (stored in the special '_'
     variable) to the next input prompt. Allows you to create elaborate command
     lines without using copy-paste::
     
         $ l = ["hei", "vaan"]       
         $ "".join(l)        
         ==> heivaan        
         $ %rep        
         $ heivaan_ <== cursor blinking    
     
     %rep 45
     
     Place history line 45 to next input prompt. Use %hist to find out the
     number.
     
     %rep 1-4 6-7 3
     
     Repeat the specified lines immediately. Input slice syntax is the same as
     in %macro and %save.
     
     %rep foo
     
     Place the most recent line that has the substring "foo" to next input.
     (e.g. 'svn ci -m foobar').    
     """
     
     opts,args = self.parse_options(arg,'',mode='list')
     if not args:
         self.set_next_input(str(self.shell.user_ns["_"]))
         return
 
     if len(args) == 1 and not '-' in args[0]:
         arg = args[0]
         if len(arg) > 1 and arg.startswith('0'):
             # get from shadow hist
             num = int(arg[1:])
             line = self.shell.shadowhist.get(num)
             self.set_next_input(str(line))
             return
         try:
             num = int(args[0])
             self.set_next_input(str(self.shell.input_hist_raw[num]).rstrip())
             return
         except ValueError:
             pass
         
         for h in reversed(self.shell.input_hist_raw):
             if 'rep' in h:
                 continue
             if fnmatch.fnmatch(h,'*' + arg + '*'):
                 self.set_next_input(str(h).rstrip())
                 return
         
     try:
         lines = self.extract_input_slices(args, True)
         print "lines",lines
         self.runlines(lines)
     except ValueError:
         print "Not found in recent history:", args
         
 
 _sentinel = object()
 
 class ShadowHist(object):
     def __init__(self, db):
         # cmd => idx mapping
         self.curidx = 0
         self.db = db
         self.disabled = False
     
     def inc_idx(self):
         idx = self.db.get('shadowhist_idx', 1)
         self.db['shadowhist_idx'] = idx + 1
         return idx
         
     def add(self, ent):
         if self.disabled:
             return
         try:
             old = self.db.hget('shadowhist', ent, _sentinel)
             if old is not _sentinel:
                 return
             newidx = self.inc_idx()
             #print "new",newidx # dbg
             self.db.hset('shadowhist',ent, newidx)
         except:
             ipapi.get().showtraceback()
             print "WARNING: disabling shadow history"
             self.disabled = True
     
     def all(self):
         d = self.db.hdict('shadowhist')
-        items = [(i,s) for (s,i) in d.items()]
+        items = [(i,s) for (s,i) in d.iteritems()]
         items.sort()
         return items
 
     def get(self, idx):
         all = self.all()
         
         for k, v in all:
             #print k,v
             if k == idx:
                 return v
 
 
 def init_ipython(ip):
     ip.define_magic("rep",rep_f)        
     ip.define_magic("hist",magic_hist)            
     ip.define_magic("history",magic_history)
 
     # XXX - ipy_completers are in quarantine, need to be updated to new apis
     #import ipy_completers
     #ipy_completers.quick_completer('%hist' ,'-g -t -r -n')

IPython/core/interactiveshell.py

@@ -1,2587 +1,2584 b''
 # -*- coding: utf-8 -*-
 """Main IPython class."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001 Janko Hauser <[e-mail]>
 #  Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #  Copyright (C) 2008-2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
 from __future__ import absolute_import
 
 import __builtin__
 import __future__
 import abc
 import atexit
 import codeop
-import exceptions
-import new
 import os
 import re
 import string
 import sys
 import tempfile
+import types
 from contextlib import nested
 
 from IPython.config.configurable import Configurable
 from IPython.core import debugger, oinspect
 from IPython.core import history as ipcorehist
 from IPython.core import page
 from IPython.core import prefilter
 from IPython.core import shadowns
 from IPython.core import ultratb
 from IPython.core.alias import AliasManager
 from IPython.core.builtin_trap import BuiltinTrap
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.displayhook import DisplayHook
 from IPython.core.error import TryNext, UsageError
 from IPython.core.extensions import ExtensionManager
 from IPython.core.fakemodule import FakeModule, init_fakemod_dict
 from IPython.core.inputlist import InputList
 from IPython.core.inputsplitter import IPythonInputSplitter
 from IPython.core.logger import Logger
 from IPython.core.magic import Magic
 from IPython.core.payload import PayloadManager
 from IPython.core.plugin import PluginManager
 from IPython.core.prefilter import PrefilterManager, ESC_MAGIC
 from IPython.external.Itpl import ItplNS
 from IPython.utils import PyColorize
 from IPython.utils import io
 from IPython.utils import pickleshare
 from IPython.utils.doctestreload import doctest_reload
 from IPython.utils.io import ask_yes_no, rprint
 from IPython.utils.ipstruct import Struct
 from IPython.utils.path import get_home_dir, get_ipython_dir, HomeDirError
 from IPython.utils.process import system, getoutput
 from IPython.utils.strdispatch import StrDispatch
 from IPython.utils.syspathcontext import prepended_to_syspath
 from IPython.utils.text import num_ini_spaces, format_screen, LSString, SList
 from IPython.utils.traitlets import (Int, Str, CBool, CaselessStrEnum, Enum,
                                      List, Unicode, Instance, Type)
 from IPython.utils.warn import warn, error, fatal
 import IPython.core.hooks
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # compiled regexps for autoindent management
 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
 
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
 # store the builtin raw_input globally, and use this always, in case user code
 # overwrites it (like wx.py.PyShell does)
 raw_input_original = raw_input
 
 def softspace(file, newvalue):
     """Copied from code.py, to remove the dependency"""
 
     oldvalue = 0
     try:
         oldvalue = file.softspace
     except AttributeError:
         pass
     try:
         file.softspace = newvalue
     except (AttributeError, TypeError):
         # "attribute-less object" or "read-only attributes"
         pass
     return oldvalue
 
 
 def no_op(*a, **kw): pass
 
-class SpaceInInput(exceptions.Exception): pass
+class SpaceInInput(Exception): pass
 
 class Bunch: pass
 
 
 def get_default_colors():
     if sys.platform=='darwin':
         return "LightBG"
     elif os.name=='nt':
         return 'Linux'
     else:
         return 'Linux'
 
 
 class SeparateStr(Str):
     """A Str subclass to validate separate_in, separate_out, etc.
 
     This is a Str based trait that converts '0'->'' and '\\n'->'\n'.
     """
 
     def validate(self, obj, value):
         if value == '0': value = ''
         value = value.replace('\\n','\n')
         return super(SeparateStr, self).validate(obj, value)
 
 class MultipleInstanceError(Exception):
     pass
 
 
 #-----------------------------------------------------------------------------
 # Main IPython class
 #-----------------------------------------------------------------------------
 
 
 class InteractiveShell(Configurable, Magic):
     """An enhanced, interactive shell for Python."""
 
     _instance = None
     autocall = Enum((0,1,2), default_value=1, config=True)
     # TODO: remove all autoindent logic and put into frontends.
     # We can't do this yet because even runlines uses the autoindent.
     autoindent = CBool(True, config=True)
     automagic = CBool(True, config=True)
     cache_size = Int(1000, config=True)
     color_info = CBool(True, config=True)
     colors = CaselessStrEnum(('NoColor','LightBG','Linux'), 
                              default_value=get_default_colors(), config=True)
     debug = CBool(False, config=True)
     deep_reload = CBool(False, config=True)
     displayhook_class = Type(DisplayHook)
     exit_now = CBool(False)
     filename = Str("<ipython console>")
     ipython_dir= Unicode('', config=True) # Set to get_ipython_dir() in __init__
 
     # Input splitter, to split entire cells of input into either individual
     # interactive statements or whole blocks.
     input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
                               (), {})
     logstart = CBool(False, config=True)
     logfile = Str('', config=True)
     logappend = Str('', config=True)
     object_info_string_level = Enum((0,1,2), default_value=0,
                                     config=True)
     pdb = CBool(False, config=True)
 
     pprint = CBool(True, config=True)
     profile = Str('', config=True)
     prompt_in1 = Str('In [\\#]: ', config=True)
     prompt_in2 = Str('   .\\D.: ', config=True)
     prompt_out = Str('Out[\\#]: ', config=True)
     prompts_pad_left = CBool(True, config=True)
     quiet = CBool(False, config=True)
 
     # The readline stuff will eventually be moved to the terminal subclass
     # but for now, we can't do that as readline is welded in everywhere.
     readline_use = CBool(True, config=True)
     readline_merge_completions = CBool(True, config=True)
     readline_omit__names = Enum((0,1,2), default_value=0, config=True)
     readline_remove_delims = Str('-/~', config=True)
     readline_parse_and_bind = List([
             'tab: complete',
             '"\C-l": clear-screen',
             'set show-all-if-ambiguous on',
             '"\C-o": tab-insert',
             '"\M-i": "    "',
             '"\M-o": "\d\d\d\d"',
             '"\M-I": "\d\d\d\d"',
             '"\C-r": reverse-search-history',
             '"\C-s": forward-search-history',
             '"\C-p": history-search-backward',
             '"\C-n": history-search-forward',
             '"\e[A": history-search-backward',
             '"\e[B": history-search-forward',
             '"\C-k": kill-line',
             '"\C-u": unix-line-discard',
         ], allow_none=False, config=True)
 
     # TODO: this part of prompt management should be moved to the frontends.
     # Use custom TraitTypes that convert '0'->'' and '\\n'->'\n'
     separate_in = SeparateStr('\n', config=True)
     separate_out = SeparateStr('', config=True)
     separate_out2 = SeparateStr('', config=True)
     wildcards_case_sensitive = CBool(True, config=True)
     xmode = CaselessStrEnum(('Context','Plain', 'Verbose'), 
                             default_value='Context', config=True)
 
     # Subcomponents of InteractiveShell
     alias_manager = Instance('IPython.core.alias.AliasManager')
     prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager')
     builtin_trap = Instance('IPython.core.builtin_trap.BuiltinTrap')
     display_trap = Instance('IPython.core.display_trap.DisplayTrap')
     extension_manager = Instance('IPython.core.extensions.ExtensionManager')
     plugin_manager = Instance('IPython.core.plugin.PluginManager')
     payload_manager = Instance('IPython.core.payload.PayloadManager')
 
     # Private interface
     _post_execute = set()
 
     def __init__(self, config=None, ipython_dir=None,
                  user_ns=None, user_global_ns=None,
                  custom_exceptions=((), None)):
 
         # This is where traits with a config_key argument are updated
         # from the values on config.
         super(InteractiveShell, self).__init__(config=config)
 
         # These are relatively independent and stateless
         self.init_ipython_dir(ipython_dir)
         self.init_instance_attrs()
         self.init_environment()
 
         # Create namespaces (user_ns, user_global_ns, etc.)
         self.init_create_namespaces(user_ns, user_global_ns)
         # This has to be done after init_create_namespaces because it uses
         # something in self.user_ns, but before init_sys_modules, which
         # is the first thing to modify sys.
         # TODO: When we override sys.stdout and sys.stderr before this class
         # is created, we are saving the overridden ones here. Not sure if this
         # is what we want to do.
         self.save_sys_module_state()
         self.init_sys_modules()
 
         self.init_history()
         self.init_encoding()
         self.init_prefilter()
 
         Magic.__init__(self, self)
 
         self.init_syntax_highlighting()
         self.init_hooks()
         self.init_pushd_popd_magic()
         # self.init_traceback_handlers use to be here, but we moved it below
         # because it and init_io have to come after init_readline.
         self.init_user_ns()
         self.init_logger()
         self.init_alias()
         self.init_builtins()
 
         # pre_config_initialization
         self.init_shadow_hist()
 
         # The next section should contain everything that was in ipmaker.
         self.init_logstart()
 
         # The following was in post_config_initialization
         self.init_inspector()
         # init_readline() must come before init_io(), because init_io uses
         # readline related things.
         self.init_readline()
         # init_completer must come after init_readline, because it needs to
         # know whether readline is present or not system-wide to configure the
         # completers, since the completion machinery can now operate
         # independently of readline (e.g. over the network)
         self.init_completer()
         # TODO: init_io() needs to happen before init_traceback handlers
         # because the traceback handlers hardcode the stdout/stderr streams.
         # This logic in in debugger.Pdb and should eventually be changed.
         self.init_io()
         self.init_traceback_handlers(custom_exceptions)
         self.init_prompts()
         self.init_displayhook()
         self.init_reload_doctest()
         self.init_magics()
         self.init_pdb()
         self.init_extension_manager()
         self.init_plugin_manager()
         self.init_payload()
         self.hooks.late_startup_hook()
         atexit.register(self.atexit_operations)
 
     @classmethod
     def instance(cls, *args, **kwargs):
         """Returns a global InteractiveShell instance."""
         if cls._instance is None:
             inst = cls(*args, **kwargs)
             # Now make sure that the instance will also be returned by
             # the subclasses instance attribute.
             for subclass in cls.mro():
                 if issubclass(cls, subclass) and \
                        issubclass(subclass, InteractiveShell):
                     subclass._instance = inst
                 else:
                     break
         if isinstance(cls._instance, cls):
             return cls._instance
         else:
             raise MultipleInstanceError(
                 'Multiple incompatible subclass instances of '
                 'InteractiveShell are being created.'
             )
 
     @classmethod
     def initialized(cls):
         return hasattr(cls, "_instance")
 
     def get_ipython(self):
         """Return the currently running IPython instance."""
         return self
 
     #-------------------------------------------------------------------------
     # Trait changed handlers
     #-------------------------------------------------------------------------
 
     def _ipython_dir_changed(self, name, new):
         if not os.path.isdir(new):
             os.makedirs(new, mode = 0777)
 
     def set_autoindent(self,value=None):
         """Set the autoindent flag, checking for readline support.
 
         If called with no arguments, it acts as a toggle."""
 
         if not self.has_readline:
             if os.name == 'posix':
                 warn("The auto-indent feature requires the readline library")
             self.autoindent = 0
             return
         if value is None:
             self.autoindent = not self.autoindent
         else:
             self.autoindent = value
 
     #-------------------------------------------------------------------------
     # init_* methods called by __init__
     #-------------------------------------------------------------------------
 
     def init_ipython_dir(self, ipython_dir):
         if ipython_dir is not None:
             self.ipython_dir = ipython_dir
             self.config.Global.ipython_dir = self.ipython_dir
             return
 
         if hasattr(self.config.Global, 'ipython_dir'):
             self.ipython_dir = self.config.Global.ipython_dir
         else:
             self.ipython_dir = get_ipython_dir()
 
         # All children can just read this
         self.config.Global.ipython_dir = self.ipython_dir
 
     def init_instance_attrs(self):
         self.more = False
 
         # command compiler
         self.compile = codeop.CommandCompiler()
 
         # User input buffer
         self.buffer = []
 
         # Make an empty namespace, which extension writers can rely on both
         # existing and NEVER being used by ipython itself.  This gives them a
         # convenient location for storing additional information and state
         # their extensions may require, without fear of collisions with other
         # ipython names that may develop later.
         self.meta = Struct()
 
         # Object variable to store code object waiting execution.  This is
         # used mainly by the multithreaded shells, but it can come in handy in
         # other situations.  No need to use a Queue here, since it's a single
         # item which gets cleared once run.
         self.code_to_run = None
 
         # Temporary files used for various purposes.  Deleted at exit.
         self.tempfiles = []
 
         # Keep track of readline usage (later set by init_readline)
         self.has_readline = False
 
         # keep track of where we started running (mainly for crash post-mortem)
         # This is not being used anywhere currently.
         self.starting_dir = os.getcwd()
 
         # Indentation management
         self.indent_current_nsp = 0
 
     def init_environment(self):
         """Any changes we need to make to the user's environment."""
         pass
 
     def init_encoding(self):
         # Get system encoding at startup time.  Certain terminals (like Emacs
         # under Win32 have it set to None, and we need to have a known valid
         # encoding to use in the raw_input() method
         try:
             self.stdin_encoding = sys.stdin.encoding or 'ascii'
         except AttributeError:
             self.stdin_encoding = 'ascii'
 
     def init_syntax_highlighting(self):
         # Python source parser/formatter for syntax highlighting
         pyformat = PyColorize.Parser().format
         self.pycolorize = lambda src: pyformat(src,'str',self.colors)
 
     def init_pushd_popd_magic(self):
         # for pushd/popd management
         try:
             self.home_dir = get_home_dir()
         except HomeDirError, msg:
             fatal(msg)
 
         self.dir_stack = []
 
     def init_logger(self):
         self.logger = Logger(self, logfname='ipython_log.py', logmode='rotate')
         # local shortcut, this is used a LOT
         self.log = self.logger.log
 
     def init_logstart(self):
         if self.logappend:
             self.magic_logstart(self.logappend + ' append')
         elif self.logfile:
             self.magic_logstart(self.logfile)
         elif self.logstart:
             self.magic_logstart()
 
     def init_builtins(self):
         self.builtin_trap = BuiltinTrap(shell=self)
 
     def init_inspector(self):
         # Object inspector
         self.inspector = oinspect.Inspector(oinspect.InspectColors,
                                             PyColorize.ANSICodeColors,
                                             'NoColor',
                                             self.object_info_string_level)
 
     def init_io(self):
         # This will just use sys.stdout and sys.stderr. If you want to
         # override sys.stdout and sys.stderr themselves, you need to do that
         # *before* instantiating this class, because Term holds onto 
         # references to the underlying streams.
         if sys.platform == 'win32' and self.has_readline:
             Term = io.IOTerm(cout=self.readline._outputfile,
                              cerr=self.readline._outputfile)
         else:
             Term = io.IOTerm()
         io.Term = Term
 
     def init_prompts(self):
         # TODO: This is a pass for now because the prompts are managed inside
         # the DisplayHook. Once there is a separate prompt manager, this 
         # will initialize that object and all prompt related information.
         pass
 
     def init_displayhook(self):
         # Initialize displayhook, set in/out prompts and printing system
         self.displayhook = self.displayhook_class(
             shell=self,
             cache_size=self.cache_size,
             input_sep = self.separate_in,
             output_sep = self.separate_out,
             output_sep2 = self.separate_out2,
             ps1 = self.prompt_in1,
             ps2 = self.prompt_in2,
             ps_out = self.prompt_out,
             pad_left = self.prompts_pad_left
         )
         # This is a context manager that installs/revmoes the displayhook at
         # the appropriate time.
         self.display_trap = DisplayTrap(hook=self.displayhook)
 
     def init_reload_doctest(self):
         # Do a proper resetting of doctest, including the necessary displayhook
         # monkeypatching
         try:
             doctest_reload()
         except ImportError:
             warn("doctest module does not exist.")
 
     #-------------------------------------------------------------------------
     # Things related to injections into the sys module
     #-------------------------------------------------------------------------
 
     def save_sys_module_state(self):
         """Save the state of hooks in the sys module.
 
         This has to be called after self.user_ns is created.
         """
         self._orig_sys_module_state = {}
         self._orig_sys_module_state['stdin'] = sys.stdin
         self._orig_sys_module_state['stdout'] = sys.stdout
         self._orig_sys_module_state['stderr'] = sys.stderr
         self._orig_sys_module_state['excepthook'] = sys.excepthook
         try:
             self._orig_sys_modules_main_name = self.user_ns['__name__']
         except KeyError:
             pass
 
     def restore_sys_module_state(self):
         """Restore the state of the sys module."""
         try:
-            for k, v in self._orig_sys_module_state.items():
+            for k, v in self._orig_sys_module_state.iteritems():
                 setattr(sys, k, v)
         except AttributeError:
             pass
         # Reset what what done in self.init_sys_modules
         try:
             sys.modules[self.user_ns['__name__']] = self._orig_sys_modules_main_name
         except (AttributeError, KeyError):
             pass
 
     #-------------------------------------------------------------------------
     # Things related to hooks
     #-------------------------------------------------------------------------
 
     def init_hooks(self):
         # hooks holds pointers used for user-side customizations
         self.hooks = Struct()
 
         self.strdispatchers = {}
 
         # Set all default hooks, defined in the IPython.hooks module.
         hooks = IPython.core.hooks
         for hook_name in hooks.__all__:
             # default hooks have priority 100, i.e. low; user hooks should have
             # 0-100 priority
             self.set_hook(hook_name,getattr(hooks,hook_name), 100)
 
     def set_hook(self,name,hook, priority = 50, str_key = None, re_key = None):
         """set_hook(name,hook) -> sets an internal IPython hook.
 
         IPython exposes some of its internal API as user-modifiable hooks.  By
         adding your function to one of these hooks, you can modify IPython's 
         behavior to call at runtime your own routines."""
 
         # At some point in the future, this should validate the hook before it
         # accepts it.  Probably at least check that the hook takes the number
         # of args it's supposed to.
         
-        f = new.instancemethod(hook,self,self.__class__)
+        f = types.MethodType(hook, self)
 
         # check if the hook is for strdispatcher first
         if str_key is not None:
             sdp = self.strdispatchers.get(name, StrDispatch())
             sdp.add_s(str_key, f, priority )
             self.strdispatchers[name] = sdp
             return
         if re_key is not None:
             sdp = self.strdispatchers.get(name, StrDispatch())
             sdp.add_re(re.compile(re_key), f, priority )
             self.strdispatchers[name] = sdp
             return
             
         dp = getattr(self.hooks, name, None)
         if name not in IPython.core.hooks.__all__:
             print "Warning! Hook '%s' is not one of %s" % \
                   (name, IPython.core.hooks.__all__ )
         if not dp:
             dp = IPython.core.hooks.CommandChainDispatcher()
         
         try:
             dp.add(f,priority)
         except AttributeError:
             # it was not commandchain, plain old func - replace
             dp = f
 
         setattr(self.hooks,name, dp)
 
     def register_post_execute(self, func):
         """Register a function for calling after code execution.
         """
         if not callable(func):
             raise ValueError('argument %s must be callable' % func)
         self._post_execute.add(func)
 
     #-------------------------------------------------------------------------
     # Things related to the "main" module
     #-------------------------------------------------------------------------
 
     def new_main_mod(self,ns=None):
         """Return a new 'main' module object for user code execution.
         """
         main_mod = self._user_main_module
         init_fakemod_dict(main_mod,ns)
         return main_mod
 
     def cache_main_mod(self,ns,fname):
         """Cache a main module's namespace.
 
         When scripts are executed via %run, we must keep a reference to the
         namespace of their __main__ module (a FakeModule instance) around so
         that Python doesn't clear it, rendering objects defined therein
         useless.
 
         This method keeps said reference in a private dict, keyed by the
         absolute path of the module object (which corresponds to the script
         path).  This way, for multiple executions of the same script we only
         keep one copy of the namespace (the last one), thus preventing memory
         leaks from old references while allowing the objects from the last
         execution to be accessible.
 
         Note: we can not allow the actual FakeModule instances to be deleted,
         because of how Python tears down modules (it hard-sets all their
         references to None without regard for reference counts).  This method
         must therefore make a *copy* of the given namespace, to allow the
         original module's __dict__ to be cleared and reused.
 
         
         Parameters
         ----------
           ns : a namespace (a dict, typically)
 
           fname : str
             Filename associated with the namespace.
 
         Examples
         --------
 
         In [10]: import IPython
 
         In [11]: _ip.cache_main_mod(IPython.__dict__,IPython.__file__)
 
         In [12]: IPython.__file__ in _ip._main_ns_cache
         Out[12]: True
         """
         self._main_ns_cache[os.path.abspath(fname)] = ns.copy()
 
     def clear_main_mod_cache(self):
         """Clear the cache of main modules.
 
         Mainly for use by utilities like %reset.
 
         Examples
         --------
 
         In [15]: import IPython
 
         In [16]: _ip.cache_main_mod(IPython.__dict__,IPython.__file__)
 
         In [17]: len(_ip._main_ns_cache) > 0
         Out[17]: True
 
         In [18]: _ip.clear_main_mod_cache()
 
         In [19]: len(_ip._main_ns_cache) == 0
         Out[19]: True
         """
         self._main_ns_cache.clear()
 
     #-------------------------------------------------------------------------
     # Things related to debugging
     #-------------------------------------------------------------------------
 
     def init_pdb(self):
         # Set calling of pdb on exceptions
         # self.call_pdb is a property
         self.call_pdb = self.pdb
 
     def _get_call_pdb(self):
         return self._call_pdb
 
     def _set_call_pdb(self,val):
 
         if val not in (0,1,False,True):
             raise ValueError,'new call_pdb value must be boolean'
 
         # store value in instance
         self._call_pdb = val
 
         # notify the actual exception handlers
         self.InteractiveTB.call_pdb = val
 
     call_pdb = property(_get_call_pdb,_set_call_pdb,None,
                         'Control auto-activation of pdb at exceptions')
 
     def debugger(self,force=False):
         """Call the pydb/pdb debugger.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
           flag and does not actually invoke the debugger if the flag is false.
           The 'force' option forces the debugger to activate even if the flag
           is false.
         """
 
         if not (force or self.call_pdb):
             return
 
         if not hasattr(sys,'last_traceback'):
             error('No traceback has been produced, nothing to debug.')
             return
 
         # use pydb if available
         if debugger.has_pydb:
             from pydb import pm
         else:
             # fallback to our internal debugger
             pm = lambda : self.InteractiveTB.debugger(force=True)
         self.history_saving_wrapper(pm)()
 
     #-------------------------------------------------------------------------
     # Things related to IPython's various namespaces
     #-------------------------------------------------------------------------
 
     def init_create_namespaces(self, user_ns=None, user_global_ns=None):
         # Create the namespace where the user will operate.  user_ns is
         # normally the only one used, and it is passed to the exec calls as
         # the locals argument.  But we do carry a user_global_ns namespace
         # given as the exec 'globals' argument,  This is useful in embedding
         # situations where the ipython shell opens in a context where the
         # distinction between locals and globals is meaningful.  For
         # non-embedded contexts, it is just the same object as the user_ns dict.
 
         # FIXME. For some strange reason, __builtins__ is showing up at user
         # level as a dict instead of a module. This is a manual fix, but I
         # should really track down where the problem is coming from. Alex
         # Schmolck reported this problem first.
 
         # A useful post by Alex Martelli on this topic:
         # Re: inconsistent value from __builtins__
         # Von: Alex Martelli <[e-mail]>
         # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
         # Gruppen: comp.lang.python
 
         # Michael Hohn <[e-mail]> wrote:
         # > >>> print type(builtin_check.get_global_binding('__builtins__'))
         # > <type 'dict'>
         # > >>> print type(__builtins__)
         # > <type 'module'>
         # > Is this difference in return value intentional?
 
         # Well, it's documented that '__builtins__' can be either a dictionary
         # or a module, and it's been that way for a long time. Whether it's
         # intentional (or sensible), I don't know. In any case, the idea is
         # that if you need to access the built-in namespace directly, you
         # should start with "import __builtin__" (note, no 's') which will
         # definitely give you a module. Yeah, it's somewhat confusing:-(.
 
         # These routines return properly built dicts as needed by the rest of
         # the code, and can also be used by extension writers to generate
         # properly initialized namespaces.
         user_ns, user_global_ns = self.make_user_namespaces(user_ns,
                                                             user_global_ns)
 
         # Assign namespaces
         # This is the namespace where all normal user variables live
         self.user_ns = user_ns
         self.user_global_ns = user_global_ns
 
         # An auxiliary namespace that checks what parts of the user_ns were
         # loaded at startup, so we can list later only variables defined in
         # actual interactive use.  Since it is always a subset of user_ns, it
         # doesn't need to be separately tracked in the ns_table.
         self.user_ns_hidden = {}
 
         # A namespace to keep track of internal data structures to prevent
         # them from cluttering user-visible stuff.  Will be updated later
         self.internal_ns = {}
 
         # Now that FakeModule produces a real module, we've run into a nasty
         # problem: after script execution (via %run), the module where the user
         # code ran is deleted.  Now that this object is a true module (needed
         # so docetst and other tools work correctly), the Python module
         # teardown mechanism runs over it, and sets to None every variable
         # present in that module.  Top-level references to objects from the
         # script survive, because the user_ns is updated with them.  However,
         # calling functions defined in the script that use other things from
         # the script will fail, because the function's closure had references
         # to the original objects, which are now all None.  So we must protect
         # these modules from deletion by keeping a cache.
         # 
         # To avoid keeping stale modules around (we only need the one from the
         # last run), we use a dict keyed with the full path to the script, so
         # only the last version of the module is held in the cache.  Note,
         # however, that we must cache the module *namespace contents* (their
         # __dict__).  Because if we try to cache the actual modules, old ones
         # (uncached) could be destroyed while still holding references (such as
         # those held by GUI objects that tend to be long-lived)>
         # 
         # The %reset command will flush this cache.  See the cache_main_mod()
         # and clear_main_mod_cache() methods for details on use.
 
         # This is the cache used for 'main' namespaces
         self._main_ns_cache = {}
         # And this is the single instance of FakeModule whose __dict__ we keep
         # copying and clearing for reuse on each %run
         self._user_main_module = FakeModule()
 
         # A table holding all the namespaces IPython deals with, so that
         # introspection facilities can search easily.
         self.ns_table = {'user':user_ns,
                          'user_global':user_global_ns,
                          'internal':self.internal_ns,
                          'builtin':__builtin__.__dict__
                          }
 
         # Similarly, track all namespaces where references can be held and that
         # we can safely clear (so it can NOT include builtin).  This one can be
         # a simple list.  Note that the main execution namespaces, user_ns and
         # user_global_ns, can NOT be listed here, as clearing them blindly
         # causes errors in object __del__ methods.  Instead, the reset() method
         # clears them manually and carefully.
         self.ns_refs_table = [ self.user_ns_hidden,
                                self.internal_ns, self._main_ns_cache ]
 
     def make_user_namespaces(self, user_ns=None, user_global_ns=None):
         """Return a valid local and global user interactive namespaces.
 
         This builds a dict with the minimal information needed to operate as a
         valid IPython user namespace, which you can pass to the various
         embedding classes in ipython. The default implementation returns the
         same dict for both the locals and the globals to allow functions to
         refer to variables in the namespace. Customized implementations can
         return different dicts. The locals dictionary can actually be anything
         following the basic mapping protocol of a dict, but the globals dict
         must be a true dict, not even a subclass. It is recommended that any
         custom object for the locals namespace synchronize with the globals
         dict somehow.
 
         Raises TypeError if the provided globals namespace is not a true dict.
 
         Parameters
         ----------
         user_ns : dict-like, optional
             The current user namespace. The items in this namespace should
             be included in the output. If None, an appropriate blank
             namespace should be created.
         user_global_ns : dict, optional
             The current user global namespace. The items in this namespace
             should be included in the output. If None, an appropriate
             blank namespace should be created.
 
         Returns
         -------
             A pair of dictionary-like object to be used as the local namespace
             of the interpreter and a dict to be used as the global namespace.
         """
 
 
         # We must ensure that __builtin__ (without the final 's') is always
         # available and pointing to the __builtin__ *module*.  For more details:
         # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
 
         if user_ns is None:
             # Set __name__ to __main__ to better match the behavior of the
             # normal interpreter.
             user_ns = {'__name__'     :'__main__',
                        '__builtin__' : __builtin__,
                        '__builtins__' : __builtin__,
                       }
         else:
             user_ns.setdefault('__name__','__main__')
             user_ns.setdefault('__builtin__',__builtin__)
             user_ns.setdefault('__builtins__',__builtin__)
 
         if user_global_ns is None:
             user_global_ns = user_ns
         if type(user_global_ns) is not dict:
             raise TypeError("user_global_ns must be a true dict; got %r"
                 % type(user_global_ns))
 
         return user_ns, user_global_ns
 
     def init_sys_modules(self):
         # We need to insert into sys.modules something that looks like a
         # module but which accesses the IPython namespace, for shelve and
         # pickle to work interactively. Normally they rely on getting
         # everything out of __main__, but for embedding purposes each IPython
         # instance has its own private namespace, so we can't go shoving
         # everything into __main__.
 
         # note, however, that we should only do this for non-embedded
         # ipythons, which really mimic the __main__.__dict__ with their own
         # namespace.  Embedded instances, on the other hand, should not do
         # this because they need to manage the user local/global namespaces
         # only, but they live within a 'normal' __main__ (meaning, they
         # shouldn't overtake the execution environment of the script they're
         # embedded in).
 
         # This is overridden in the InteractiveShellEmbed subclass to a no-op.
 
         try:
             main_name = self.user_ns['__name__']
         except KeyError:
             raise KeyError('user_ns dictionary MUST have a "__name__" key')
         else:
             sys.modules[main_name] = FakeModule(self.user_ns)
 
     def init_user_ns(self):
         """Initialize all user-visible namespaces to their minimum defaults.
 
         Certain history lists are also initialized here, as they effectively
         act as user namespaces.
 
         Notes
         -----
         All data structures here are only filled in, they are NOT reset by this
         method.  If they were not empty before, data will simply be added to
         therm.
         """
         # This function works in two parts: first we put a few things in
         # user_ns, and we sync that contents into user_ns_hidden so that these
         # initial variables aren't shown by %who.  After the sync, we add the
         # rest of what we *do* want the user to see with %who even on a new
         # session (probably nothing, so theye really only see their own stuff)
 
         # The user dict must *always* have a __builtin__ reference to the
         # Python standard __builtin__ namespace,  which must be imported.
         # This is so that certain operations in prompt evaluation can be
         # reliably executed with builtins.  Note that we can NOT use
         # __builtins__ (note the 's'),  because that can either be a dict or a
         # module, and can even mutate at runtime, depending on the context
         # (Python makes no guarantees on it).  In contrast, __builtin__ is
         # always a module object, though it must be explicitly imported.
         
         # For more details:
         # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
         ns = dict(__builtin__ = __builtin__)
         
         # Put 'help' in the user namespace
         try:
             from site import _Helper
             ns['help'] = _Helper()
         except ImportError:
             warn('help() not available - check site.py')
 
         # make global variables for user access to the histories
         ns['_ih'] = self.input_hist
         ns['_oh'] = self.output_hist
         ns['_dh'] = self.dir_hist
 
         ns['_sh'] = shadowns
 
         # user aliases to input and output histories.  These shouldn't show up
         # in %who, as they can have very large reprs.
         ns['In']  = self.input_hist
         ns['Out'] = self.output_hist
 
         # Store myself as the public api!!!
         ns['get_ipython'] = self.get_ipython
 
         # Sync what we've added so far to user_ns_hidden so these aren't seen
         # by %who
         self.user_ns_hidden.update(ns)
 
         # Anything put into ns now would show up in %who.  Think twice before
         # putting anything here, as we really want %who to show the user their
         # stuff, not our variables.
         
         # Finally, update the real user's namespace
         self.user_ns.update(ns)
 
 
     def reset(self):
         """Clear all internal namespaces.
 
         Note that this is much more aggressive than %reset, since it clears
         fully all namespaces, as well as all input/output lists.
         """
         self.alias_manager.clear_aliases()
 
         # Clear input and output histories
         self.input_hist[:] = []
         self.input_hist_raw[:] = []
         self.output_hist.clear()
 
         # Clear namespaces holding user references
         for ns in self.ns_refs_table:
             ns.clear()
             
         # The main execution namespaces must be cleared very carefully,
         # skipping the deletion of the builtin-related keys, because doing so
         # would cause errors in many object's __del__ methods.
         for ns in [self.user_ns, self.user_global_ns]:
             drop_keys = set(ns.keys())
             drop_keys.discard('__builtin__')
             drop_keys.discard('__builtins__')
             for k in drop_keys:
                 del ns[k]
 
         # Restore the user namespaces to minimal usability
         self.init_user_ns()
         
         # Restore the default and user aliases
         self.alias_manager.init_aliases()
 
     def reset_selective(self, regex=None):
         """Clear selective variables from internal namespaces based on a
         specified regular expression.
 
         Parameters
         ----------
         regex : string or compiled pattern, optional
             A regular expression pattern that will be used in searching
             variable names in the users namespaces.
         """
         if regex is not None:
             try:
                 m = re.compile(regex)
             except TypeError:
                 raise TypeError('regex must be a string or compiled pattern')
             # Search for keys in each namespace that match the given regex
             # If a match is found, delete the key/value pair.
             for ns in self.ns_refs_table:
                 for var in ns:
                     if m.search(var):
                         del ns[var]        
         
     def push(self, variables, interactive=True):
         """Inject a group of variables into the IPython user namespace.
 
         Parameters
         ----------
         variables : dict, str or list/tuple of str
             The variables to inject into the user's namespace.  If a dict, a
             simple update is done.  If a str, the string is assumed to have
             variable names separated by spaces.  A list/tuple of str can also
             be used to give the variable names.  If just the variable names are
             give (list/tuple/str) then the variable values looked up in the
             callers frame.
         interactive : bool
             If True (default), the variables will be listed with the ``who``
             magic.
         """
         vdict = None
 
         # We need a dict of name/value pairs to do namespace updates.
         if isinstance(variables, dict):
             vdict = variables
         elif isinstance(variables, (basestring, list, tuple)):
             if isinstance(variables, basestring):
                 vlist = variables.split()
             else:
                 vlist = variables
             vdict = {}
             cf = sys._getframe(1)
             for name in vlist:
                 try:
                     vdict[name] = eval(name, cf.f_globals, cf.f_locals)
                 except:
                     print ('Could not get variable %s from %s' %
                            (name,cf.f_code.co_name))
         else:
             raise ValueError('variables must be a dict/str/list/tuple')
             
         # Propagate variables to user namespace
         self.user_ns.update(vdict)
 
         # And configure interactive visibility
         config_ns = self.user_ns_hidden
         if interactive:
             for name, val in vdict.iteritems():
                 config_ns.pop(name, None)
         else:
             for name,val in vdict.iteritems():
                 config_ns[name] = val
 
     #-------------------------------------------------------------------------
     # Things related to object introspection
     #-------------------------------------------------------------------------
 
     def _ofind(self, oname, namespaces=None):
         """Find an object in the available namespaces.
 
         self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
 
         Has special code to detect magic functions.
         """
         #oname = oname.strip()
         #print '1- oname: <%r>' % oname  # dbg
         try:
             oname = oname.strip().encode('ascii')
             #print '2- oname: <%r>' % oname  # dbg
         except UnicodeEncodeError:
             print 'Python identifiers can only contain ascii characters.'
             return dict(found=False)
 
         alias_ns = None
         if namespaces is None:
             # Namespaces to search in:
             # Put them in a list. The order is important so that we
             # find things in the same order that Python finds them.
             namespaces = [ ('Interactive', self.user_ns),
                            ('IPython internal', self.internal_ns),
                            ('Python builtin', __builtin__.__dict__),
                            ('Alias', self.alias_manager.alias_table),
                            ]
             alias_ns = self.alias_manager.alias_table
 
         # initialize results to 'null'
         found = False; obj = None;  ospace = None;  ds = None;
         ismagic = False; isalias = False; parent = None
 
         # We need to special-case 'print', which as of python2.6 registers as a
         # function but should only be treated as one if print_function was
         # loaded with a future import.  In this case, just bail.
         if (oname == 'print' and not (self.compile.compiler.flags &
                                       __future__.CO_FUTURE_PRINT_FUNCTION)):
             return {'found':found, 'obj':obj, 'namespace':ospace,
                     'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
 
         # Look for the given name by splitting it in parts.  If the head is
         # found, then we look for all the remaining parts as members, and only
         # declare success if we can find them all.
         oname_parts = oname.split('.')
         oname_head, oname_rest = oname_parts[0],oname_parts[1:]
         for nsname,ns in namespaces:
             try:
                 obj = ns[oname_head]
             except KeyError:
                 continue
             else:
                 #print 'oname_rest:', oname_rest  # dbg
                 for part in oname_rest:
                     try:
                         parent = obj
                         obj = getattr(obj,part)
                     except:
                         # Blanket except b/c some badly implemented objects
                         # allow __getattr__ to raise exceptions other than
                         # AttributeError, which then crashes IPython.
                         break
                 else:
                     # If we finish the for loop (no break), we got all members
                     found = True
                     ospace = nsname
                     if ns == alias_ns:
                         isalias = True
                     break  # namespace loop
 
         # Try to see if it's magic
         if not found:
             if oname.startswith(ESC_MAGIC):
                 oname = oname[1:]
             obj = getattr(self,'magic_'+oname,None)
             if obj is not None:
                 found = True
                 ospace = 'IPython internal'
                 ismagic = True
 
         # Last try: special-case some literals like '', [], {}, etc:
         if not found and oname_head in ["''",'""','[]','{}','()']:
             obj = eval(oname_head)
             found = True
             ospace = 'Interactive'
 
         return {'found':found, 'obj':obj, 'namespace':ospace,
                 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
 
     def _ofind_property(self, oname, info):
         """Second part of object finding, to look for property details."""
         if info.found:
             # Get the docstring of the class property if it exists.
             path = oname.split('.')
             root = '.'.join(path[:-1])
             if info.parent is not None:
                 try:
                     target = getattr(info.parent, '__class__') 
                     # The object belongs to a class instance. 
                     try: 
                         target = getattr(target, path[-1])
                         # The class defines the object. 
                         if isinstance(target, property):
                             oname = root + '.__class__.' + path[-1]
                             info = Struct(self._ofind(oname))
                     except AttributeError: pass
                 except AttributeError: pass
 
         # We return either the new info or the unmodified input if the object
         # hadn't been found
         return info
 
     def _object_find(self, oname, namespaces=None):
         """Find an object and return a struct with info about it."""
         inf = Struct(self._ofind(oname, namespaces))
         return Struct(self._ofind_property(oname, inf))
         
     def _inspect(self, meth, oname, namespaces=None, **kw):
         """Generic interface to the inspector system.
 
         This function is meant to be called by pdef, pdoc & friends."""
         info = self._object_find(oname)
         if info.found:
             pmethod = getattr(self.inspector, meth)
             formatter = format_screen if info.ismagic else None
             if meth == 'pdoc':
                 pmethod(info.obj, oname, formatter)
             elif meth == 'pinfo':
                 pmethod(info.obj, oname, formatter, info, **kw)
             else:
                 pmethod(info.obj, oname)
         else:
             print 'Object `%s` not found.' % oname
             return 'not found'  # so callers can take other action
 
     def object_inspect(self, oname):
         info = self._object_find(oname)
         if info.found:
             return self.inspector.info(info.obj, oname, info=info)
         else:
             return oinspect.object_info(name=oname, found=False)
 
     #-------------------------------------------------------------------------
     # Things related to history management
     #-------------------------------------------------------------------------
 
     def init_history(self):
         # List of input with multi-line handling.
         self.input_hist = InputList()
         # This one will hold the 'raw' input history, without any
         # pre-processing.  This will allow users to retrieve the input just as
         # it was exactly typed in by the user, with %hist -r.
         self.input_hist_raw = InputList()
 
         # list of visited directories
         try:
             self.dir_hist = [os.getcwd()]
         except OSError:
             self.dir_hist = []
 
         # dict of output history
         self.output_hist = {}
 
         # Now the history file
         if self.profile:
             histfname = 'history-%s' % self.profile
         else:
             histfname = 'history'
         self.histfile = os.path.join(self.ipython_dir, histfname)
 
         # Fill the history zero entry, user counter starts at 1
         self.input_hist.append('\n')
         self.input_hist_raw.append('\n')
 
     def init_shadow_hist(self):
         try:
             self.db = pickleshare.PickleShareDB(self.ipython_dir + "/db")
-        except exceptions.UnicodeDecodeError:
+        except UnicodeDecodeError:
             print "Your ipython_dir can't be decoded to unicode!"
             print "Please set HOME environment variable to something that"
             print r"only has ASCII characters, e.g. c:\home"
             print "Now it is", self.ipython_dir
             sys.exit()
         self.shadowhist = ipcorehist.ShadowHist(self.db)
 
     def savehist(self):
         """Save input history to a file (via readline library)."""
 
         try:
             self.readline.write_history_file(self.histfile)
         except:
             print 'Unable to save IPython command history to file: ' + \
                   `self.histfile`
 
     def reloadhist(self):
         """Reload the input history from disk file."""
 
         try:
             self.readline.clear_history()
             self.readline.read_history_file(self.shell.histfile)
         except AttributeError:
             pass
 
     def history_saving_wrapper(self, func):
         """ Wrap func for readline history saving
 
         Convert func into callable that saves & restores
         history around the call """
 
         if self.has_readline:
             from IPython.utils import rlineimpl as readline
         else:
             return func
 
         def wrapper():
             self.savehist()
             try:
                 func()
             finally:
                 readline.read_history_file(self.histfile)
         return wrapper
 
     def get_history(self, index=None, raw=False, output=True):
         """Get the history list.
 
         Get the input and output history.
 
         Parameters
         ----------
         index : n or (n1, n2) or None
             If n, then the last entries. If a tuple, then all in
             range(n1, n2). If None, then all entries. Raises IndexError if
             the format of index is incorrect.
         raw : bool
             If True, return the raw input.
         output : bool
             If True, then return the output as well.
 
         Returns
         -------
         If output is True, then return a dict of tuples, keyed by the prompt
         numbers and with values of (input, output). If output is False, then
         a dict, keyed by the prompt number with the values of input. Raises
         IndexError if no history is found.
         """
         if raw:
             input_hist = self.input_hist_raw
         else:
             input_hist = self.input_hist
         if output:
             output_hist = self.user_ns['Out']
         n = len(input_hist)
         if index is None:
             start=0; stop=n
         elif isinstance(index, int):
             start=n-index; stop=n
         elif isinstance(index, tuple) and len(index) == 2:
             start=index[0]; stop=index[1]
         else:
             raise IndexError('Not a valid index for the input history: %r'
                              % index)
         hist = {}
         for i in range(start, stop):
             if output:
                 hist[i] = (input_hist[i], output_hist.get(i))
             else:
                 hist[i] = input_hist[i]
         if len(hist)==0:
             raise IndexError('No history for range of indices: %r' % index)
         return hist
 
     #-------------------------------------------------------------------------
     # Things related to exception handling and tracebacks (not debugging)
     #-------------------------------------------------------------------------
 
     def init_traceback_handlers(self, custom_exceptions):
         # Syntax error handler.
         self.SyntaxTB = ultratb.SyntaxTB(color_scheme='NoColor')
         
         # The interactive one is initialized with an offset, meaning we always
         # want to remove the topmost item in the traceback, which is our own
         # internal code. Valid modes: ['Plain','Context','Verbose']
         self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
                                                      color_scheme='NoColor',
                                                      tb_offset = 1)
 
         # The instance will store a pointer to the system-wide exception hook,
         # so that runtime code (such as magics) can access it.  This is because
         # during the read-eval loop, it may get temporarily overwritten.
         self.sys_excepthook = sys.excepthook
 
         # and add any custom exception handlers the user may have specified
         self.set_custom_exc(*custom_exceptions)
 
         # Set the exception mode
         self.InteractiveTB.set_mode(mode=self.xmode)
 
     def set_custom_exc(self, exc_tuple, handler):
         """set_custom_exc(exc_tuple,handler)
 
         Set a custom exception handler, which will be called if any of the
         exceptions in exc_tuple occur in the mainloop (specifically, in the
         runcode() method.
 
         Inputs:
 
           - exc_tuple: a *tuple* of valid exceptions to call the defined
           handler for.  It is very important that you use a tuple, and NOT A
           LIST here, because of the way Python's except statement works.  If
           you only want to trap a single exception, use a singleton tuple:
 
             exc_tuple == (MyCustomException,)
 
           - handler: this must be defined as a function with the following
           basic interface::
 
             def my_handler(self, etype, value, tb, tb_offset=None)
                 ...
                 # The return value must be
                 return structured_traceback
 
           This will be made into an instance method (via new.instancemethod)
           of IPython itself, and it will be called if any of the exceptions
           listed in the exc_tuple are caught.  If the handler is None, an
           internal basic one is used, which just prints basic info.
 
         WARNING: by putting in your own exception handler into IPython's main
         execution loop, you run a very good chance of nasty crashes.  This
         facility should only be used if you really know what you are doing."""
 
         assert type(exc_tuple)==type(()) , \
                "The custom exceptions must be given AS A TUPLE."
 
         def dummy_handler(self,etype,value,tb):
             print '*** Simple custom exception handler ***'
             print 'Exception type :',etype
             print 'Exception value:',value
             print 'Traceback      :',tb
             print 'Source code    :','\n'.join(self.buffer)
 
         if handler is None: handler = dummy_handler
 
-        self.CustomTB = new.instancemethod(handler,self,self.__class__)
+        self.CustomTB = types.MethodType(handler, self)
         self.custom_exceptions = exc_tuple
 
     def excepthook(self, etype, value, tb):
       """One more defense for GUI apps that call sys.excepthook.
 
       GUI frameworks like wxPython trap exceptions and call
       sys.excepthook themselves.  I guess this is a feature that
       enables them to keep running after exceptions that would
       otherwise kill their mainloop. This is a bother for IPython
       which excepts to catch all of the program exceptions with a try:
       except: statement.
 
       Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
       any app directly invokes sys.excepthook, it will look to the user like
       IPython crashed.  In order to work around this, we can disable the
       CrashHandler and replace it with this excepthook instead, which prints a
       regular traceback using our InteractiveTB.  In this fashion, apps which
       call sys.excepthook will generate a regular-looking exception from
       IPython, and the CrashHandler will only be triggered by real IPython
       crashes.
 
       This hook should be used sparingly, only in places which are not likely
       to be true IPython errors.
       """
       self.showtraceback((etype,value,tb),tb_offset=0)
 
     def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None,
                       exception_only=False):
         """Display the exception that just occurred.
 
         If nothing is known about the exception, this is the method which
         should be used throughout the code for presenting user tracebacks,
         rather than directly invoking the InteractiveTB object.
 
         A specific showsyntaxerror() also exists, but this method can take
         care of calling it if needed, so unless you are explicitly catching a
         SyntaxError exception, don't try to analyze the stack manually and
         simply call this method."""
         
         try:
             if exc_tuple is None:
                 etype, value, tb = sys.exc_info()
             else:
                 etype, value, tb = exc_tuple
 
             if etype is None:
                 if hasattr(sys, 'last_type'):
                     etype, value, tb = sys.last_type, sys.last_value, \
                                        sys.last_traceback
                 else:
                     self.write_err('No traceback available to show.\n')
                     return
     
             if etype is SyntaxError:
                 # Though this won't be called by syntax errors in the input
                 # line, there may be SyntaxError cases whith imported code.
                 self.showsyntaxerror(filename)
             elif etype is UsageError:
                 print "UsageError:", value
             else:
                 # WARNING: these variables are somewhat deprecated and not
                 # necessarily safe to use in a threaded environment, but tools
                 # like pdb depend on their existence, so let's set them.  If we
                 # find problems in the field, we'll need to revisit their use.
                 sys.last_type = etype
                 sys.last_value = value
                 sys.last_traceback = tb
     
                 if etype in self.custom_exceptions:
                     # FIXME: Old custom traceback objects may just return a
                     # string, in that case we just put it into a list
                     stb = self.CustomTB(etype, value, tb, tb_offset)
                     if isinstance(ctb, basestring):
                         stb = [stb]
                 else:
                     if exception_only:
                         stb = ['An exception has occurred, use %tb to see '
                                'the full traceback.\n']
                         stb.extend(self.InteractiveTB.get_exception_only(etype,
                                                                          value))
                     else:
                         stb = self.InteractiveTB.structured_traceback(etype,
                                                 value, tb, tb_offset=tb_offset)
                         # FIXME: the pdb calling should be done by us, not by
                         # the code computing the traceback.
                         if self.InteractiveTB.call_pdb:
                             # pdb mucks up readline, fix it back
                             self.set_readline_completer()
 
                 # Actually show the traceback
                 self._showtraceback(etype, value, stb)
                 
         except KeyboardInterrupt:
             self.write_err("\nKeyboardInterrupt\n")
 
     def _showtraceback(self, etype, evalue, stb):
         """Actually show a traceback.
 
         Subclasses may override this method to put the traceback on a different
         place, like a side channel.
         """
         print >> io.Term.cout, self.InteractiveTB.stb2text(stb)
 
     def showsyntaxerror(self, filename=None):
         """Display the syntax error that just occurred.
 
         This doesn't display a stack trace because there isn't one.
 
         If a filename is given, it is stuffed in the exception instead
         of what was there before (because Python's parser always uses
         "<string>" when reading from a string).
         """
         etype, value, last_traceback = sys.exc_info()
 
         # See note about these variables in showtraceback() above
         sys.last_type = etype
         sys.last_value = value
         sys.last_traceback = last_traceback
         
         if filename and etype is SyntaxError:
             # Work hard to stuff the correct filename in the exception
             try:
                 msg, (dummy_filename, lineno, offset, line) = value
             except:
                 # Not the format we expect; leave it alone
                 pass
             else:
                 # Stuff in the right filename
                 try:
                     # Assume SyntaxError is a class exception
                     value = SyntaxError(msg, (filename, lineno, offset, line))
                 except:
                     # If that failed, assume SyntaxError is a string
                     value = msg, (filename, lineno, offset, line)
         stb = self.SyntaxTB.structured_traceback(etype, value, [])
         self._showtraceback(etype, value, stb)
 
     #-------------------------------------------------------------------------
     # Things related to readline
     #-------------------------------------------------------------------------
 
     def init_readline(self):
         """Command history completion/saving/reloading."""
 
         if self.readline_use:
             import IPython.utils.rlineimpl as readline
 
         self.rl_next_input = None
         self.rl_do_indent = False
 
         if not self.readline_use or not readline.have_readline:
             self.has_readline = False
             self.readline = None
             # Set a number of methods that depend on readline to be no-op
             self.savehist = no_op
             self.reloadhist = no_op
             self.set_readline_completer = no_op
             self.set_custom_completer = no_op
             self.set_completer_frame = no_op
             warn('Readline services not available or not loaded.')
         else:
             self.has_readline = True
             self.readline = readline
             sys.modules['readline'] = readline
             
             # Platform-specific configuration
             if os.name == 'nt':
                 # FIXME - check with Frederick to see if we can harmonize
                 # naming conventions with pyreadline to avoid this
                 # platform-dependent check
                 self.readline_startup_hook = readline.set_pre_input_hook
             else:
                 self.readline_startup_hook = readline.set_startup_hook
 
             # Load user's initrc file (readline config)
             # Or if libedit is used, load editrc.
             inputrc_name = os.environ.get('INPUTRC')
             if inputrc_name is None:
                 home_dir = get_home_dir()
                 if home_dir is not None:
                     inputrc_name = '.inputrc'
                     if readline.uses_libedit:
                         inputrc_name = '.editrc'
                     inputrc_name = os.path.join(home_dir, inputrc_name)
             if os.path.isfile(inputrc_name):
                 try:
                     readline.read_init_file(inputrc_name)
                 except:
                     warn('Problems reading readline initialization file <%s>'
                          % inputrc_name)
             
             # Configure readline according to user's prefs
             # This is only done if GNU readline is being used.  If libedit
             # is being used (as on Leopard) the readline config is
             # not run as the syntax for libedit is different.
             if not readline.uses_libedit:
                 for rlcommand in self.readline_parse_and_bind:
                     #print "loading rl:",rlcommand  # dbg
                     readline.parse_and_bind(rlcommand)
 
             # Remove some chars from the delimiters list.  If we encounter
             # unicode chars, discard them.
             delims = readline.get_completer_delims().encode("ascii", "ignore")
             delims = delims.translate(string._idmap,
                                       self.readline_remove_delims)
             delims = delims.replace(ESC_MAGIC, '')
             readline.set_completer_delims(delims)
             # otherwise we end up with a monster history after a while:
             readline.set_history_length(1000)
             try:
                 #print '*** Reading readline history'  # dbg
                 readline.read_history_file(self.histfile)
             except IOError:
                 pass  # It doesn't exist yet.
 
             # If we have readline, we want our history saved upon ipython
             # exiting. 
             atexit.register(self.savehist)
 
         # Configure auto-indent for all platforms
         self.set_autoindent(self.autoindent)
 
     def set_next_input(self, s):
         """ Sets the 'default' input string for the next command line.
         
         Requires readline.
         
         Example:
         
         [D:\ipython]|1> _ip.set_next_input("Hello Word")
         [D:\ipython]|2> Hello Word_  # cursor is here        
         """
 
         self.rl_next_input = s
 
     # Maybe move this to the terminal subclass?
     def pre_readline(self):
         """readline hook to be used at the start of each line.
 
         Currently it handles auto-indent only."""
 
         if self.rl_do_indent:
             self.readline.insert_text(self._indent_current_str())
         if self.rl_next_input is not None:
             self.readline.insert_text(self.rl_next_input)
             self.rl_next_input = None
 
     def _indent_current_str(self):
         """return the current level of indentation as a string"""
         return self.indent_current_nsp * ' '
 
     #-------------------------------------------------------------------------
     # Things related to text completion
     #-------------------------------------------------------------------------
 
     def init_completer(self):
         """Initialize the completion machinery.
 
         This creates completion machinery that can be used by client code,
         either interactively in-process (typically triggered by the readline
         library), programatically (such as in test suites) or out-of-prcess
         (typically over the network by remote frontends).
         """
         from IPython.core.completer import IPCompleter
         from IPython.core.completerlib import (module_completer,
                                                magic_run_completer, cd_completer)
         
         self.Completer = IPCompleter(self,
                                      self.user_ns,
                                      self.user_global_ns,
                                      self.readline_omit__names,
                                      self.alias_manager.alias_table,
                                      self.has_readline)
         
         # Add custom completers to the basic ones built into IPCompleter
         sdisp = self.strdispatchers.get('complete_command', StrDispatch())
         self.strdispatchers['complete_command'] = sdisp
         self.Completer.custom_completers = sdisp
 
         self.set_hook('complete_command', module_completer, str_key = 'import')
         self.set_hook('complete_command', module_completer, str_key = 'from')
         self.set_hook('complete_command', magic_run_completer, str_key = '%run')
         self.set_hook('complete_command', cd_completer, str_key = '%cd')
 
         # Only configure readline if we truly are using readline.  IPython can
         # do tab-completion over the network, in GUIs, etc, where readline
         # itself may be absent
         if self.has_readline:
             self.set_readline_completer()
 
     def complete(self, text, line=None, cursor_pos=None):
         """Return the completed text and a list of completions.
 
         Parameters
         ----------
 
            text : string
              A string of text to be completed on.  It can be given as empty and
              instead a line/position pair are given.  In this case, the
              completer itself will split the line like readline does.
 
            line : string, optional
              The complete line that text is part of.
 
            cursor_pos : int, optional
              The position of the cursor on the input line.
 
         Returns
         -------
           text : string
             The actual text that was completed.
 
           matches : list
             A sorted list with all possible completions.
 
         The optional arguments allow the completion to take more context into
         account, and are part of the low-level completion API.
         
         This is a wrapper around the completion mechanism, similar to what
         readline does at the command line when the TAB key is hit.  By
         exposing it as a method, it can be used by other non-readline
         environments (such as GUIs) for text completion.
 
         Simple usage example:
 
         In [1]: x = 'hello'
 
         In [2]: _ip.complete('x.l')
         Out[2]: ('x.l', ['x.ljust', 'x.lower', 'x.lstrip'])
         """
 
         # Inject names into __builtin__ so we can complete on the added names.
         with self.builtin_trap:
             return self.Completer.complete(text, line, cursor_pos)
 
     def set_custom_completer(self, completer, pos=0):
         """Adds a new custom completer function.
 
         The position argument (defaults to 0) is the index in the completers
         list where you want the completer to be inserted."""
 
-        newcomp = new.instancemethod(completer,self.Completer,
-                                     self.Completer.__class__)
+        newcomp = types.MethodType(completer, self.Completer)
         self.Completer.matchers.insert(pos,newcomp)
 
     def set_readline_completer(self):
         """Reset readline's completer to be our own."""
         self.readline.set_completer(self.Completer.rlcomplete)
 
     def set_completer_frame(self, frame=None):
         """Set the frame of the completer."""
         if frame:
             self.Completer.namespace = frame.f_locals
             self.Completer.global_namespace = frame.f_globals
         else:
             self.Completer.namespace = self.user_ns
             self.Completer.global_namespace = self.user_global_ns
 
     #-------------------------------------------------------------------------
     # Things related to magics
     #-------------------------------------------------------------------------
 
     def init_magics(self):
         # FIXME: Move the color initialization to the DisplayHook, which
         # should be split into a prompt manager and displayhook. We probably
         # even need a centralize colors management object.
         self.magic_colors(self.colors)
         # History was moved to a separate module
         from . import history
         history.init_ipython(self)
 
     def magic(self,arg_s):
         """Call a magic function by name.
 
         Input: a string containing the name of the magic function to call and
         any additional arguments to be passed to the magic.
 
         magic('name -opt foo bar') is equivalent to typing at the ipython
         prompt:
 
         In[1]: %name -opt foo bar
 
         To call a magic without arguments, simply use magic('name').
 
         This provides a proper Python function to call IPython's magics in any
         valid Python code you can type at the interpreter, including loops and
         compound statements.
         """
         args = arg_s.split(' ',1)
         magic_name = args[0]
         magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
 
         try:
             magic_args = args[1]
         except IndexError:
             magic_args = ''
         fn = getattr(self,'magic_'+magic_name,None)
         if fn is None:
             error("Magic function `%s` not found." % magic_name)
         else:
             magic_args = self.var_expand(magic_args,1)
             with nested(self.builtin_trap,):
                 result = fn(magic_args)
                 return result
 
     def define_magic(self, magicname, func):
         """Expose own function as magic function for ipython 
     
         def foo_impl(self,parameter_s=''):
             'My very own magic!. (Use docstrings, IPython reads them).'
             print 'Magic function. Passed parameter is between < >:'
             print '<%s>' % parameter_s
             print 'The self object is:',self
-    
+    newcomp = types.MethodType(completer, self.Completer)
         self.define_magic('foo',foo_impl)
         """
         
-        import new
-        im = new.instancemethod(func,self, self.__class__)
+        im = types.MethodType(func, self)
         old = getattr(self, "magic_" + magicname, None)
         setattr(self, "magic_" + magicname, im)
         return old
 
     #-------------------------------------------------------------------------
     # Things related to macros
     #-------------------------------------------------------------------------
 
     def define_macro(self, name, themacro):
         """Define a new macro
 
         Parameters
         ----------
         name : str
             The name of the macro.
         themacro : str or Macro
             The action to do upon invoking the macro.  If a string, a new 
             Macro object is created by passing the string to it.
         """
         
         from IPython.core import macro
 
         if isinstance(themacro, basestring):
             themacro = macro.Macro(themacro)
         if not isinstance(themacro, macro.Macro):
             raise ValueError('A macro must be a string or a Macro instance.')
         self.user_ns[name] = themacro
 
     #-------------------------------------------------------------------------
     # Things related to the running of system commands
     #-------------------------------------------------------------------------
 
     def system(self, cmd):
         """Call the given cmd in a subprocess.
 
         Parameters
         ----------
         cmd : str
           Command to execute (can not end in '&', as bacground processes are
           not supported.
         """
         # We do not support backgrounding processes because we either use
         # pexpect or pipes to read from.  Users can always just call
         # os.system() if they really want a background process.
         if cmd.endswith('&'):
             raise OSError("Background processes not supported.")
 
         return system(self.var_expand(cmd, depth=2))
 
     def getoutput(self, cmd, split=True):
         """Get output (possibly including stderr) from a subprocess.
 
         Parameters
         ----------
         cmd : str
           Command to execute (can not end in '&', as background processes are
           not supported.
         split : bool, optional
         
           If True, split the output into an IPython SList.  Otherwise, an
           IPython LSString is returned.  These are objects similar to normal
           lists and strings, with a few convenience attributes for easier
           manipulation of line-based output.  You can use '?' on them for
           details.
           """
         if cmd.endswith('&'):
             raise OSError("Background processes not supported.")
         out = getoutput(self.var_expand(cmd, depth=2))
         if split:
             out = SList(out.splitlines())
         else:
             out = LSString(out)
         return out
 
     #-------------------------------------------------------------------------
     # Things related to aliases
     #-------------------------------------------------------------------------
 
     def init_alias(self):
         self.alias_manager = AliasManager(shell=self, config=self.config)
         self.ns_table['alias'] = self.alias_manager.alias_table,
 
     #-------------------------------------------------------------------------
     # Things related to extensions and plugins
     #-------------------------------------------------------------------------
 
     def init_extension_manager(self):
         self.extension_manager = ExtensionManager(shell=self, config=self.config)
 
     def init_plugin_manager(self):
         self.plugin_manager = PluginManager(config=self.config)
 
     #-------------------------------------------------------------------------
     # Things related to payloads
     #-------------------------------------------------------------------------
 
     def init_payload(self):
         self.payload_manager = PayloadManager(config=self.config)
 
     #-------------------------------------------------------------------------
     # Things related to the prefilter
     #-------------------------------------------------------------------------
 
     def init_prefilter(self):
         self.prefilter_manager = PrefilterManager(shell=self, config=self.config)
         # Ultimately this will be refactored in the new interpreter code, but
         # for now, we should expose the main prefilter method (there's legacy
         # code out there that may rely on this).
         self.prefilter = self.prefilter_manager.prefilter_lines
 
 
     def auto_rewrite_input(self, cmd):
         """Print to the screen the rewritten form of the user's command.
 
         This shows visual feedback by rewriting input lines that cause
         automatic calling to kick in, like::
 
           /f x
 
         into::
 
           ------> f(x)
           
         after the user's input prompt.  This helps the user understand that the
         input line was transformed automatically by IPython.
         """
         rw = self.displayhook.prompt1.auto_rewrite() + cmd
 
         try:
             # plain ascii works better w/ pyreadline, on some machines, so
             # we use it and only print uncolored rewrite if we have unicode
             rw = str(rw)
             print >> IPython.utils.io.Term.cout, rw
         except UnicodeEncodeError:
             print "------> " + cmd
             
     #-------------------------------------------------------------------------
     # Things related to extracting values/expressions from kernel and user_ns
     #-------------------------------------------------------------------------
 
     def _simple_error(self):
         etype, value = sys.exc_info()[:2]
         return u'[ERROR] {e.__name__}: {v}'.format(e=etype, v=value)
 
     def user_variables(self, names):
         """Get a list of variable names from the user's namespace.
 
         Parameters
         ----------
         names : list of strings
           A list of names of variables to be read from the user namespace.
 
         Returns
         -------
         A dict, keyed by the input names and with the repr() of each value.
         """
         out = {}
         user_ns = self.user_ns
         for varname in names:
             try:
                 value = repr(user_ns[varname])
             except:
                 value = self._simple_error()
             out[varname] = value
         return out
         
     def user_expressions(self, expressions):
         """Evaluate a dict of expressions in the user's namespace.
 
         Parameters
         ----------
         expressions : dict
           A dict with string keys and string values.  The expression values
           should be valid Python expressions, each of which will be evaluated
           in the user namespace.
         
         Returns
         -------
         A dict, keyed like the input expressions dict, with the repr() of each
         value.
         """
         out = {}
         user_ns = self.user_ns
         global_ns = self.user_global_ns
         for key, expr in expressions.iteritems():
             try:
                 value = repr(eval(expr, global_ns, user_ns))
             except:
                 value = self._simple_error()
             out[key] = value
         return out
 
     #-------------------------------------------------------------------------
     # Things related to the running of code
     #-------------------------------------------------------------------------
 
     def ex(self, cmd):
         """Execute a normal python statement in user namespace."""
         with nested(self.builtin_trap,):
             exec cmd in self.user_global_ns, self.user_ns
 
     def ev(self, expr):
         """Evaluate python expression expr in user namespace.
 
         Returns the result of evaluation
         """
         with nested(self.builtin_trap,):
             return eval(expr, self.user_global_ns, self.user_ns)
 
     def safe_execfile(self, fname, *where, **kw):
         """A safe version of the builtin execfile().
 
         This version will never throw an exception, but instead print
         helpful error messages to the screen.  This only works on pure
         Python files with the .py extension.
 
         Parameters
         ----------
         fname : string
             The name of the file to be executed.
         where : tuple
             One or two namespaces, passed to execfile() as (globals,locals).
             If only one is given, it is passed as both.
         exit_ignore : bool (False)
             If True, then silence SystemExit for non-zero status (it is always
             silenced for zero status, as it is so common).
         """
         kw.setdefault('exit_ignore', False)
 
         fname = os.path.abspath(os.path.expanduser(fname))
 
         # Make sure we have a .py file
         if not fname.endswith('.py'):
             warn('File must end with .py to be run using execfile: <%s>' % fname)
 
         # Make sure we can open the file
         try:
             with open(fname) as thefile:
                 pass
         except:
             warn('Could not open file <%s> for safe execution.' % fname)
             return
 
         # Find things also in current directory.  This is needed to mimic the
         # behavior of running a script from the system command line, where
         # Python inserts the script's directory into sys.path
         dname = os.path.dirname(fname)
 
         with prepended_to_syspath(dname):
             try:
                 execfile(fname,*where)
             except SystemExit, status:
                 # If the call was made with 0 or None exit status (sys.exit(0)
                 # or sys.exit() ), don't bother showing a traceback, as both of
                 # these are considered normal by the OS:
                 # > python -c'import sys;sys.exit(0)'; echo $?
                 # 0
                 # > python -c'import sys;sys.exit()'; echo $?
                 # 0
                 # For other exit status, we show the exception unless
                 # explicitly silenced, but only in short form.
                 if status.code not in (0, None) and not kw['exit_ignore']:
                     self.showtraceback(exception_only=True)
             except:
                 self.showtraceback()
 
     def safe_execfile_ipy(self, fname):
         """Like safe_execfile, but for .ipy files with IPython syntax.
 
         Parameters
         ----------
         fname : str
             The name of the file to execute.  The filename must have a
             .ipy extension.
         """
         fname = os.path.abspath(os.path.expanduser(fname))
 
         # Make sure we have a .py file
         if not fname.endswith('.ipy'):
             warn('File must end with .py to be run using execfile: <%s>' % fname)
 
         # Make sure we can open the file
         try:
             with open(fname) as thefile:
                 pass
         except:
             warn('Could not open file <%s> for safe execution.' % fname)
             return
 
         # Find things also in current directory.  This is needed to mimic the
         # behavior of running a script from the system command line, where
         # Python inserts the script's directory into sys.path
         dname = os.path.dirname(fname)
 
         with prepended_to_syspath(dname):
             try:
                 with open(fname) as thefile:
                     script = thefile.read()
                     # self.runlines currently captures all exceptions
                     # raise in user code.  It would be nice if there were
                     # versions of runlines, execfile that did raise, so
                     # we could catch the errors.
                     self.runlines(script, clean=True)
             except:
                 self.showtraceback()
                 warn('Unknown failure executing file: <%s>' % fname)
 
     def run_cell(self, cell):
         """Run the contents of an entire multiline 'cell' of code.
 
         The cell is split into separate blocks which can be executed
         individually.  Then, based on how many blocks there are, they are
         executed as follows:
 
         - A single block: 'single' mode.
 
         If there's more than one block, it depends:
 
         - if the last one is no more than two lines long, run all but the last
         in 'exec' mode and the very last one in 'single' mode.  This makes it
         easy to type simple expressions at the end to see computed values.  -
         otherwise (last one is also multiline), run all in 'exec' mode
 
         When code is executed in 'single' mode, :func:`sys.displayhook` fires,
         results are displayed and output prompts are computed.  In 'exec' mode,
         no results are displayed unless :func:`print` is called explicitly;
         this mode is more akin to running a script.
 
         Parameters
         ----------
         cell : str
           A single or multiline string.
         """
         #################################################################
         # FIXME
         # =====
         # This execution logic should stop calling runlines altogether, and
         # instead we should do what runlines does, in a controlled manner, here
         # (runlines mutates lots of state as it goes calling sub-methods that
         # also mutate state).  Basically we should:
         # - apply dynamic transforms for single-line input (the ones that
         # split_blocks won't apply since they need context).
         # - increment the global execution counter (we need to pull that out
         # from outputcache's control; outputcache should instead read it from
         # the main object).
         # - do any logging of input
         # - update histories (raw/translated)
         # - then, call plain runsource (for single blocks, so displayhook is
         # triggered) or runcode (for multiline blocks in exec mode).
         #
         # Once this is done, we'll be able to stop using runlines and we'll
         # also have a much cleaner separation of logging, input history and
         # output cache management.
         #################################################################
         
         # We need to break up the input into executable blocks that can be run
         # in 'single' mode, to provide comfortable user behavior.
         blocks = self.input_splitter.split_blocks(cell)
         
         if not blocks:
             return
         
         # Single-block input should behave like an interactive prompt
         if len(blocks) == 1:
             self.runlines(blocks[0])
             return
 
         # In multi-block input, if the last block is a simple (one-two lines)
         # expression, run it in single mode so it produces output.  Otherwise
         # just feed the whole thing to runcode.
         # This seems like a reasonable usability design.
         last = blocks[-1]
         
         # Note: below, whenever we call runcode, we must sync history
         # ourselves, because runcode is NOT meant to manage history at all.
         if len(last.splitlines()) < 2:
             # Get the main body to run as a cell
             body = ''.join(blocks[:-1])
             self.input_hist.append(body)
             self.input_hist_raw.append(body)
             retcode = self.runcode(body, post_execute=False)
             if retcode==0:
                 # And the last expression via runlines so it produces output
                 self.runlines(last)
         else:
             # Run the whole cell as one entity
             self.input_hist.append(cell)
             self.input_hist_raw.append(cell)
             self.runcode(cell)
 
     def runlines(self, lines, clean=False):
         """Run a string of one or more lines of source.
 
         This method is capable of running a string containing multiple source
         lines, as if they had been entered at the IPython prompt.  Since it
         exposes IPython's processing machinery, the given strings can contain
         magic calls (%magic), special shell access (!cmd), etc.
         """
         
         if isinstance(lines, (list, tuple)):
             lines = '\n'.join(lines)
 
         if clean:
             lines = self._cleanup_ipy_script(lines)
 
         # We must start with a clean buffer, in case this is run from an
         # interactive IPython session (via a magic, for example).
         self.resetbuffer()
         lines = lines.splitlines()
         more = 0
         with nested(self.builtin_trap, self.display_trap):
             for line in lines:
                 # skip blank lines so we don't mess up the prompt counter, but
                 # do NOT skip even a blank line if we are in a code block (more
                 # is true)
             
                 if line or more:
                     # push to raw history, so hist line numbers stay in sync
                     self.input_hist_raw.append(line + '\n')
                     prefiltered = self.prefilter_manager.prefilter_lines(line,
                                                                          more)
                     more = self.push_line(prefiltered)
                     # IPython's runsource returns None if there was an error
                     # compiling the code.  This allows us to stop processing
                     # right away, so the user gets the error message at the
                     # right place.
                     if more is None:
                         break
                 else:
                     self.input_hist_raw.append("\n")
             # final newline in case the input didn't have it, so that the code
             # actually does get executed
             if more:
                 self.push_line('\n')
 
     def runsource(self, source, filename='<input>', symbol='single'):
         """Compile and run some source in the interpreter.
 
         Arguments are as for compile_command().
 
         One several things can happen:
 
         1) The input is incorrect; compile_command() raised an
         exception (SyntaxError or OverflowError).  A syntax traceback
         will be printed by calling the showsyntaxerror() method.
 
         2) The input is incomplete, and more input is required;
         compile_command() returned None.  Nothing happens.
 
         3) The input is complete; compile_command() returned a code
         object.  The code is executed by calling self.runcode() (which
         also handles run-time exceptions, except for SystemExit).
 
         The return value is:
 
           - True in case 2
 
           - False in the other cases, unless an exception is raised, where
           None is returned instead.  This can be used by external callers to
           know whether to continue feeding input or not.
 
         The return value can be used to decide whether to use sys.ps1 or
         sys.ps2 to prompt the next line."""
 
         # We need to ensure that the source is unicode from here on.
         if type(source)==str:
             source = source.decode(self.stdin_encoding)
         
         # if the source code has leading blanks, add 'if 1:\n' to it
         # this allows execution of indented pasted code. It is tempting
         # to add '\n' at the end of source to run commands like ' a=1'
         # directly, but this fails for more complicated scenarios
 
         if source[:1] in [' ', '\t']:
             source = u'if 1:\n%s' % source
 
         try:
             code = self.compile(source,filename,symbol)
         except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError):
             # Case 1
             self.showsyntaxerror(filename)
             return None
 
         if code is None:
             # Case 2
             return True
 
         # Case 3
         # We store the code object so that threaded shells and
         # custom exception handlers can access all this info if needed.
         # The source corresponding to this can be obtained from the
         # buffer attribute as '\n'.join(self.buffer).
         self.code_to_run = code
         # now actually execute the code object
         if self.runcode(code) == 0:
             return False
         else:
             return None
 
     def runcode(self, code_obj, post_execute=True):
         """Execute a code object.
 
         When an exception occurs, self.showtraceback() is called to display a
         traceback.
 
         Return value: a flag indicating whether the code to be run completed
         successfully:
 
           - 0: successful execution.
           - 1: an error occurred.
         """
 
         # Set our own excepthook in case the user code tries to call it
         # directly, so that the IPython crash handler doesn't get triggered
         old_excepthook,sys.excepthook = sys.excepthook, self.excepthook
 
         # we save the original sys.excepthook in the instance, in case config
         # code (such as magics) needs access to it.
         self.sys_excepthook = old_excepthook
         outflag = 1  # happens in more places, so it's easier as default
         try:
             try:
                 self.hooks.pre_runcode_hook()
                 #rprint('Running code') # dbg
                 exec code_obj in self.user_global_ns, self.user_ns
             finally:
                 # Reset our crash handler in place
                 sys.excepthook = old_excepthook
         except SystemExit:
             self.resetbuffer()
             self.showtraceback(exception_only=True)
             warn("To exit: use any of 'exit', 'quit', %Exit or Ctrl-D.", level=1)
         except self.custom_exceptions:
             etype,value,tb = sys.exc_info()
             self.CustomTB(etype,value,tb)
         except:
             self.showtraceback()
         else:
             outflag = 0
             if softspace(sys.stdout, 0):
                 print
 
         # Execute any registered post-execution functions.  Here, any errors
         # are reported only minimally and just on the terminal, because the
         # main exception channel may be occupied with a user traceback.
         # FIXME: we need to think this mechanism a little more carefully.
         if post_execute:
             for func in self._post_execute:
                 try:
                     func()
                 except:
                     head = '[ ERROR ] Evaluating post_execute function: %s' % \
                            func
                     print >> io.Term.cout, head
                     print >> io.Term.cout, self._simple_error()
                     print >> io.Term.cout, 'Removing from post_execute'
                     self._post_execute.remove(func)
 
         # Flush out code object which has been run (and source)
         self.code_to_run = None
         return outflag
         
     def push_line(self, line):
         """Push a line to the interpreter.
 
         The line should not have a trailing newline; it may have
         internal newlines.  The line is appended to a buffer and the
         interpreter's runsource() method is called with the
         concatenated contents of the buffer as source.  If this
         indicates that the command was executed or invalid, the buffer
         is reset; otherwise, the command is incomplete, and the buffer
         is left as it was after the line was appended.  The return
         value is 1 if more input is required, 0 if the line was dealt
         with in some way (this is the same as runsource()).
         """
 
         # autoindent management should be done here, and not in the
         # interactive loop, since that one is only seen by keyboard input.  We
         # need this done correctly even for code run via runlines (which uses
         # push).
 
         #print 'push line: <%s>' % line  # dbg
         for subline in line.splitlines():
             self._autoindent_update(subline)
         self.buffer.append(line)
         more = self.runsource('\n'.join(self.buffer), self.filename)
         if not more:
             self.resetbuffer()
         return more
 
     def resetbuffer(self):
         """Reset the input buffer."""
         self.buffer[:] = []
 
     def _is_secondary_block_start(self, s):
         if not s.endswith(':'):
             return False
         if (s.startswith('elif') or 
             s.startswith('else') or 
             s.startswith('except') or
             s.startswith('finally')):
             return True
 
     def _cleanup_ipy_script(self, script):
         """Make a script safe for self.runlines()
 
         Currently, IPython is lines based, with blocks being detected by
         empty lines.  This is a problem for block based scripts that may
         not have empty lines after blocks.  This script adds those empty
         lines to make scripts safe for running in the current line based
         IPython.
         """
         res = []
         lines = script.splitlines()
         level = 0
 
         for l in lines:
             lstripped = l.lstrip()
             stripped = l.strip()                
             if not stripped:
                 continue
             newlevel = len(l) - len(lstripped)                    
             if level > 0 and newlevel == 0 and \
                    not self._is_secondary_block_start(stripped): 
                 # add empty line
                 res.append('')
             res.append(l)
             level = newlevel
 
         return '\n'.join(res) + '\n'
 
     def _autoindent_update(self,line):
         """Keep track of the indent level."""
 
         #debugx('line')
         #debugx('self.indent_current_nsp')
         if self.autoindent:
             if line:
                 inisp = num_ini_spaces(line)
                 if inisp < self.indent_current_nsp:
                     self.indent_current_nsp = inisp
 
                 if line[-1] == ':':
                     self.indent_current_nsp += 4
                 elif dedent_re.match(line):
                     self.indent_current_nsp -= 4
             else:
                 self.indent_current_nsp = 0
 
     #-------------------------------------------------------------------------
     # Things related to GUI support and pylab
     #-------------------------------------------------------------------------
 
     def enable_pylab(self, gui=None):
         raise NotImplementedError('Implement enable_pylab in a subclass')
 
     #-------------------------------------------------------------------------
     # Utilities
     #-------------------------------------------------------------------------
 
     def var_expand(self,cmd,depth=0):
         """Expand python variables in a string.
 
         The depth argument indicates how many frames above the caller should
         be walked to look for the local namespace where to expand variables.
 
         The global namespace for expansion is always the user's interactive
         namespace.
         """
 
         return str(ItplNS(cmd,
                           self.user_ns,  # globals
                           # Skip our own frame in searching for locals:
                           sys._getframe(depth+1).f_locals # locals
                           ))
 
     def mktempfile(self,data=None):
         """Make a new tempfile and return its filename.
 
         This makes a call to tempfile.mktemp, but it registers the created
         filename internally so ipython cleans it up at exit time.
 
         Optional inputs:
 
           - data(None): if data is given, it gets written out to the temp file
           immediately, and the file is closed again."""
 
         filename = tempfile.mktemp('.py','ipython_edit_')
         self.tempfiles.append(filename)
         
         if data:
             tmp_file = open(filename,'w')
             tmp_file.write(data)
             tmp_file.close()
         return filename
 
     # TODO:  This should be removed when Term is refactored.
     def write(self,data):
         """Write a string to the default output"""
         io.Term.cout.write(data)
 
     # TODO:  This should be removed when Term is refactored.
     def write_err(self,data):
         """Write a string to the default error output"""
         io.Term.cerr.write(data)
 
     def ask_yes_no(self,prompt,default=True):
         if self.quiet:
             return True
         return ask_yes_no(prompt,default)
     
     def show_usage(self):
         """Show a usage message"""
         page.page(IPython.core.usage.interactive_usage)
 
     #-------------------------------------------------------------------------
     # Things related to IPython exiting
     #-------------------------------------------------------------------------
     def atexit_operations(self):
         """This will be executed at the time of exit.
 
         Cleanup operations and saving of persistent data that is done
         unconditionally by IPython should be performed here.
 
         For things that may depend on startup flags or platform specifics (such
         as having readline or not), register a separate atexit function in the
         code that has the appropriate information, rather than trying to
         clutter 
         """
         # Cleanup all tempfiles left around
         for tfile in self.tempfiles:
             try:
                 os.unlink(tfile)
             except OSError:
                 pass
 
         # Clear all user namespaces to release all references cleanly.
         self.reset()
 
         # Run user hooks
         self.hooks.shutdown_hook()
 
     def cleanup(self):
         self.restore_sys_module_state()
 
 
 class InteractiveShellABC(object):
     """An abstract base class for InteractiveShell."""
     __metaclass__ = abc.ABCMeta
 
 InteractiveShellABC.register(InteractiveShell)

IPython/core/prefilter.py

@@ -1,1014 +1,1014 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 Prefiltering components.
 
 Prefilters transform user input before it is exec'd by Python.  These
 transforms are used to implement additional syntax such as !ls and %magic.
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 * Dan Milstein
 * Ville Vainio
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import __builtin__
 import codeop
 import re
 
 from IPython.core.alias import AliasManager
 from IPython.core.autocall import IPyAutocall
 from IPython.config.configurable import Configurable
 from IPython.core.splitinput import split_user_input
 from IPython.core import page
 
 from IPython.utils.traitlets import List, Int, Any, Str, CBool, Bool, Instance
 import IPython.utils.io
 from IPython.utils.text import make_quoted_expr
 from IPython.utils.autoattr import auto_attr
 
 #-----------------------------------------------------------------------------
 # Global utilities, errors and constants
 #-----------------------------------------------------------------------------
 
 # Warning, these cannot be changed unless various regular expressions
 # are updated in a number of places.  Not great, but at least we told you.
 ESC_SHELL  = '!'
 ESC_SH_CAP = '!!'
 ESC_HELP   = '?'
 ESC_MAGIC  = '%'
 ESC_QUOTE  = ','
 ESC_QUOTE2 = ';'
 ESC_PAREN  = '/'
 
 
 class PrefilterError(Exception):
     pass
 
 
 # RegExp to identify potential function names
 re_fun_name = re.compile(r'[a-zA-Z_]([a-zA-Z0-9_.]*) *$')
 
 # RegExp to exclude strings with this start from autocalling.  In
 # particular, all binary operators should be excluded, so that if foo is
 # callable, foo OP bar doesn't become foo(OP bar), which is invalid.  The
 # characters '!=()' don't need to be checked for, as the checkPythonChars
 # routine explicitely does so, to catch direct calls and rebindings of
 # existing names.
 
 # Warning: the '-' HAS TO BE AT THE END of the first group, otherwise
 # it affects the rest of the group in square brackets.
 re_exclude_auto = re.compile(r'^[,&^\|\*/\+-]'
                              r'|^is |^not |^in |^and |^or ')
 
 # try to catch also methods for stuff in lists/tuples/dicts: off
 # (experimental). For this to work, the line_split regexp would need
 # to be modified so it wouldn't break things at '['. That line is
 # nasty enough that I shouldn't change it until I can test it _well_.
 #self.re_fun_name = re.compile (r'[a-zA-Z_]([a-zA-Z0-9_.\[\]]*) ?$')
 
 
 # Handler Check Utilities
 def is_shadowed(identifier, ip):
     """Is the given identifier defined in one of the namespaces which shadow
     the alias and magic namespaces?  Note that an identifier is different
     than ifun, because it can not contain a '.' character."""
     # This is much safer than calling ofind, which can change state
     return (identifier in ip.user_ns \
             or identifier in ip.internal_ns \
             or identifier in ip.ns_table['builtin'])
 
 
 #-----------------------------------------------------------------------------
 # The LineInfo class used throughout
 #-----------------------------------------------------------------------------
 
 
 class LineInfo(object):
     """A single line of input and associated info.
 
     Includes the following as properties: 
 
     line
       The original, raw line
     
     continue_prompt
       Is this line a continuation in a sequence of multiline input?
     
     pre
       The initial esc character or whitespace.
     
     pre_char
       The escape character(s) in pre or the empty string if there isn't one.
       Note that '!!' is a possible value for pre_char.  Otherwise it will
       always be a single character.
     
     pre_whitespace
       The leading whitespace from pre if it exists.  If there is a pre_char,
       this is just ''.
     
     ifun
       The 'function part', which is basically the maximal initial sequence
       of valid python identifiers and the '.' character.  This is what is
       checked for alias and magic transformations, used for auto-calling,
       etc.
     
     the_rest
       Everything else on the line.
     """
     def __init__(self, line, continue_prompt):
         self.line            = line
         self.continue_prompt = continue_prompt
         self.pre, self.ifun, self.the_rest = split_user_input(line)
 
         self.pre_char       = self.pre.strip()
         if self.pre_char:
             self.pre_whitespace = '' # No whitespace allowd before esc chars
         else: 
             self.pre_whitespace = self.pre
 
         self._oinfo = None
 
     def ofind(self, ip):
         """Do a full, attribute-walking lookup of the ifun in the various
         namespaces for the given IPython InteractiveShell instance.
 
         Return a dict with keys: found,obj,ospace,ismagic
 
         Note: can cause state changes because of calling getattr, but should
         only be run if autocall is on and if the line hasn't matched any
         other, less dangerous handlers.
 
         Does cache the results of the call, so can be called multiple times
         without worrying about *further* damaging state.
         """
         if not self._oinfo:
             # ip.shell._ofind is actually on the Magic class!
             self._oinfo = ip.shell._ofind(self.ifun)
         return self._oinfo
 
     def __str__(self):                                                         
         return "Lineinfo [%s|%s|%s]" %(self.pre, self.ifun, self.the_rest) 
 
 
 #-----------------------------------------------------------------------------
 # Main Prefilter manager
 #-----------------------------------------------------------------------------
 
 
 class PrefilterManager(Configurable):
     """Main prefilter component.
 
     The IPython prefilter is run on all user input before it is run.  The
     prefilter consumes lines of input and produces transformed lines of 
     input.
 
     The iplementation consists of two phases:
 
     1. Transformers
     2. Checkers and handlers
 
     Over time, we plan on deprecating the checkers and handlers and doing
     everything in the transformers.
 
     The transformers are instances of :class:`PrefilterTransformer` and have
     a single method :meth:`transform` that takes a line and returns a
     transformed line.  The transformation can be accomplished using any
     tool, but our current ones use regular expressions for speed.  We also
     ship :mod:`pyparsing` in :mod:`IPython.external` for use in transformers.
 
     After all the transformers have been run, the line is fed to the checkers,
     which are instances of :class:`PrefilterChecker`.  The line is passed to
     the :meth:`check` method, which either returns `None` or a 
     :class:`PrefilterHandler` instance.  If `None` is returned, the other
     checkers are tried.  If an :class:`PrefilterHandler` instance is returned,
     the line is passed to the :meth:`handle` method of the returned
     handler and no further checkers are tried.
     
     Both transformers and checkers have a `priority` attribute, that determines
     the order in which they are called.  Smaller priorities are tried first.
 
     Both transformers and checkers also have `enabled` attribute, which is
     a boolean that determines if the instance is used.
 
     Users or developers can change the priority or enabled attribute of
     transformers or checkers, but they must call the :meth:`sort_checkers`
     or :meth:`sort_transformers` method after changing the priority.
     """
 
     multi_line_specials = CBool(True, config=True)
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
 
     def __init__(self, shell=None, config=None):
         super(PrefilterManager, self).__init__(shell=shell, config=config)
         self.shell = shell
         self.init_transformers()
         self.init_handlers()
         self.init_checkers()
 
     #-------------------------------------------------------------------------
     # API for managing transformers
     #-------------------------------------------------------------------------
 
     def init_transformers(self):
         """Create the default transformers."""
         self._transformers = []
         for transformer_cls in _default_transformers:
             transformer_cls(
                 shell=self.shell, prefilter_manager=self, config=self.config
             )
 
     def sort_transformers(self):
         """Sort the transformers by priority.
 
         This must be called after the priority of a transformer is changed.
         The :meth:`register_transformer` method calls this automatically.
         """
-        self._transformers.sort(cmp=lambda x,y: x.priority-y.priority)
+        self._transformers.sort(key=lambda x: x.priority)
 
     @property
     def transformers(self):
         """Return a list of checkers, sorted by priority."""
         return self._transformers
 
     def register_transformer(self, transformer):
         """Register a transformer instance."""
         if transformer not in self._transformers:
             self._transformers.append(transformer)
             self.sort_transformers()
 
     def unregister_transformer(self, transformer):
         """Unregister a transformer instance."""
         if transformer in self._transformers:
             self._transformers.remove(transformer)
 
     #-------------------------------------------------------------------------
     # API for managing checkers
     #-------------------------------------------------------------------------
 
     def init_checkers(self):
         """Create the default checkers."""
         self._checkers = []
         for checker in _default_checkers:
             checker(
                 shell=self.shell, prefilter_manager=self, config=self.config
             )
 
     def sort_checkers(self):
         """Sort the checkers by priority.
 
         This must be called after the priority of a checker is changed.
         The :meth:`register_checker` method calls this automatically.
         """
-        self._checkers.sort(cmp=lambda x,y: x.priority-y.priority)
+        self._checkers.sort(key=lambda x: x.priority)
 
     @property
     def checkers(self):
         """Return a list of checkers, sorted by priority."""
         return self._checkers
 
     def register_checker(self, checker):
         """Register a checker instance."""
         if checker not in self._checkers:
             self._checkers.append(checker)
             self.sort_checkers()
 
     def unregister_checker(self, checker):
         """Unregister a checker instance."""
         if checker in self._checkers:
             self._checkers.remove(checker)
 
     #-------------------------------------------------------------------------
     # API for managing checkers
     #-------------------------------------------------------------------------
 
     def init_handlers(self):
         """Create the default handlers."""
         self._handlers = {}
         self._esc_handlers = {}
         for handler in _default_handlers:
             handler(
                 shell=self.shell, prefilter_manager=self, config=self.config
             )
 
     @property
     def handlers(self):
         """Return a dict of all the handlers."""
         return self._handlers
 
     def register_handler(self, name, handler, esc_strings):
         """Register a handler instance by name with esc_strings."""
         self._handlers[name] = handler
         for esc_str in esc_strings:
             self._esc_handlers[esc_str] = handler
 
     def unregister_handler(self, name, handler, esc_strings):
         """Unregister a handler instance by name with esc_strings."""
         try:
             del self._handlers[name]
         except KeyError:
             pass
         for esc_str in esc_strings:
             h = self._esc_handlers.get(esc_str)
             if h is handler:
                 del self._esc_handlers[esc_str]
 
     def get_handler_by_name(self, name):
         """Get a handler by its name."""
         return self._handlers.get(name)
 
     def get_handler_by_esc(self, esc_str):
         """Get a handler by its escape string."""
         return self._esc_handlers.get(esc_str)
 
     #-------------------------------------------------------------------------
     # Main prefiltering API
     #-------------------------------------------------------------------------
 
     def prefilter_line_info(self, line_info):
         """Prefilter a line that has been converted to a LineInfo object.
 
         This implements the checker/handler part of the prefilter pipe.
         """
         # print "prefilter_line_info: ", line_info
         handler = self.find_handler(line_info)
         return handler.handle(line_info)
 
     def find_handler(self, line_info):
         """Find a handler for the line_info by trying checkers."""
         for checker in self.checkers:
             if checker.enabled:
                 handler = checker.check(line_info)
                 if handler:
                     return handler
         return self.get_handler_by_name('normal')
 
     def transform_line(self, line, continue_prompt):
         """Calls the enabled transformers in order of increasing priority."""
         for transformer in self.transformers:
             if transformer.enabled:
                 line = transformer.transform(line, continue_prompt)
         return line
 
     def prefilter_line(self, line, continue_prompt=False):
         """Prefilter a single input line as text.
 
         This method prefilters a single line of text by calling the
         transformers and then the checkers/handlers.
         """
 
         # print "prefilter_line: ", line, continue_prompt
         # All handlers *must* return a value, even if it's blank ('').
 
         # Lines are NOT logged here. Handlers should process the line as
         # needed, update the cache AND log it (so that the input cache array
         # stays synced).
 
         # save the line away in case we crash, so the post-mortem handler can
         # record it
         self.shell._last_input_line = line
 
         if not line:
             # Return immediately on purely empty lines, so that if the user
             # previously typed some whitespace that started a continuation
             # prompt, he can break out of that loop with just an empty line.
             # This is how the default python prompt works.
 
             # Only return if the accumulated input buffer was just whitespace!
             if ''.join(self.shell.buffer).isspace():
                 self.shell.buffer[:] = []
             return ''
 
         # At this point, we invoke our transformers.
         if not continue_prompt or (continue_prompt and self.multi_line_specials):
             line = self.transform_line(line, continue_prompt)
 
         # Now we compute line_info for the checkers and handlers
         line_info = LineInfo(line, continue_prompt)
         
         # the input history needs to track even empty lines
         stripped = line.strip()
 
         normal_handler = self.get_handler_by_name('normal')
         if not stripped:
             if not continue_prompt:
                 self.shell.displayhook.prompt_count -= 1
 
             return normal_handler.handle(line_info)
 
         # special handlers are only allowed for single line statements
         if continue_prompt and not self.multi_line_specials:
             return normal_handler.handle(line_info)
 
         prefiltered = self.prefilter_line_info(line_info)
         # print "prefiltered line: %r" % prefiltered
         return prefiltered
 
     def prefilter_lines(self, lines, continue_prompt=False):
         """Prefilter multiple input lines of text.
 
         This is the main entry point for prefiltering multiple lines of
         input.  This simply calls :meth:`prefilter_line` for each line of
         input.
 
         This covers cases where there are multiple lines in the user entry,
         which is the case when the user goes back to a multiline history
         entry and presses enter.
         """
         llines = lines.rstrip('\n').split('\n')
         # We can get multiple lines in one shot, where multiline input 'blends'
         # into one line, in cases like recalling from the readline history
         # buffer.  We need to make sure that in such cases, we correctly
         # communicate downstream which line is first and which are continuation
         # ones.
         if len(llines) > 1:
             out = '\n'.join([self.prefilter_line(line, lnum>0)
                              for lnum, line in enumerate(llines) ])
         else:
             out = self.prefilter_line(llines[0], continue_prompt)
             
         return out
 
 #-----------------------------------------------------------------------------
 # Prefilter transformers
 #-----------------------------------------------------------------------------
 
 
 class PrefilterTransformer(Configurable):
     """Transform a line of user input."""
 
     priority = Int(100, config=True)
     # Transformers don't currently use shell or prefilter_manager, but as we
     # move away from checkers and handlers, they will need them.
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
     prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager')
     enabled = Bool(True, config=True)
 
     def __init__(self, shell=None, prefilter_manager=None, config=None):
         super(PrefilterTransformer, self).__init__(
             shell=shell, prefilter_manager=prefilter_manager, config=config
         )
         self.prefilter_manager.register_transformer(self)
 
     def transform(self, line, continue_prompt):
         """Transform a line, returning the new one."""
         return None
 
     def __repr__(self):
         return "<%s(priority=%r, enabled=%r)>" % (
             self.__class__.__name__, self.priority, self.enabled)
 
 
 _assign_system_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
                                r'\s*=\s*!(?P<cmd>.*)')
 
 
 class AssignSystemTransformer(PrefilterTransformer):
     """Handle the `files = !ls` syntax."""
 
     priority = Int(100, config=True)
 
     def transform(self, line, continue_prompt):
         m = _assign_system_re.match(line)
         if m is not None:
             cmd = m.group('cmd')
             lhs = m.group('lhs')
             expr = make_quoted_expr("sc =%s" % cmd)
             new_line = '%s = get_ipython().magic(%s)' % (lhs, expr)
             return new_line
         return line
 
 
 _assign_magic_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
                                r'\s*=\s*%(?P<cmd>.*)')
 
 class AssignMagicTransformer(PrefilterTransformer):
     """Handle the `a = %who` syntax."""
 
     priority = Int(200, config=True)
 
     def transform(self, line, continue_prompt):
         m = _assign_magic_re.match(line)
         if m is not None:
             cmd = m.group('cmd')
             lhs = m.group('lhs')
             expr = make_quoted_expr(cmd)
             new_line = '%s = get_ipython().magic(%s)' % (lhs, expr)
             return new_line
         return line
 
 
 _classic_prompt_re = re.compile(r'(^[ \t]*>>> |^[ \t]*\.\.\. )')
 
 class PyPromptTransformer(PrefilterTransformer):
     """Handle inputs that start with '>>> ' syntax."""
 
     priority = Int(50, config=True)
 
     def transform(self, line, continue_prompt):
 
         if not line or line.isspace() or line.strip() == '...':
             # This allows us to recognize multiple input prompts separated by
             # blank lines and pasted in a single chunk, very common when
             # pasting doctests or long tutorial passages.
             return ''
         m = _classic_prompt_re.match(line)
         if m:
             return line[len(m.group(0)):]
         else:
             return line
 
 
 _ipy_prompt_re = re.compile(r'(^[ \t]*In \[\d+\]: |^[ \t]*\ \ \ \.\.\.+: )')
 
 class IPyPromptTransformer(PrefilterTransformer):
     """Handle inputs that start classic IPython prompt syntax."""
 
     priority = Int(50, config=True)
 
     def transform(self, line, continue_prompt):
 
         if not line or line.isspace() or line.strip() == '...':
             # This allows us to recognize multiple input prompts separated by
             # blank lines and pasted in a single chunk, very common when
             # pasting doctests or long tutorial passages.
             return ''
         m = _ipy_prompt_re.match(line)
         if m:
             return line[len(m.group(0)):]
         else:
             return line
 
 #-----------------------------------------------------------------------------
 # Prefilter checkers
 #-----------------------------------------------------------------------------
 
 
 class PrefilterChecker(Configurable):
     """Inspect an input line and return a handler for that line."""
 
     priority = Int(100, config=True)
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
     prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager')
     enabled = Bool(True, config=True)
 
     def __init__(self, shell=None, prefilter_manager=None, config=None):
         super(PrefilterChecker, self).__init__(
             shell=shell, prefilter_manager=prefilter_manager, config=config
         )
         self.prefilter_manager.register_checker(self)
 
     def check(self, line_info):
         """Inspect line_info and return a handler instance or None."""
         return None
 
     def __repr__(self):
         return "<%s(priority=%r, enabled=%r)>" % (
             self.__class__.__name__, self.priority, self.enabled)
 
 
 class EmacsChecker(PrefilterChecker):
 
     priority = Int(100, config=True)
     enabled = Bool(False, config=True)
 
     def check(self, line_info):
         "Emacs ipython-mode tags certain input lines."
         if line_info.line.endswith('# PYTHON-MODE'):
             return self.prefilter_manager.get_handler_by_name('emacs')
         else:
             return None
 
 
 class ShellEscapeChecker(PrefilterChecker):
 
     priority = Int(200, config=True)
 
     def check(self, line_info):
         if line_info.line.lstrip().startswith(ESC_SHELL):
             return self.prefilter_manager.get_handler_by_name('shell')
 
 
 class IPyAutocallChecker(PrefilterChecker):
 
     priority = Int(300, config=True)
 
     def check(self, line_info):
         "Instances of IPyAutocall in user_ns get autocalled immediately"
         obj = self.shell.user_ns.get(line_info.ifun, None)
         if isinstance(obj, IPyAutocall):
             obj.set_ip(self.shell)
             return self.prefilter_manager.get_handler_by_name('auto')
         else:
             return None
 
 
 class MultiLineMagicChecker(PrefilterChecker):
 
     priority = Int(400, config=True)
 
     def check(self, line_info):
         "Allow ! and !! in multi-line statements if multi_line_specials is on"
         # Note that this one of the only places we check the first character of
         # ifun and *not* the pre_char.  Also note that the below test matches
         # both ! and !!.    
         if line_info.continue_prompt \
             and self.prefilter_manager.multi_line_specials:
                 if line_info.ifun.startswith(ESC_MAGIC):
                     return self.prefilter_manager.get_handler_by_name('magic')
         else:
             return None
 
 
 class EscCharsChecker(PrefilterChecker):
 
     priority = Int(500, config=True)
 
     def check(self, line_info):
         """Check for escape character and return either a handler to handle it,
         or None if there is no escape char."""
         if line_info.line[-1] == ESC_HELP \
                and line_info.pre_char != ESC_SHELL \
                and line_info.pre_char != ESC_SH_CAP:
             # the ? can be at the end, but *not* for either kind of shell escape,
             # because a ? can be a vaild final char in a shell cmd
             return self.prefilter_manager.get_handler_by_name('help')
         else:
             # This returns None like it should if no handler exists
             return self.prefilter_manager.get_handler_by_esc(line_info.pre_char)
 
 
 class AssignmentChecker(PrefilterChecker):
 
     priority = Int(600, config=True)
 
     def check(self, line_info):
         """Check to see if user is assigning to a var for the first time, in
         which case we want to avoid any sort of automagic / autocall games.
     
         This allows users to assign to either alias or magic names true python
         variables (the magic/alias systems always take second seat to true
         python code).  E.g. ls='hi', or ls,that=1,2"""
         if line_info.the_rest:
             if line_info.the_rest[0] in '=,':
                 return self.prefilter_manager.get_handler_by_name('normal')
         else:
             return None
 
 
 class AutoMagicChecker(PrefilterChecker):
 
     priority = Int(700, config=True)
 
     def check(self, line_info):
         """If the ifun is magic, and automagic is on, run it.  Note: normal,
         non-auto magic would already have been triggered via '%' in
         check_esc_chars. This just checks for automagic.  Also, before
         triggering the magic handler, make sure that there is nothing in the
         user namespace which could shadow it."""
         if not self.shell.automagic or not hasattr(self.shell,'magic_'+line_info.ifun):
             return None
 
         # We have a likely magic method.  Make sure we should actually call it.
         if line_info.continue_prompt and not self.prefilter_manager.multi_line_specials:
             return None
 
         head = line_info.ifun.split('.',1)[0]
         if is_shadowed(head, self.shell):
             return None
 
         return self.prefilter_manager.get_handler_by_name('magic')
 
 
 class AliasChecker(PrefilterChecker):
 
     priority = Int(800, config=True)
 
     def check(self, line_info):
         "Check if the initital identifier on the line is an alias."
         # Note: aliases can not contain '.'
         head = line_info.ifun.split('.',1)[0]
         if line_info.ifun not in self.shell.alias_manager \
                or head not in self.shell.alias_manager \
                or is_shadowed(head, self.shell):
             return None
 
         return self.prefilter_manager.get_handler_by_name('alias')
 
 
 class PythonOpsChecker(PrefilterChecker):
 
     priority = Int(900, config=True)
 
     def check(self, line_info):
         """If the 'rest' of the line begins with a function call or pretty much
         any python operator, we should simply execute the line (regardless of
         whether or not there's a possible autocall expansion).  This avoids
         spurious (and very confusing) geattr() accesses."""
         if line_info.the_rest and line_info.the_rest[0] in '!=()<>,+*/%^&|':
             return self.prefilter_manager.get_handler_by_name('normal')
         else:
             return None
 
 
 class AutocallChecker(PrefilterChecker):
 
     priority = Int(1000, config=True)
 
     def check(self, line_info):
         "Check if the initial word/function is callable and autocall is on."
         if not self.shell.autocall:
             return None
 
         oinfo = line_info.ofind(self.shell) # This can mutate state via getattr
         if not oinfo['found']:
             return None
         
         if callable(oinfo['obj']) \
                and (not re_exclude_auto.match(line_info.the_rest)) \
                and re_fun_name.match(line_info.ifun):
             return self.prefilter_manager.get_handler_by_name('auto')
         else:
             return None
 
 
 #-----------------------------------------------------------------------------
 # Prefilter handlers
 #-----------------------------------------------------------------------------
 
 
 class PrefilterHandler(Configurable):
 
     handler_name = Str('normal')
     esc_strings = List([])
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
     prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager')
 
     def __init__(self, shell=None, prefilter_manager=None, config=None):
         super(PrefilterHandler, self).__init__(
             shell=shell, prefilter_manager=prefilter_manager, config=config
         )
         self.prefilter_manager.register_handler(
             self.handler_name,
             self,
             self.esc_strings
         )
 
     def handle(self, line_info):
         # print "normal: ", line_info
         """Handle normal input lines. Use as a template for handlers."""
 
         # With autoindent on, we need some way to exit the input loop, and I
         # don't want to force the user to have to backspace all the way to
         # clear the line.  The rule will be in this case, that either two
         # lines of pure whitespace in a row, or a line of pure whitespace but
         # of a size different to the indent level, will exit the input loop.
         line = line_info.line
         continue_prompt = line_info.continue_prompt
 
         if (continue_prompt and
             self.shell.autoindent and
             line.isspace() and
             
             (0 < abs(len(line) - self.shell.indent_current_nsp) <= 2
              or
              not self.shell.buffer
              or
              (self.shell.buffer[-1]).isspace()
              )
             ):
             line = ''
 
         self.shell.log(line, line, continue_prompt)
         return line
 
     def __str__(self):
         return "<%s(name=%s)>" % (self.__class__.__name__, self.handler_name)
 
 
 class AliasHandler(PrefilterHandler):
 
     handler_name = Str('alias')
 
     def handle(self, line_info):
         """Handle alias input lines. """
         transformed = self.shell.alias_manager.expand_aliases(line_info.ifun,line_info.the_rest)
         # pre is needed, because it carries the leading whitespace.  Otherwise
         # aliases won't work in indented sections.
         line_out = '%sget_ipython().system(%s)' % (line_info.pre_whitespace,
                                          make_quoted_expr(transformed))
         
         self.shell.log(line_info.line, line_out, line_info.continue_prompt)
         return line_out
 
 
 class ShellEscapeHandler(PrefilterHandler):
 
     handler_name = Str('shell')
     esc_strings = List([ESC_SHELL, ESC_SH_CAP])
 
     def handle(self, line_info):
         """Execute the line in a shell, empty return value"""
         magic_handler = self.prefilter_manager.get_handler_by_name('magic')
 
         line = line_info.line
         if line.lstrip().startswith(ESC_SH_CAP):
             # rewrite LineInfo's line, ifun and the_rest to properly hold the
             # call to %sx and the actual command to be executed, so
             # handle_magic can work correctly.  Note that this works even if
             # the line is indented, so it handles multi_line_specials
             # properly.
             new_rest = line.lstrip()[2:]
             line_info.line = '%ssx %s' % (ESC_MAGIC, new_rest)
             line_info.ifun = 'sx'
             line_info.the_rest = new_rest
             return magic_handler.handle(line_info)
         else:
             cmd = line.lstrip().lstrip(ESC_SHELL)
             line_out = '%sget_ipython().system(%s)' % (line_info.pre_whitespace,
                                              make_quoted_expr(cmd))
         # update cache/log and return
         self.shell.log(line, line_out, line_info.continue_prompt)
         return line_out
 
 
 class MagicHandler(PrefilterHandler):
 
     handler_name = Str('magic')
     esc_strings = List([ESC_MAGIC])
 
     def handle(self, line_info):
         """Execute magic functions."""
         ifun    = line_info.ifun
         the_rest = line_info.the_rest
         cmd = '%sget_ipython().magic(%s)' % (line_info.pre_whitespace,
                                    make_quoted_expr(ifun + " " + the_rest))
         self.shell.log(line_info.line, cmd, line_info.continue_prompt)
         return cmd
 
 
 class AutoHandler(PrefilterHandler):
 
     handler_name = Str('auto')
     esc_strings = List([ESC_PAREN, ESC_QUOTE, ESC_QUOTE2])
 
     def handle(self, line_info):
         """Handle lines which can be auto-executed, quoting if requested."""
         line    = line_info.line
         ifun    = line_info.ifun
         the_rest = line_info.the_rest
         pre     = line_info.pre
         continue_prompt = line_info.continue_prompt
         obj = line_info.ofind(self)['obj']
         #print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun,the_rest)  # dbg
 
         # This should only be active for single-line input!
         if continue_prompt:
             self.shell.log(line,line,continue_prompt)
             return line
 
         force_auto = isinstance(obj, IPyAutocall)
         auto_rewrite = True
         
         if pre == ESC_QUOTE:
             # Auto-quote splitting on whitespace
             newcmd = '%s("%s")' % (ifun,'", "'.join(the_rest.split()) )
         elif pre == ESC_QUOTE2:
             # Auto-quote whole string
             newcmd = '%s("%s")' % (ifun,the_rest)
         elif pre == ESC_PAREN:
             newcmd = '%s(%s)' % (ifun,",".join(the_rest.split()))
         else:
             # Auto-paren.
             # We only apply it to argument-less calls if the autocall
             # parameter is set to 2.  We only need to check that autocall is <
             # 2, since this function isn't called unless it's at least 1.
             if not the_rest and (self.shell.autocall < 2) and not force_auto:
                 newcmd = '%s %s' % (ifun,the_rest)
                 auto_rewrite = False
             else:
                 if not force_auto and the_rest.startswith('['):
                     if hasattr(obj,'__getitem__'):
                         # Don't autocall in this case: item access for an object
                         # which is BOTH callable and implements __getitem__.
                         newcmd = '%s %s' % (ifun,the_rest)
                         auto_rewrite = False
                     else:
                         # if the object doesn't support [] access, go ahead and
                         # autocall
                         newcmd = '%s(%s)' % (ifun.rstrip(),the_rest)
                 elif the_rest.endswith(';'):
                     newcmd = '%s(%s);' % (ifun.rstrip(),the_rest[:-1])
                 else:
                     newcmd = '%s(%s)' % (ifun.rstrip(), the_rest)
 
         if auto_rewrite:
             self.shell.auto_rewrite_input(newcmd)
             
         # log what is now valid Python, not the actual user input (without the
         # final newline)
         self.shell.log(line,newcmd,continue_prompt)
         return newcmd
 
 
 class HelpHandler(PrefilterHandler):
 
     handler_name = Str('help')
     esc_strings = List([ESC_HELP])
 
     def handle(self, line_info):
         """Try to get some help for the object.
 
         obj? or ?obj   -> basic information.
         obj?? or ??obj -> more details.
         """
         normal_handler = self.prefilter_manager.get_handler_by_name('normal')
         line = line_info.line
         # We need to make sure that we don't process lines which would be
         # otherwise valid python, such as "x=1 # what?"
         try:
             codeop.compile_command(line)
         except SyntaxError:
             # We should only handle as help stuff which is NOT valid syntax
             if line[0]==ESC_HELP:
                 line = line[1:]
             elif line[-1]==ESC_HELP:
                 line = line[:-1]
             self.shell.log(line, '#?'+line, line_info.continue_prompt)
             if line:
                 #print 'line:<%r>' % line  # dbg
                 self.shell.magic_pinfo(line)
             else:
                 self.shell.show_usage()
             return '' # Empty string is needed here!
         except:
             raise
             # Pass any other exceptions through to the normal handler
             return normal_handler.handle(line_info)
         else:
             # If the code compiles ok, we should handle it normally
             return normal_handler.handle(line_info)
 
 
 class EmacsHandler(PrefilterHandler):
 
     handler_name = Str('emacs')
     esc_strings = List([])
 
     def handle(self, line_info):
         """Handle input lines marked by python-mode."""
 
         # Currently, nothing is done.  Later more functionality can be added
         # here if needed.
 
         # The input cache shouldn't be updated
         return line_info.line
 
 
 #-----------------------------------------------------------------------------
 # Defaults
 #-----------------------------------------------------------------------------
 
 
 _default_transformers = [
     AssignSystemTransformer,
     AssignMagicTransformer,
     PyPromptTransformer,
     IPyPromptTransformer,
 ]
 
 _default_checkers = [
     EmacsChecker,
     ShellEscapeChecker,
     IPyAutocallChecker,
     MultiLineMagicChecker,
     EscCharsChecker,
     AssignmentChecker,
     AutoMagicChecker,
     AliasChecker,
     PythonOpsChecker,
     AutocallChecker
 ]
 
 _default_handlers = [
     PrefilterHandler,
     AliasHandler,
     ShellEscapeHandler,
     MagicHandler,
     AutoHandler,
     HelpHandler,
     EmacsHandler
 ]

IPython/core/tests/test_inputsplitter.py

@@ -1,658 +1,658 b''
 # -*- coding: utf-8 -*-
 """Tests for the inputsplitter module.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 # stdlib
 import unittest
 import sys
 
 # Third party
 import nose.tools as nt
 
 # Our own
 from IPython.core import inputsplitter as isp
 
 #-----------------------------------------------------------------------------
 # Semi-complete examples (also used as tests)
 #-----------------------------------------------------------------------------
 
 # Note: at the bottom, there's a slightly more complete version of this that
 # can be useful during development of code here.
 
-def mini_interactive_loop(raw_input):
+def mini_interactive_loop(input_func):
     """Minimal example of the logic of an interactive interpreter loop.
 
     This serves as an example, and it is used by the test system with a fake
     raw_input that simulates interactive input."""
 
     from IPython.core.inputsplitter import InputSplitter
     
     isp = InputSplitter()
     # In practice, this input loop would be wrapped in an outside loop to read
     # input indefinitely, until some exit/quit command was issued.  Here we
     # only illustrate the basic inner loop.
     while isp.push_accepts_more():
         indent = ' '*isp.indent_spaces
         prompt = '>>> ' + indent
-        line = indent + raw_input(prompt)
+        line = indent + input_func(prompt)
         isp.push(line)
 
     # Here we just return input so we can use it in a test suite, but a real
     # interpreter would instead send it for execution somewhere.
     src = isp.source_reset()
     #print 'Input source was:\n', src  # dbg
     return src
 
 #-----------------------------------------------------------------------------
 # Test utilities, just for local use
 #-----------------------------------------------------------------------------
 
 def assemble(block):
     """Assemble a block into multi-line sub-blocks."""
     return ['\n'.join(sub_block)+'\n' for sub_block in block]
 
 
 def pseudo_input(lines):
     """Return a function that acts like raw_input but feeds the input list."""
     ilines = iter(lines)
     def raw_in(prompt):
         try:
             return next(ilines)
         except StopIteration:
             return ''
     return raw_in
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 def test_spaces():
     tests = [('', 0),
              (' ', 1),
              ('\n', 0),
              (' \n', 1),
              ('x', 0),
              (' x', 1),
              ('  x',2),
              ('    x',4),
              # Note: tabs are counted as a single whitespace!
              ('\tx', 1),
              ('\t x', 2),
              ]
     
     for s, nsp in tests:
         nt.assert_equal(isp.num_ini_spaces(s), nsp)
 
 
 def test_remove_comments():
     tests = [('text', 'text'),
              ('text # comment', 'text '),
              ('text # comment\n', 'text \n'),
              ('text # comment \n', 'text \n'),
              ('line # c \nline\n','line \nline\n'),
              ('line # c \nline#c2  \nline\nline #c\n\n',
               'line \nline\nline\nline \n\n'),
              ]
 
     for inp, out in tests:
         nt.assert_equal(isp.remove_comments(inp), out)
 
 
 def test_get_input_encoding():
     encoding = isp.get_input_encoding()
     nt.assert_true(isinstance(encoding, basestring))
     # simple-minded check that at least encoding a simple string works with the
     # encoding we got.
     nt.assert_equal('test'.encode(encoding), 'test')
 
 
 class NoInputEncodingTestCase(unittest.TestCase):
     def setUp(self):
         self.old_stdin = sys.stdin
         class X: pass
         fake_stdin = X()
         sys.stdin = fake_stdin
         
     def test(self):
         # Verify that if sys.stdin has no 'encoding' attribute we do the right
         # thing
         enc = isp.get_input_encoding()
         self.assertEqual(enc, 'ascii')
         
     def tearDown(self):
         sys.stdin = self.old_stdin
 
 
 class InputSplitterTestCase(unittest.TestCase):
     def setUp(self):
         self.isp = isp.InputSplitter()
 
     def test_reset(self):
         isp = self.isp
         isp.push('x=1')
         isp.reset()
         self.assertEqual(isp._buffer, [])
         self.assertEqual(isp.indent_spaces, 0)
         self.assertEqual(isp.source, '')
         self.assertEqual(isp.code, None)
         self.assertEqual(isp._is_complete, False)
 
     def test_source(self):
         self.isp._store('1')
         self.isp._store('2')
         self.assertEqual(self.isp.source, '1\n2\n')
         self.assertTrue(len(self.isp._buffer)>0)
         self.assertEqual(self.isp.source_reset(), '1\n2\n')
         self.assertEqual(self.isp._buffer, [])
         self.assertEqual(self.isp.source, '')
         
     def test_indent(self):
         isp = self.isp # shorthand
         isp.push('x=1')
         self.assertEqual(isp.indent_spaces, 0)
         isp.push('if 1:\n    x=1')
         self.assertEqual(isp.indent_spaces, 4)
         isp.push('y=2\n')
         self.assertEqual(isp.indent_spaces, 0)
         isp.push('if 1:')
         self.assertEqual(isp.indent_spaces, 4)
         isp.push('    x=1')
         self.assertEqual(isp.indent_spaces, 4)
         # Blank lines shouldn't change the indent level
         isp.push(' '*2)
         self.assertEqual(isp.indent_spaces, 4)
 
     def test_indent2(self):
         isp = self.isp
         # When a multiline statement contains parens or multiline strings, we
         # shouldn't get confused.
         isp.push("if 1:")
         isp.push("    x = (1+\n    2)")
         self.assertEqual(isp.indent_spaces, 4)
 
     def test_dedent(self):
         isp = self.isp # shorthand
         isp.push('if 1:')
         self.assertEqual(isp.indent_spaces, 4)
         isp.push('    pass')
         self.assertEqual(isp.indent_spaces, 0)
         
     def test_push(self):
         isp = self.isp
         self.assertTrue(isp.push('x=1'))
 
     def test_push2(self):
         isp = self.isp
         self.assertFalse(isp.push('if 1:'))
         for line in ['  x=1', '# a comment', '  y=2']:
             self.assertTrue(isp.push(line))
             
     def test_push3(self):
         """Test input with leading whitespace"""
         isp = self.isp
         isp.push('  x=1')
         isp.push('  y=2')
         self.assertEqual(isp.source, 'if 1:\n  x=1\n  y=2\n')
 
     def test_replace_mode(self):
         isp = self.isp
         isp.input_mode = 'cell'
         isp.push('x=1')
         self.assertEqual(isp.source, 'x=1\n')
         isp.push('x=2')
         self.assertEqual(isp.source, 'x=2\n')
 
     def test_push_accepts_more(self):
         isp = self.isp
         isp.push('x=1')
         self.assertFalse(isp.push_accepts_more())
 
     def test_push_accepts_more2(self):
         isp = self.isp
         isp.push('if 1:')
         self.assertTrue(isp.push_accepts_more())
         isp.push('  x=1')
         self.assertTrue(isp.push_accepts_more())
         isp.push('')
         self.assertFalse(isp.push_accepts_more())
         
     def test_push_accepts_more3(self):
         isp = self.isp
         isp.push("x = (2+\n3)")
         self.assertFalse(isp.push_accepts_more())
 
     def test_push_accepts_more4(self):
         isp = self.isp
         # When a multiline statement contains parens or multiline strings, we
         # shouldn't get confused.
         # FIXME: we should be able to better handle de-dents in statements like
         # multiline strings and multiline expressions (continued with \ or
         # parens).  Right now we aren't handling the indentation tracking quite
         # correctly with this, though in practice it may not be too much of a
         # problem.  We'll need to see.
         isp.push("if 1:")
         isp.push("    x = (2+")
         isp.push("    3)")
         self.assertTrue(isp.push_accepts_more())
         isp.push("    y = 3")
         self.assertTrue(isp.push_accepts_more())
         isp.push('')
         self.assertFalse(isp.push_accepts_more())
 
     def test_continuation(self):
         isp = self.isp
         isp.push("import os, \\")
         self.assertTrue(isp.push_accepts_more())
         isp.push("sys")
         self.assertFalse(isp.push_accepts_more())
 
     def test_syntax_error(self):
         isp = self.isp
         # Syntax errors immediately produce a 'ready' block, so the invalid
         # Python can be sent to the kernel for evaluation with possible ipython
         # special-syntax conversion.
         isp.push('run foo')
         self.assertFalse(isp.push_accepts_more())
 
     def check_split(self, block_lines, compile=True):
         blocks = assemble(block_lines)
         lines = ''.join(blocks)
         oblock = self.isp.split_blocks(lines)
         self.assertEqual(oblock, blocks)
         if compile:
             for block in blocks:
                 self.isp._compile(block)
 
     def test_split(self):
         # All blocks of input we want to test in a list.  The format for each
         # block is a list of lists, with each inner lists consisting of all the
         # lines (as single-lines) that should make up a sub-block.
 
         # Note: do NOT put here sub-blocks that don't compile, as the
         # check_split() routine makes a final verification pass to check that
         # each sub_block, as returned by split_blocks(), does compile
         # correctly.
         all_blocks = [ [['x=1']],
 
                        [['x=1'],
                         ['y=2']],
 
                        [['x=1',
                          '# a comment'],
                         ['y=11']],
 
                        [['if 1:',
                          '  x=1'],
                         ['y=3']],
 
                        [['def f(x):',
                          '  return x'],
                         ['x=1']],
 
                        [['def f(x):',
                          '  x+=1',
                          '  ',
                          '  return x'],
                         ['x=1']],
 
                        [['def f(x):',
                          '  if x>0:',
                          '    y=1',
                          '  # a comment',
                          '  else:',
                          '    y=4',
                          ' ',
                          '  return y'],
                         ['x=1'],
                         ['if 1:',
                          '  y=11'] ],
                        
                        [['for i in range(10):'
                          '  x=i**2']],
 
                        [['for i in range(10):'
                          '  x=i**2'],
                         ['z = 1']],
                        ]
         for block_lines in all_blocks:
             self.check_split(block_lines)
         
     def test_split_syntax_errors(self):
         # Block splitting with invalid syntax
         all_blocks = [ [['a syntax error']],
             
                        [['x=1',
                          'another syntax error']],
 
                        [['for i in range(10):'
                          '  yet another error']],
                        
                        ]
         for block_lines in all_blocks:
             self.check_split(block_lines, compile=False)
 
 
 class InteractiveLoopTestCase(unittest.TestCase):
     """Tests for an interactive loop like a python shell.
     """
     def check_ns(self, lines, ns):
         """Validate that the given input lines produce the resulting namespace.
 
         Note: the input lines are given exactly as they would be typed in an
         auto-indenting environment, as mini_interactive_loop above already does
         auto-indenting and prepends spaces to the input.
         """
         src = mini_interactive_loop(pseudo_input(lines))
         test_ns = {}
         exec src in test_ns
         # We can't check that the provided ns is identical to the test_ns,
         # because Python fills test_ns with extra keys (copyright, etc).  But
         # we can check that the given dict is *contained* in test_ns
-        for k,v in ns.items():
+        for k,v in ns.iteritems():
             self.assertEqual(test_ns[k], v)
         
     def test_simple(self):
         self.check_ns(['x=1'], dict(x=1))
 
     def test_simple2(self):
         self.check_ns(['if 1:', 'x=2'], dict(x=2))
 
     def test_xy(self):
         self.check_ns(['x=1; y=2'], dict(x=1, y=2))
 
     def test_abc(self):
         self.check_ns(['if 1:','a=1','b=2','c=3'], dict(a=1, b=2, c=3))
     
     def test_multi(self):
         self.check_ns(['x =(1+','1+','2)'], dict(x=4))
     
 
 def test_LineInfo():
     """Simple test for LineInfo construction and str()"""
     linfo = isp.LineInfo('  %cd /home')
     nt.assert_equals(str(linfo), 'LineInfo [  |%|cd|/home]')
 
 
 def test_split_user_input():
     """Unicode test - split_user_input already has good doctests"""
     line = u"Pérez Fernando"
     parts = isp.split_user_input(line)
     parts_expected = (u'', u'', u'', line)
     nt.assert_equal(parts, parts_expected)
 
 
 # Transformer tests
 def transform_checker(tests, func):
     """Utility to loop over test inputs"""
     for inp, tr in tests:
         nt.assert_equals(func(inp), tr)
 
 # Data for all the syntax tests in the form of lists of pairs of
 # raw/transformed input.  We store it here as a global dict so that we can use
 # it both within single-function tests and also to validate the behavior of the
 # larger objects
 
 syntax = \
   dict(assign_system =
        [('a =! ls', 'a = get_ipython().getoutput("ls")'),
         ('b = !ls', 'b = get_ipython().getoutput("ls")'),
         ('x=1', 'x=1'), # normal input is unmodified
         ('    ','    '),  # blank lines are kept intact
         ],
 
        assign_magic =
        [('a =% who', 'a = get_ipython().magic("who")'),
         ('b = %who', 'b = get_ipython().magic("who")'),
         ('x=1', 'x=1'), # normal input is unmodified
         ('    ','    '),  # blank lines are kept intact
         ],
 
        classic_prompt =
        [('>>> x=1', 'x=1'),
         ('x=1', 'x=1'), # normal input is unmodified
         ('    ', '    '),  # blank lines are kept intact
         ('... ', ''), # continuation prompts
         ],
 
        ipy_prompt =
        [('In [1]: x=1', 'x=1'),
         ('x=1', 'x=1'), # normal input is unmodified
         ('    ','    '),  # blank lines are kept intact
         ('   ....: ', ''), # continuation prompts
         ],
 
        # Tests for the escape transformer to leave normal code alone
        escaped_noesc =
        [ ('    ', '    '),
          ('x=1', 'x=1'),
          ],
 
        # System calls
        escaped_shell =
        [ ('!ls', 'get_ipython().system("ls")'),
          # Double-escape shell, this means to capture the output of the
          # subprocess and return it
          ('!!ls', 'get_ipython().getoutput("ls")'),
          ],
 
        # Help/object info
        escaped_help =
        [ ('?', 'get_ipython().show_usage()'),
          ('?x1', 'get_ipython().magic("pinfo x1")'),
          ('??x2', 'get_ipython().magic("pinfo2 x2")'),
          ('x3?', 'get_ipython().magic("pinfo x3")'),
          ('x4??', 'get_ipython().magic("pinfo2 x4")'),
          ('%hist?', 'get_ipython().magic("pinfo %hist")'),
          ('f*?', 'get_ipython().magic("psearch f*")'),
          ('ax.*aspe*?', 'get_ipython().magic("psearch ax.*aspe*")'),
          ],
 
        # Explicit magic calls
        escaped_magic =
        [ ('%cd', 'get_ipython().magic("cd")'),
          ('%cd /home', 'get_ipython().magic("cd /home")'),
          ('    %magic', '    get_ipython().magic("magic")'),
          ],
        
        # Quoting with separate arguments
        escaped_quote =
        [ (',f', 'f("")'),
          (',f x', 'f("x")'),
          ('  ,f y', '  f("y")'),
          (',f a b', 'f("a", "b")'),
          ],
        
        # Quoting with single argument
        escaped_quote2 = 
        [ (';f', 'f("")'),
          (';f x', 'f("x")'),
          ('  ;f y', '  f("y")'),
          (';f a b', 'f("a b")'),
          ],
        
        # Simply apply parens
        escaped_paren = 
        [ ('/f', 'f()'),
          ('/f x', 'f(x)'),
          ('  /f y', '  f(y)'),
          ('/f a b', 'f(a, b)'),
          ],
 
        )
 
 # multiline syntax examples.  Each of these should be a list of lists, with
 # each entry itself having pairs of raw/transformed input.  The union (with
 # '\n'.join() of the transformed inputs is what the splitter should produce
 # when fed the raw lines one at a time via push.
 syntax_ml = \
   dict(classic_prompt =
        [ [('>>> for i in range(10):','for i in range(10):'),
           ('...     print i','    print i'),
           ('... ', ''),
           ],
         ],
 
        ipy_prompt =
        [ [('In [24]: for i in range(10):','for i in range(10):'),
           ('   ....:     print i','    print i'),
           ('   ....: ', ''),
           ],
          ],
        )
 
 
 def test_assign_system():
     transform_checker(syntax['assign_system'], isp.transform_assign_system)
 
     
 def test_assign_magic():
     transform_checker(syntax['assign_magic'], isp.transform_assign_magic)
 
 
 def test_classic_prompt():
     transform_checker(syntax['classic_prompt'], isp.transform_classic_prompt)
     for example in syntax_ml['classic_prompt']:
         transform_checker(example, isp.transform_classic_prompt)
     
 
 def test_ipy_prompt():
     transform_checker(syntax['ipy_prompt'], isp.transform_ipy_prompt)
     for example in syntax_ml['ipy_prompt']:
         transform_checker(example, isp.transform_ipy_prompt)
     
 
 def test_escaped_noesc():
     transform_checker(syntax['escaped_noesc'], isp.transform_escaped)
 
 
 def test_escaped_shell():
     transform_checker(syntax['escaped_shell'], isp.transform_escaped)
 
 
 def test_escaped_help():
     transform_checker(syntax['escaped_help'], isp.transform_escaped)
 
 
 def test_escaped_magic():
     transform_checker(syntax['escaped_magic'], isp.transform_escaped)
 
 
 def test_escaped_quote():
     transform_checker(syntax['escaped_quote'], isp.transform_escaped)
 
 
 def test_escaped_quote2():
     transform_checker(syntax['escaped_quote2'], isp.transform_escaped)
 
 
 def test_escaped_paren():
     transform_checker(syntax['escaped_paren'], isp.transform_escaped)
 
 
 class IPythonInputTestCase(InputSplitterTestCase):
     """By just creating a new class whose .isp is a different instance, we
     re-run the same test battery on the new input splitter.
 
     In addition, this runs the tests over the syntax and syntax_ml dicts that
     were tested by individual functions, as part of the OO interface.
     """
 
     def setUp(self):
         self.isp = isp.IPythonInputSplitter(input_mode='line')
 
     def test_syntax(self):
         """Call all single-line syntax tests from the main object"""
         isp = self.isp
         for example in syntax.itervalues():
             for raw, out_t in example:
                 if raw.startswith(' '):
                     continue
                 
                 isp.push(raw)
                 out = isp.source_reset().rstrip()
                 self.assertEqual(out, out_t)
 
     def test_syntax_multiline(self):
         isp = self.isp
         for example in syntax_ml.itervalues():
             out_t_parts = []
             for line_pairs in example:
                 for raw, out_t_part in line_pairs:
                     isp.push(raw)
                     out_t_parts.append(out_t_part)
 
                 out = isp.source_reset().rstrip()
                 out_t = '\n'.join(out_t_parts).rstrip()
                 self.assertEqual(out, out_t)
                 
 
 class BlockIPythonInputTestCase(IPythonInputTestCase):
 
     # Deactivate tests that don't make sense for the block mode
     test_push3 = test_split = lambda s: None
     
     def setUp(self):
         self.isp = isp.IPythonInputSplitter(input_mode='cell')
 
     def test_syntax_multiline(self):
         isp = self.isp
         for example in syntax_ml.itervalues():
             raw_parts = []
             out_t_parts = []
             for line_pairs in example:
                 for raw, out_t_part in line_pairs:
                     raw_parts.append(raw)
                     out_t_parts.append(out_t_part)
 
                 raw = '\n'.join(raw_parts)
                 out_t = '\n'.join(out_t_parts)
 
                 isp.push(raw)
                 out = isp.source_reset()
                 # Match ignoring trailing whitespace
                 self.assertEqual(out.rstrip(), out_t.rstrip())
     
 
 #-----------------------------------------------------------------------------
 # Main - use as a script, mostly for developer experiments
 #-----------------------------------------------------------------------------
 
 if __name__ == '__main__':
     # A simple demo for interactive experimentation.  This code will not get
     # picked up by any test suite.
     from IPython.core.inputsplitter import InputSplitter, IPythonInputSplitter
 
     # configure here the syntax to use, prompt and whether to autoindent
     #isp, start_prompt = InputSplitter(), '>>> '
     isp, start_prompt = IPythonInputSplitter(), 'In> '
 
     autoindent = True
     #autoindent = False
     
     try:
         while True:
             prompt = start_prompt
             while isp.push_accepts_more():
                 indent = ' '*isp.indent_spaces
                 if autoindent:
                     line = indent + raw_input(prompt+indent)
                 else:
                     line = raw_input(prompt)
                 isp.push(line)
                 prompt = '... '
 
             # Here we just return input so we can use it in a test suite, but a
             # real interpreter would instead send it for execution somewhere.
             #src = isp.source; raise EOFError # dbg
             src = isp.source_reset()
             print 'Input source was:\n', src
     except EOFError:
         print 'Bye'

IPython/core/tests/test_magic.py

@@ -1,357 +1,357 b''
 """Tests for various magic functions.
 
 Needs to be run by nose (to make ipython session available).
 """
 from __future__ import absolute_import
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 import tempfile
 import types
 from cStringIO import StringIO
 
 import nose.tools as nt
 
 from IPython.utils.path import get_long_path_name
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 
 #-----------------------------------------------------------------------------
 # Test functions begin
 #-----------------------------------------------------------------------------
 def test_rehashx():
     # clear up everything
     _ip = get_ipython()
     _ip.alias_manager.alias_table.clear()
     del _ip.db['syscmdlist']
     
     _ip.magic('rehashx')
     # Practically ALL ipython development systems will have more than 10 aliases
 
     yield (nt.assert_true, len(_ip.alias_manager.alias_table) > 10)
-    for key, val in _ip.alias_manager.alias_table.items():
+    for key, val in _ip.alias_manager.alias_table.iteritems():
         # we must strip dots from alias names
         nt.assert_true('.' not in key)
 
     # rehashx must fill up syscmdlist
     scoms = _ip.db['syscmdlist']
     yield (nt.assert_true, len(scoms) > 10)
 
 
 def test_magic_parse_options():
     """Test that we don't mangle paths when parsing magic options."""
     ip = get_ipython()
     path = 'c:\\x'
     opts = ip.parse_options('-f %s' % path,'f:')[0]
     # argv splitting is os-dependent
     if os.name == 'posix':
         expected = 'c:x'
     else:
         expected = path
     nt.assert_equals(opts['f'], expected)
 
     
 def doctest_hist_f():
     """Test %hist -f with temporary filename.
 
     In [9]: import tempfile
 
     In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
 
     In [11]: %hist -n -f $tfile 3
 
     In [13]: import os; os.unlink(tfile)
     """
 
 
 def doctest_hist_r():
     """Test %hist -r
 
     XXX - This test is not recording the output correctly.  For some reason, in
     testing mode the raw history isn't getting populated.  No idea why.
     Disabling the output checking for now, though at least we do run it.
 
     In [1]: 'hist' in _ip.lsmagic()
     Out[1]: True
 
     In [2]: x=1
 
     In [3]: %hist -r 2
     x=1 # random
     %hist -r 2
     """
 
 def doctest_hist_op():
     """Test %hist -op
 
     In [1]: class b:
        ...:         pass
        ...: 
 
     In [2]: class s(b):
        ...:         def __str__(self):
        ...:             return 's'
        ...: 
 
     In [3]: 
 
     In [4]: class r(b):
        ...:         def __repr__(self):
        ...:             return 'r'
        ...: 
 
     In [5]: class sr(s,r): pass
        ...: 
 
     In [6]: 
 
     In [7]: bb=b()
 
     In [8]: ss=s()
 
     In [9]: rr=r()
 
     In [10]: ssrr=sr()
 
     In [11]: bb
     Out[11]: <...b instance at ...>
 
     In [12]: ss
     Out[12]: <...s instance at ...>
 
     In [13]: 
 
     In [14]: %hist -op
     >>> class b:
     ...     pass
     ... 
     >>> class s(b):
     ...     def __str__(self):
     ...         return 's'
     ... 
     >>> 
     >>> class r(b):
     ...     def __repr__(self):
     ...         return 'r'
     ... 
     >>> class sr(s,r): pass
     >>> 
     >>> bb=b()
     >>> ss=s()
     >>> rr=r()
     >>> ssrr=sr()
     >>> bb
     <...b instance at ...>
     >>> ss
     <...s instance at ...>
     >>> 
     """
 
 def test_shist():
     # Simple tests of ShadowHist class - test generator.
     import os, shutil, tempfile
 
     from IPython.utils import pickleshare
     from IPython.core.history import ShadowHist
     
     tfile = tempfile.mktemp('','tmp-ipython-')
     
     db = pickleshare.PickleShareDB(tfile)
     s = ShadowHist(db)
     s.add('hello')
     s.add('world')
     s.add('hello')
     s.add('hello')
     s.add('karhu')
 
     yield nt.assert_equals,s.all(),[(1, 'hello'), (2, 'world'), (3, 'karhu')]
     
     yield nt.assert_equal,s.get(2),'world'
     
     shutil.rmtree(tfile)
 
     
 # XXX failing for now, until we get clearcmd out of quarantine.  But we should
 # fix this and revert the skip to happen only if numpy is not around.
 #@dec.skipif_not_numpy
 @dec.skip_known_failure
 def test_numpy_clear_array_undec():
     from IPython.extensions import clearcmd
 
     _ip.ex('import numpy as np')
     _ip.ex('a = np.empty(2)')
     yield (nt.assert_true, 'a' in _ip.user_ns)
     _ip.magic('clear array')
     yield (nt.assert_false, 'a' in _ip.user_ns)
     
 
 # Multiple tests for clipboard pasting
 @dec.parametric
 def test_paste():
     _ip = get_ipython()
     def paste(txt, flags='-q'):
         """Paste input text, by default in quiet mode"""
         hooks.clipboard_get = lambda : txt
         _ip.magic('paste '+flags)
 
     # Inject fake clipboard hook but save original so we can restore it later
     hooks = _ip.hooks
     user_ns = _ip.user_ns
     original_clip = hooks.clipboard_get
 
     try:
         # This try/except with an emtpy except clause is here only because
         # try/yield/finally is invalid syntax in Python 2.4.  This will be
         # removed when we drop 2.4-compatibility, and the emtpy except below
         # will be changed to a finally.
 
         # Run tests with fake clipboard function
         user_ns.pop('x', None)
         paste('x=1')
         yield nt.assert_equal(user_ns['x'], 1)
 
         user_ns.pop('x', None)
         paste('>>> x=2')
         yield nt.assert_equal(user_ns['x'], 2)
 
         paste("""
         >>> x = [1,2,3]
         >>> y = []
         >>> for i in x:
         ...     y.append(i**2)
         ...
         """)
         yield nt.assert_equal(user_ns['x'], [1,2,3])
         yield nt.assert_equal(user_ns['y'], [1,4,9])
 
         # Now, test that paste -r works
         user_ns.pop('x', None)
         yield nt.assert_false('x' in user_ns)
         _ip.magic('paste -r')
         yield nt.assert_equal(user_ns['x'], [1,2,3])
 
         # Also test paste echoing, by temporarily faking the writer
         w = StringIO()
         writer = _ip.write
         _ip.write = w.write
         code = """
         a = 100
         b = 200"""
         try:
             paste(code,'')
             out = w.getvalue()
         finally:
             _ip.write = writer
         yield nt.assert_equal(user_ns['a'], 100)
         yield nt.assert_equal(user_ns['b'], 200)
         yield nt.assert_equal(out, code+"\n## -- End pasted text --\n")
         
     finally:
         # This should be in a finally clause, instead of the bare except above.
         # Restore original hook
         hooks.clipboard_get = original_clip
 
 
 def test_time():
     _ip.magic('time None')
 
 
 def doctest_time():
     """
     In [10]: %time None
     CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
     Wall time: 0.00 s
     """
 
 
 def test_doctest_mode():
     "Toggle doctest_mode twice, it should be a no-op and run without error"
     _ip.magic('doctest_mode')
     _ip.magic('doctest_mode')
 
 
 def test_parse_options():
     """Tests for basic options parsing in magics."""
     # These are only the most minimal of tests, more should be added later.  At
     # the very least we check that basic text/unicode calls work OK.
     nt.assert_equal(_ip.parse_options('foo', '')[1], 'foo')
     nt.assert_equal(_ip.parse_options(u'foo', '')[1], u'foo')
 
     
 def test_dirops():
     """Test various directory handling operations."""
     curpath = lambda :os.path.splitdrive(os.getcwd())[1].replace('\\','/')
 
     startdir = os.getcwd()
     ipdir = _ip.ipython_dir
     try:
         _ip.magic('cd "%s"' % ipdir)
         nt.assert_equal(curpath(), ipdir)
         _ip.magic('cd -')
         nt.assert_equal(curpath(), startdir)
         _ip.magic('pushd "%s"' % ipdir)
         nt.assert_equal(curpath(), ipdir)
         _ip.magic('popd')
         nt.assert_equal(curpath(), startdir)
     finally:
         os.chdir(startdir)
 
 
 def check_cpaste(code, should_fail=False):
     """Execute code via 'cpaste' and ensure it was executed, unless
     should_fail is set.
     """
     _ip.user_ns['code_ran'] = False
 
     src = StringIO()
     src.write('\n')
     src.write(code)
     src.write('\n--\n')
     src.seek(0)
 
     stdin_save = sys.stdin
     sys.stdin = src
     
     try:
         _ip.magic('cpaste')
     except:
         if not should_fail:
             raise AssertionError("Failure not expected : '%s'" %
                                  code)
     else:
         assert _ip.user_ns['code_ran']
         if should_fail:
             raise AssertionError("Failure expected : '%s'" % code)
     finally:
         sys.stdin = stdin_save
 
 
 def test_cpaste():
     """Test cpaste magic"""
 
     def run():
         """Marker function: sets a flag when executed.
         """
         _ip.user_ns['code_ran'] = True
         return 'run' # return string so '+ run()' doesn't result in success
 
     tests = {'pass': ["> > > run()",
                       ">>> > run()",
                       "+++ run()",
                       "++ run()",
                       "  >>> run()"],
 
              'fail': ["+ + run()",
                       " ++ run()"]}
 
     _ip.user_ns['run'] = run
 
     for code in tests['pass']:
         check_cpaste(code)
 
     for code in tests['fail']:
         check_cpaste(code, should_fail=True)

IPython/core/ultratb.py

@@ -1,1224 +1,1224 b''
 # -*- coding: utf-8 -*-
 """
 ultratb.py -- Spice up your tracebacks!
 
 * ColorTB
 I've always found it a bit hard to visually parse tracebacks in Python.  The
 ColorTB class is a solution to that problem.  It colors the different parts of a
 traceback in a manner similar to what you would expect from a syntax-highlighting
 text editor.
 
 Installation instructions for ColorTB:
     import sys,ultratb
     sys.excepthook = ultratb.ColorTB()
 
 * VerboseTB  
 I've also included a port of Ka-Ping Yee's "cgitb.py" that produces all kinds
 of useful info when a traceback occurs.  Ping originally had it spit out HTML
 and intended it for CGI programmers, but why should they have all the fun?  I
 altered it to spit out colored text to the terminal.  It's a bit overwhelming,
 but kind of neat, and maybe useful for long-running programs that you believe
 are bug-free.  If a crash *does* occur in that type of program you want details.
 Give it a shot--you'll love it or you'll hate it.
 
 Note:
 
   The Verbose mode prints the variables currently visible where the exception
   happened (shortening their strings if too long). This can potentially be
   very slow, if you happen to have a huge data structure whose string
   representation is complex to compute. Your computer may appear to freeze for
   a while with cpu usage at 100%. If this occurs, you can cancel the traceback
   with Ctrl-C (maybe hitting it more than once).
 
   If you encounter this kind of situation often, you may want to use the
   Verbose_novars mode instead of the regular Verbose, which avoids formatting
   variables (but otherwise includes the information and context given by
   Verbose).
   
 
 Installation instructions for ColorTB:
     import sys,ultratb
     sys.excepthook = ultratb.VerboseTB()
 
 Note:  Much of the code in this module was lifted verbatim from the standard
 library module 'traceback.py' and Ka-Ping Yee's 'cgitb.py'.
 
 * Color schemes
 The colors are defined in the class TBTools through the use of the
 ColorSchemeTable class. Currently the following exist:
 
   - NoColor: allows all of this module to be used in any terminal (the color
   escapes are just dummy blank strings).
 
   - Linux: is meant to look good in a terminal like the Linux console (black
   or very dark background).
 
   - LightBG: similar to Linux but swaps dark/light colors to be more readable
   in light background terminals.
 
 You can implement other color schemes easily, the syntax is fairly
 self-explanatory. Please send back new schemes you develop to the author for
 possible inclusion in future releases.
 """
 
 #*****************************************************************************
 #       Copyright (C) 2001 Nathaniel Gray <[e-mail]>
 #       Copyright (C) 2001-2004 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from __future__ import with_statement
 
 import inspect
 import keyword
 import linecache
 import os
 import pydoc
 import re
 import string
 import sys
 import time
 import tokenize
 import traceback
 import types
 
 # For purposes of monkeypatching inspect to fix a bug in it.
 from inspect import getsourcefile, getfile, getmodule,\
      ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode
 
 # IPython's own modules
 # Modified pdb which doesn't damage IPython's readline handling
 from IPython.core import debugger, ipapi
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.excolors import exception_colors
 from IPython.utils import PyColorize
 from IPython.utils import io
 from IPython.utils.data import uniq_stable
 from IPython.utils.warn import info, error
 
 # Globals
 # amount of space to put line numbers before verbose tracebacks
 INDENT_SIZE = 8
 
 # Default color scheme.  This is used, for example, by the traceback
 # formatter.  When running in an actual IPython instance, the user's rc.colors
 # value is used, but havinga module global makes this functionality available
 # to users of ultratb who are NOT running inside ipython.
 DEFAULT_SCHEME = 'NoColor'
 
 #---------------------------------------------------------------------------
 # Code begins
 
 # Utility functions
 def inspect_error():
     """Print a message about internal inspect errors.
 
     These are unfortunately quite common."""
     
     error('Internal Python error in the inspect module.\n'
           'Below is the traceback from this internal error.\n')
 
 
 def findsource(object):
     """Return the entire source file and starting line number for an object.
 
     The argument may be a module, class, method, function, traceback, frame,
     or code object.  The source code is returned as a list of all the lines
     in the file and the line number indexes a line in that list.  An IOError
     is raised if the source code cannot be retrieved.
 
     FIXED version with which we monkeypatch the stdlib to work around a bug."""
 
     file = getsourcefile(object) or getfile(object)
     # If the object is a frame, then trying to get the globals dict from its
     # module won't work. Instead, the frame object itself has the globals
     # dictionary.
     globals_dict = None
     if inspect.isframe(object):
         # XXX: can this ever be false?
         globals_dict = object.f_globals
     else:
         module = getmodule(object, file)
         if module:
             globals_dict = module.__dict__
     lines = linecache.getlines(file, globals_dict)
     if not lines:
         raise IOError('could not get source code')
 
     if ismodule(object):
         return lines, 0
 
     if isclass(object):
         name = object.__name__
         pat = re.compile(r'^(\s*)class\s*' + name + r'\b')
         # make some effort to find the best matching class definition:
         # use the one with the least indentation, which is the one
         # that's most probably not inside a function definition.
         candidates = []
         for i in range(len(lines)):
             match = pat.match(lines[i])
             if match:
                 # if it's at toplevel, it's already the best one
                 if lines[i][0] == 'c':
                     return lines, i
                 # else add whitespace to candidate list
                 candidates.append((match.group(1), i))
         if candidates:
             # this will sort by whitespace, and by line number,
             # less whitespace first
             candidates.sort()
             return lines, candidates[0][1]
         else:
             raise IOError('could not find class definition')
 
     if ismethod(object):
         object = object.im_func
     if isfunction(object):
         object = object.func_code
     if istraceback(object):
         object = object.tb_frame
     if isframe(object):
         object = object.f_code
     if iscode(object):
         if not hasattr(object, 'co_firstlineno'):
             raise IOError('could not find function definition')
         pat = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
         pmatch = pat.match
         # fperez - fix: sometimes, co_firstlineno can give a number larger than
         # the length of lines, which causes an error.  Safeguard against that.
         lnum = min(object.co_firstlineno,len(lines))-1
         while lnum > 0:
             if pmatch(lines[lnum]): break
             lnum -= 1
  
         return lines, lnum
     raise IOError('could not find code object')
 
 # Monkeypatch inspect to apply our bugfix.  This code only works with py25
 if sys.version_info[:2] >= (2,5):
     inspect.findsource = findsource
 
 def fix_frame_records_filenames(records):
     """Try to fix the filenames in each record from inspect.getinnerframes().
 
     Particularly, modules loaded from within zip files have useless filenames
     attached to their code object, and inspect.getinnerframes() just uses it.
     """
     fixed_records = []
     for frame, filename, line_no, func_name, lines, index in records:
         # Look inside the frame's globals dictionary for __file__, which should
         # be better.
         better_fn = frame.f_globals.get('__file__', None)
         if isinstance(better_fn, str):
             # Check the type just in case someone did something weird with
             # __file__. It might also be None if the error occurred during
             # import.
             filename = better_fn
         fixed_records.append((frame, filename, line_no, func_name, lines, index))
     return fixed_records
 
 
 def _fixed_getinnerframes(etb, context=1,tb_offset=0):
     import linecache
     LNUM_POS, LINES_POS, INDEX_POS =  2, 4, 5
 
     records  = fix_frame_records_filenames(inspect.getinnerframes(etb, context))
 
     # If the error is at the console, don't build any context, since it would
     # otherwise produce 5 blank lines printed out (there is no file at the
     # console)
     rec_check = records[tb_offset:]
     try:
         rname = rec_check[0][1]
         if rname == '<ipython console>' or rname.endswith('<string>'):
             return rec_check
     except IndexError:
         pass
 
     aux = traceback.extract_tb(etb)
     assert len(records) == len(aux)
     for i, (file, lnum, _, _) in zip(range(len(records)), aux):
         maybeStart = lnum-1 - context//2
         start =  max(maybeStart, 0)
         end   = start + context
         lines = linecache.getlines(file)[start:end]
         # pad with empty lines if necessary
         if maybeStart < 0:
             lines = (['\n'] * -maybeStart) + lines
         if len(lines) < context:
             lines += ['\n'] * (context - len(lines))
         buf = list(records[i])
         buf[LNUM_POS] = lnum
         buf[INDEX_POS] = lnum - 1 - start
         buf[LINES_POS] = lines
         records[i] = tuple(buf)
     return records[tb_offset:]
 
 # Helper function -- largely belongs to VerboseTB, but we need the same
 # functionality to produce a pseudo verbose TB for SyntaxErrors, so that they
 # can be recognized properly by ipython.el's py-traceback-line-re
 # (SyntaxErrors have to be treated specially because they have no traceback)
 
 _parser = PyColorize.Parser()
     
 def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,scheme=None):
     numbers_width = INDENT_SIZE - 1
     res = []
     i = lnum - index
 
     # This lets us get fully syntax-highlighted tracebacks.
     if scheme is None:
         ipinst = ipapi.get()
         if ipinst is not None:
             scheme = ipinst.colors
         else:
             scheme = DEFAULT_SCHEME
 
     _line_format = _parser.format2
 
     for line in lines:
         new_line, err = _line_format(line,'str',scheme)
         if not err: line = new_line
         
         if i == lnum:
             # This is the line with the error
             pad = numbers_width - len(str(i))
             if pad >= 3:
                 marker = '-'*(pad-3) + '-> '
             elif pad == 2:
                 marker = '> '
             elif pad == 1:
                 marker = '>'
             else:
                 marker = ''
             num = marker + str(i)
             line = '%s%s%s %s%s' %(Colors.linenoEm, num, 
                                    Colors.line, line, Colors.Normal)
         else:
             num = '%*s' % (numbers_width,i)
             line = '%s%s%s %s' %(Colors.lineno, num, 
                                  Colors.Normal, line)
 
         res.append(line)
         if lvals and i == lnum:
             res.append(lvals + '\n')
         i = i + 1
     return res
 
 
 #---------------------------------------------------------------------------
 # Module classes
 class TBTools(object):
     """Basic tools used by all traceback printer classes."""
 
     # Number of frames to skip when reporting tracebacks
     tb_offset = 0
 
     def __init__(self, color_scheme='NoColor', call_pdb=False, ostream=None):
         # Whether to call the interactive pdb debugger after printing
         # tracebacks or not
         self.call_pdb = call_pdb
 
         # Output stream to write to.  Note that we store the original value in
         # a private attribute and then make the public ostream a property, so
         # that we can delay accessing io.Term.cout until runtime.  The way
         # things are written now, the Term.cout object is dynamically managed
         # so a reference to it should NEVER be stored statically.  This
         # property approach confines this detail to a single location, and all
         # subclasses can simply access self.ostream for writing.
         self._ostream = ostream
 
         # Create color table
         self.color_scheme_table = exception_colors()
 
         self.set_colors(color_scheme)
         self.old_scheme = color_scheme  # save initial value for toggles
 
         if call_pdb:
             self.pdb = debugger.Pdb(self.color_scheme_table.active_scheme_name)
         else:
             self.pdb = None
 
     def _get_ostream(self):
         """Output stream that exceptions are written to.
 
         Valid values are:
         
         - None: the default, which means that IPython will dynamically resolve
         to io.Term.cout.  This ensures compatibility with most tools, including
         Windows (where plain stdout doesn't recognize ANSI escapes).
 
         - Any object with 'write' and 'flush' attributes.
         """
         return io.Term.cout if self._ostream is None else self._ostream
 
     def _set_ostream(self, val):
         assert val is None or (hasattr(val, 'write') and hasattr(val, 'flush'))
         self._ostream = val
         
     ostream = property(_get_ostream, _set_ostream)
 
     def set_colors(self,*args,**kw):
         """Shorthand access to the color table scheme selector method."""
 
         # Set own color table
         self.color_scheme_table.set_active_scheme(*args,**kw)
         # for convenience, set Colors to the active scheme
         self.Colors = self.color_scheme_table.active_colors
         # Also set colors of debugger
         if hasattr(self,'pdb') and self.pdb is not None:
             self.pdb.set_colors(*args,**kw)
 
     def color_toggle(self):
         """Toggle between the currently active color scheme and NoColor."""
         
         if self.color_scheme_table.active_scheme_name == 'NoColor':
             self.color_scheme_table.set_active_scheme(self.old_scheme)
             self.Colors = self.color_scheme_table.active_colors
         else:
             self.old_scheme = self.color_scheme_table.active_scheme_name
             self.color_scheme_table.set_active_scheme('NoColor')
             self.Colors = self.color_scheme_table.active_colors
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return '\n'.join(stb)
 
     def text(self, etype, value, tb, tb_offset=None, context=5):
         """Return formatted traceback.
 
         Subclasses may override this if they add extra arguments.
         """
         tb_list = self.structured_traceback(etype, value, tb,
                                             tb_offset, context)
         return self.stb2text(tb_list)
 
     def structured_traceback(self, etype, evalue, tb, tb_offset=None,
                              context=5, mode=None):
         """Return a list of traceback frames.
 
         Must be implemented by each class.
         """
         raise NotImplementedError()
 
 
 #---------------------------------------------------------------------------
 class ListTB(TBTools):
     """Print traceback information from a traceback list, with optional color.
         
     Calling: requires 3 arguments:
       (etype, evalue, elist)
     as would be obtained by:
       etype, evalue, tb = sys.exc_info()
       if tb:
         elist = traceback.extract_tb(tb)
       else:
         elist = None
 
     It can thus be used by programs which need to process the traceback before
     printing (such as console replacements based on the code module from the
     standard library).
 
     Because they are meant to be called without a full traceback (only a
     list), instances of this class can't call the interactive pdb debugger."""
 
     def __init__(self,color_scheme = 'NoColor', call_pdb=False, ostream=None):
         TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                          ostream=ostream)
         
     def __call__(self, etype, value, elist):
         self.ostream.flush()
         self.ostream.write(self.text(etype, value, elist))
         self.ostream.write('\n')
 
     def structured_traceback(self, etype, value, elist, tb_offset=None,
                              context=5):
         """Return a color formatted string with the traceback info.
 
         Parameters
         ----------
         etype : exception type
           Type of the exception raised.
 
         value : object
           Data stored in the exception
 
         elist : list
           List of frames, see class docstring for details.
 
         tb_offset : int, optional
           Number of frames in the traceback to skip.  If not given, the
           instance value is used (set in constructor).
           
         context : int, optional
           Number of lines of context information to print.
 
         Returns
         -------
         String with formatted exception.
         """
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         Colors = self.Colors
         out_list = []
         if elist:
 
             if tb_offset and len(elist) > tb_offset:
                 elist = elist[tb_offset:]
             
             out_list.append('Traceback %s(most recent call last)%s:' %
                                 (Colors.normalEm, Colors.Normal) + '\n')
             out_list.extend(self._format_list(elist))
         # The exception info should be a single entry in the list.
         lines = ''.join(self._format_exception_only(etype, value))
         out_list.append(lines)
 
         # Note: this code originally read:
         
         ## for line in lines[:-1]:
         ##     out_list.append(" "+line)
         ## out_list.append(lines[-1])
 
         # This means it was indenting everything but the last line by a little
         # bit.  I've disabled this for now, but if we see ugliness somewhre we
         # can restore it.
         
         return out_list
 
     def _format_list(self, extracted_list):
         """Format a list of traceback entry tuples for printing.
 
         Given a list of tuples as returned by extract_tb() or
         extract_stack(), return a list of strings ready for printing.
         Each string in the resulting list corresponds to the item with the
         same index in the argument list.  Each string ends in a newline;
         the strings may contain internal newlines as well, for those items
         whose source text line is not None.
         
         Lifted almost verbatim from traceback.py
         """
 
         Colors = self.Colors
         list = []
         for filename, lineno, name, line in extracted_list[:-1]:
             item = '  File %s"%s"%s, line %s%d%s, in %s%s%s\n' % \
                     (Colors.filename, filename, Colors.Normal, 
                      Colors.lineno, lineno, Colors.Normal,
                      Colors.name, name, Colors.Normal)
             if line:
                 item = item + '    %s\n' % line.strip()
             list.append(item)
         # Emphasize the last entry
         filename, lineno, name, line = extracted_list[-1]
         item = '%s  File %s"%s"%s, line %s%d%s, in %s%s%s%s\n' % \
                 (Colors.normalEm,
                  Colors.filenameEm, filename, Colors.normalEm,
                  Colors.linenoEm, lineno, Colors.normalEm,
                  Colors.nameEm, name, Colors.normalEm,
                  Colors.Normal)
         if line:
             item = item + '%s    %s%s\n' % (Colors.line, line.strip(),
                                             Colors.Normal)
         list.append(item)
         #from pprint import pformat; print 'LISTTB', pformat(list) # dbg
         return list
         
     def _format_exception_only(self, etype, value):
         """Format the exception part of a traceback.
 
         The arguments are the exception type and value such as given by
         sys.exc_info()[:2]. The return value is a list of strings, each ending
         in a newline.  Normally, the list contains a single string; however,
         for SyntaxError exceptions, it contains several lines that (when
         printed) display detailed information about where the syntax error
         occurred.  The message indicating which exception occurred is the
         always last string in the list.
         
         Also lifted nearly verbatim from traceback.py
         """
 
         have_filedata = False
         Colors = self.Colors
         list = []
         try:
             stype = Colors.excName + etype.__name__ + Colors.Normal
         except AttributeError:
             stype = etype  # String exceptions don't get special coloring
         if value is None:
             list.append( str(stype) + '\n')
         else:
             if etype is SyntaxError:
                 try:
                     msg, (filename, lineno, offset, line) = value
                 except:
                     have_filedata = False
                 else:
                     have_filedata = True
                     #print 'filename is',filename  # dbg
                     if not filename: filename = "<string>"
                     list.append('%s  File %s"%s"%s, line %s%d%s\n' % \
                             (Colors.normalEm,
                              Colors.filenameEm, filename, Colors.normalEm,
                              Colors.linenoEm, lineno, Colors.Normal  ))
                     if line is not None:
                         i = 0
                         while i < len(line) and line[i].isspace():
                             i = i+1
                         list.append('%s    %s%s\n' % (Colors.line,
                                                       line.strip(), 
                                                       Colors.Normal))
                         if offset is not None:
                             s = '    '
                             for c in line[i:offset-1]:
                                 if c.isspace():
                                     s = s + c
                                 else:
                                     s = s + ' '
                             list.append('%s%s^%s\n' % (Colors.caret, s,
                                                        Colors.Normal) )
                         value = msg
             s = self._some_str(value)
             if s:
                 list.append('%s%s:%s %s\n' % (str(stype), Colors.excName,
                                               Colors.Normal, s))
             else:
                 list.append('%s\n' % str(stype))
 
         # sync with user hooks
         if have_filedata:
             ipinst = ipapi.get()
             if ipinst is not None:
                 ipinst.hooks.synchronize_with_editor(filename, lineno, 0)
 
         return list
 
     def get_exception_only(self, etype, value):
         """Only print the exception type and message, without a traceback.
         
         Parameters
         ----------
         etype : exception type
         value : exception value
         """
         return ListTB.structured_traceback(self, etype, value, [])
 
 
     def show_exception_only(self, etype, evalue):
         """Only print the exception type and message, without a traceback.
         
         Parameters
         ----------
         etype : exception type
         value : exception value
         """
         # This method needs to use __call__ from *this* class, not the one from
         # a subclass whose signature or behavior may be different
         ostream = self.ostream
         ostream.flush()
         ostream.write('\n'.join(self.get_exception_only(etype, evalue)))
         ostream.flush()
 
     def _some_str(self, value):
         # Lifted from traceback.py
         try:
             return str(value)
         except:
             return '<unprintable %s object>' % type(value).__name__
 
 #----------------------------------------------------------------------------
 class VerboseTB(TBTools):
     """A port of Ka-Ping Yee's cgitb.py module that outputs color text instead
     of HTML.  Requires inspect and pydoc.  Crazy, man.
 
     Modified version which optionally strips the topmost entries from the
     traceback, to be used with alternate interpreters (because their own code
     would appear in the traceback)."""
 
     def __init__(self,color_scheme = 'Linux', call_pdb=False, ostream=None,
                  tb_offset=0, long_header=False, include_vars=True):
         """Specify traceback offset, headers and color scheme.
 
         Define how many frames to drop from the tracebacks. Calling it with
         tb_offset=1 allows use of this handler in interpreters which will have
         their own code at the top of the traceback (VerboseTB will first
         remove that frame before printing the traceback info)."""
         TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                          ostream=ostream)
         self.tb_offset = tb_offset
         self.long_header = long_header
         self.include_vars = include_vars
 
     def structured_traceback(self, etype, evalue, etb, tb_offset=None,
                              context=5):
         """Return a nice text document describing the traceback."""
 
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
 
         # some locals
         try:
             etype = etype.__name__
         except AttributeError:
             pass
         Colors        = self.Colors   # just a shorthand + quicker name lookup
         ColorsNormal  = Colors.Normal  # used a lot
         col_scheme    = self.color_scheme_table.active_scheme_name
         indent        = ' '*INDENT_SIZE
         em_normal     = '%s\n%s%s' % (Colors.valEm, indent,ColorsNormal)
         undefined     = '%sundefined%s' % (Colors.em, ColorsNormal)
         exc = '%s%s%s' % (Colors.excName,etype,ColorsNormal)
 
         # some internal-use functions
         def text_repr(value):
             """Hopefully pretty robust repr equivalent."""
             # this is pretty horrible but should always return *something*
             try:
                 return pydoc.text.repr(value)
             except KeyboardInterrupt:
                 raise
             except:
                 try:
                     return repr(value)
                 except KeyboardInterrupt:
                     raise
                 except:
                     try:
                         # all still in an except block so we catch
                         # getattr raising
                         name = getattr(value, '__name__', None)
                         if name:
                             # ick, recursion
                             return text_repr(name)
                         klass = getattr(value, '__class__', None)
                         if klass:
                             return '%s instance' % text_repr(klass)
                     except KeyboardInterrupt:
                         raise
                     except:
                         return 'UNRECOVERABLE REPR FAILURE'
         def eqrepr(value, repr=text_repr): return '=%s' % repr(value)
         def nullrepr(value, repr=text_repr): return ''
 
         # meat of the code begins
         try:
             etype = etype.__name__
         except AttributeError:
             pass
 
         if self.long_header:
             # Header with the exception type, python version, and date
             pyver = 'Python ' + string.split(sys.version)[0] + ': ' + sys.executable
             date = time.ctime(time.time())
             
             head = '%s%s%s\n%s%s%s\n%s' % (Colors.topline, '-'*75, ColorsNormal,
                                            exc, ' '*(75-len(str(etype))-len(pyver)),
-                                           pyver, string.rjust(date, 75) )
+                                           pyver, date.rjust(75) )
             head += "\nA problem occured executing Python code.  Here is the sequence of function"\
                     "\ncalls leading up to the error, with the most recent (innermost) call last."
         else:
             # Simplified header
             head = '%s%s%s\n%s%s' % (Colors.topline, '-'*75, ColorsNormal,exc,
-                                     string.rjust('Traceback (most recent call last)',
-                                                  75 - len(str(etype)) ) )
+                                     'Traceback (most recent call last)'.\
+                                                  rjust(75 - len(str(etype)) ) )
         frames = []
         # Flush cache before calling inspect.  This helps alleviate some of the
         # problems with python 2.3's inspect.py.
         linecache.checkcache()
         # Drop topmost frames if requested
         try:
             # Try the default getinnerframes and Alex's: Alex's fixes some
             # problems, but it generates empty tracebacks for console errors
             # (5 blanks lines) where none should be returned.
             #records = inspect.getinnerframes(etb, context)[tb_offset:]
             #print 'python records:', records # dbg
             records = _fixed_getinnerframes(etb, context, tb_offset)
             #print 'alex   records:', records # dbg
         except:
 
             # FIXME: I've been getting many crash reports from python 2.3
             # users, traceable to inspect.py.  If I can find a small test-case
             # to reproduce this, I should either write a better workaround or
             # file a bug report against inspect (if that's the real problem).
             # So far, I haven't been able to find an isolated example to
             # reproduce the problem.
             inspect_error()
             traceback.print_exc(file=self.ostream)
             info('\nUnfortunately, your original traceback can not be constructed.\n')
             return ''
 
         # build some color string templates outside these nested loops
         tpl_link       = '%s%%s%s' % (Colors.filenameEm,ColorsNormal)
         tpl_call       = 'in %s%%s%s%%s%s' % (Colors.vName, Colors.valEm,
                                               ColorsNormal)
         tpl_call_fail  = 'in %s%%s%s(***failed resolving arguments***)%s' % \
                          (Colors.vName, Colors.valEm, ColorsNormal)
         tpl_local_var  = '%s%%s%s' % (Colors.vName, ColorsNormal)
         tpl_global_var = '%sglobal%s %s%%s%s' % (Colors.em, ColorsNormal,
                                                  Colors.vName, ColorsNormal)
         tpl_name_val   = '%%s %s= %%s%s' % (Colors.valEm, ColorsNormal)
         tpl_line       = '%s%%s%s %%s' % (Colors.lineno, ColorsNormal)
         tpl_line_em    = '%s%%s%s %%s%s' % (Colors.linenoEm,Colors.line,
                                             ColorsNormal)
 
         # now, loop over all records printing context and info
         abspath = os.path.abspath
         for frame, file, lnum, func, lines, index in records:
             #print '*** record:',file,lnum,func,lines,index  # dbg
             try:
                 file = file and abspath(file) or '?'
             except OSError:
                 # if file is '<console>' or something not in the filesystem,
                 # the abspath call will throw an OSError.  Just ignore it and
                 # keep the original file string.
                 pass
             link = tpl_link % file
             try:
                 args, varargs, varkw, locals = inspect.getargvalues(frame)
             except:
                 # This can happen due to a bug in python2.3.  We should be
                 # able to remove this try/except when 2.4 becomes a
                 # requirement.  Bug details at http://python.org/sf/1005466
                 inspect_error()
                 traceback.print_exc(file=self.ostream)
                 info("\nIPython's exception reporting continues...\n")
                 
             if func == '?':
                 call = ''
             else:
                 # Decide whether to include variable details or not
                 var_repr = self.include_vars and eqrepr or nullrepr
                 try:
                     call = tpl_call % (func,inspect.formatargvalues(args,
                                                 varargs, varkw,
                                                 locals,formatvalue=var_repr))
                 except KeyError:
                     # Very odd crash from inspect.formatargvalues().  The
                     # scenario under which it appeared was a call to
                     # view(array,scale) in NumTut.view.view(), where scale had
                     # been defined as a scalar (it should be a tuple). Somehow
                     # inspect messes up resolving the argument list of view()
                     # and barfs out. At some point I should dig into this one
                     # and file a bug report about it.
                     inspect_error()
                     traceback.print_exc(file=self.ostream)
                     info("\nIPython's exception reporting continues...\n")
                     call = tpl_call_fail % func
 
             # Initialize a list of names on the current line, which the
             # tokenizer below will populate.
             names = []
 
             def tokeneater(token_type, token, start, end, line):
                 """Stateful tokeneater which builds dotted names.
 
                 The list of names it appends to (from the enclosing scope) can
                 contain repeated composite names.  This is unavoidable, since
                 there is no way to disambguate partial dotted structures until
                 the full list is known.  The caller is responsible for pruning
                 the final list of duplicates before using it."""
                 
                 # build composite names
                 if token == '.':
                     try:
                         names[-1] += '.'
                         # store state so the next token is added for x.y.z names
                         tokeneater.name_cont = True
                         return
                     except IndexError:
                         pass
                 if token_type == tokenize.NAME and token not in keyword.kwlist:
                     if tokeneater.name_cont:
                         # Dotted names
                         names[-1] += token
                         tokeneater.name_cont = False
                     else:
                         # Regular new names.  We append everything, the caller
                         # will be responsible for pruning the list later.  It's
                         # very tricky to try to prune as we go, b/c composite
                         # names can fool us.  The pruning at the end is easy
                         # to do (or the caller can print a list with repeated
                         # names if so desired.
                         names.append(token)
                 elif token_type == tokenize.NEWLINE:
                     raise IndexError
             # we need to store a bit of state in the tokenizer to build
             # dotted names
             tokeneater.name_cont = False
 
             def linereader(file=file, lnum=[lnum], getline=linecache.getline):
                 line = getline(file, lnum[0])
                 lnum[0] += 1
                 return line
 
             # Build the list of names on this line of code where the exception
             # occurred.
             try:
                 # This builds the names list in-place by capturing it from the
                 # enclosing scope.
                 tokenize.tokenize(linereader, tokeneater)
             except IndexError:
                 # signals exit of tokenizer
                 pass
             except tokenize.TokenError,msg:
                 _m = ("An unexpected error occurred while tokenizing input\n"
                       "The following traceback may be corrupted or invalid\n"
                       "The error message is: %s\n" % msg)
                 error(_m)
             
             # prune names list of duplicates, but keep the right order
             unique_names = uniq_stable(names)
 
             # Start loop over vars
             lvals = []
             if self.include_vars:
                 for name_full in unique_names:
                     name_base = name_full.split('.',1)[0]
                     if name_base in frame.f_code.co_varnames:
                         if locals.has_key(name_base):
                             try:
                                 value = repr(eval(name_full,locals))
                             except:
                                 value = undefined
                         else:
                             value = undefined
                         name = tpl_local_var % name_full
                     else:
                         if frame.f_globals.has_key(name_base):
                             try:
                                 value = repr(eval(name_full,frame.f_globals))
                             except:
                                 value = undefined
                         else:
                             value = undefined
                         name = tpl_global_var % name_full
                     lvals.append(tpl_name_val % (name,value))
             if lvals:
                 lvals = '%s%s' % (indent,em_normal.join(lvals))
             else:
                 lvals = ''
 
             level = '%s %s\n' % (link,call)
 
             if index is None:
                 frames.append(level)
             else:
                 frames.append('%s%s' % (level,''.join(
                     _format_traceback_lines(lnum,index,lines,Colors,lvals,
                                             col_scheme))))
 
         # Get (safely) a string form of the exception info
         try:
             etype_str,evalue_str = map(str,(etype,evalue))
         except:
             # User exception is improperly defined.
             etype,evalue = str,sys.exc_info()[:2]
             etype_str,evalue_str = map(str,(etype,evalue))
         # ... and format it
         exception = ['%s%s%s: %s' % (Colors.excName, etype_str,
                                      ColorsNormal, evalue_str)]
         if type(evalue) is types.InstanceType:
             try:
                 names = [w for w in dir(evalue) if isinstance(w, basestring)]
             except:
                 # Every now and then, an object with funny inernals blows up
                 # when dir() is called on it.  We do the best we can to report
                 # the problem and continue
                 _m = '%sException reporting error (object with broken dir())%s:'
                 exception.append(_m % (Colors.excName,ColorsNormal))
                 etype_str,evalue_str = map(str,sys.exc_info()[:2])
                 exception.append('%s%s%s: %s' % (Colors.excName,etype_str,
                                      ColorsNormal, evalue_str))
                 names = []
             for name in names:
                 value = text_repr(getattr(evalue, name))
                 exception.append('\n%s%s = %s' % (indent, name, value))
 
         # vds: >>
         if records:
              filepath, lnum = records[-1][1:3]
              #print "file:", str(file), "linenb", str(lnum) # dbg
              filepath = os.path.abspath(filepath)
              ipinst = ipapi.get()
              if ipinst is not None:
                  ipinst.hooks.synchronize_with_editor(filepath, lnum, 0)
         # vds: <<
                 
         # return all our info assembled as a single string
         # return '%s\n\n%s\n%s' % (head,'\n'.join(frames),''.join(exception[0]) )
         return [head] + frames + [''.join(exception[0])]
     
     def debugger(self,force=False):
         """Call up the pdb debugger if desired, always clean up the tb
         reference.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
           flag and does not actually invoke the debugger if the flag is false.
           The 'force' option forces the debugger to activate even if the flag
           is false.
 
         If the call_pdb flag is set, the pdb interactive debugger is
         invoked. In all cases, the self.tb reference to the current traceback
         is deleted to prevent lingering references which hamper memory
         management.
 
         Note that each call to pdb() does an 'import readline', so if your app
         requires a special setup for the readline completers, you'll have to
         fix that by hand after invoking the exception handler."""
 
         if force or self.call_pdb:
             if self.pdb is None:
                 self.pdb = debugger.Pdb(
                     self.color_scheme_table.active_scheme_name)
             # the system displayhook may have changed, restore the original
             # for pdb
             display_trap = DisplayTrap(hook=sys.__displayhook__)
             with display_trap:
                 self.pdb.reset()
                 # Find the right frame so we don't pop up inside ipython itself
                 if hasattr(self,'tb') and self.tb is not None:
                     etb = self.tb
                 else:
                     etb = self.tb = sys.last_traceback
                 while self.tb is not None and self.tb.tb_next is not None:
                     self.tb = self.tb.tb_next
                 if etb and etb.tb_next:
                     etb = etb.tb_next
                 self.pdb.botframe = etb.tb_frame
                 self.pdb.interaction(self.tb.tb_frame, self.tb)
 
         if hasattr(self,'tb'):
             del self.tb
 
     def handler(self, info=None):
         (etype, evalue, etb) = info or sys.exc_info()
         self.tb = etb
         ostream = self.ostream
         ostream.flush()
         ostream.write(self.text(etype, evalue, etb))
         ostream.write('\n')
         ostream.flush()
 
     # Changed so an instance can just be called as VerboseTB_inst() and print
     # out the right info on its own.
     def __call__(self, etype=None, evalue=None, etb=None):
         """This hook can replace sys.excepthook (for Python 2.1 or higher)."""
         if etb is None:
             self.handler()
         else:
             self.handler((etype, evalue, etb))
         try:
             self.debugger()
         except KeyboardInterrupt:
             print "\nKeyboardInterrupt"
 
 #----------------------------------------------------------------------------
 class FormattedTB(VerboseTB, ListTB):
     """Subclass ListTB but allow calling with a traceback.
 
     It can thus be used as a sys.excepthook for Python > 2.1.
 
     Also adds 'Context' and 'Verbose' modes, not available in ListTB.
 
     Allows a tb_offset to be specified. This is useful for situations where
     one needs to remove a number of topmost frames from the traceback (such as
     occurs with python programs that themselves execute other python code,
     like Python shells).  """
     
     def __init__(self, mode='Plain', color_scheme='Linux', call_pdb=False,
                  ostream=None, 
                  tb_offset=0, long_header=False, include_vars=False):
 
         # NEVER change the order of this list. Put new modes at the end:
         self.valid_modes = ['Plain','Context','Verbose']
         self.verbose_modes = self.valid_modes[1:3]
 
         VerboseTB.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                            ostream=ostream, tb_offset=tb_offset,
                            long_header=long_header, include_vars=include_vars)
 
         # Different types of tracebacks are joined with different separators to
         # form a single string.  They are taken from this dict
         self._join_chars = dict(Plain='', Context='\n', Verbose='\n')
         # set_mode also sets the tb_join_char attribute
         self.set_mode(mode)
         
     def _extract_tb(self,tb):
         if tb:
             return traceback.extract_tb(tb)
         else:
             return None
 
     def structured_traceback(self, etype, value, tb, tb_offset=None, context=5):
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         mode = self.mode
         if mode in self.verbose_modes:
             # Verbose modes need a full traceback
             return VerboseTB.structured_traceback(
                 self, etype, value, tb, tb_offset, context
             )
         else:
             # We must check the source cache because otherwise we can print
             # out-of-date source code.
             linecache.checkcache()
             # Now we can extract and format the exception
             elist = self._extract_tb(tb)
             return ListTB.structured_traceback(
                 self, etype, value, elist, tb_offset, context
             )
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return self.tb_join_char.join(stb)
         
 
     def set_mode(self,mode=None):
         """Switch to the desired mode.
 
         If mode is not specified, cycles through the available modes."""
 
         if not mode:
             new_idx = ( self.valid_modes.index(self.mode) + 1 ) % \
                       len(self.valid_modes)
             self.mode = self.valid_modes[new_idx]
         elif mode not in self.valid_modes:
             raise ValueError, 'Unrecognized mode in FormattedTB: <'+mode+'>\n'\
                   'Valid modes: '+str(self.valid_modes)
         else:
             self.mode = mode
         # include variable details only in 'Verbose' mode
         self.include_vars = (self.mode == self.valid_modes[2])
         # Set the join character for generating text tracebacks
         self.tb_join_char = self._join_chars[mode]
 
     # some convenient shorcuts
     def plain(self):
         self.set_mode(self.valid_modes[0])
 
     def context(self):
         self.set_mode(self.valid_modes[1])
 
     def verbose(self):
         self.set_mode(self.valid_modes[2])
 
 #----------------------------------------------------------------------------
 class AutoFormattedTB(FormattedTB):
     """A traceback printer which can be called on the fly.
 
     It will find out about exceptions by itself.
 
     A brief example:
     
     AutoTB = AutoFormattedTB(mode = 'Verbose',color_scheme='Linux')
     try:
       ...
     except:
       AutoTB()  # or AutoTB(out=logfile) where logfile is an open file object
     """
     
     def __call__(self,etype=None,evalue=None,etb=None,
                  out=None,tb_offset=None):
         """Print out a formatted exception traceback.
 
         Optional arguments:
           - out: an open file-like object to direct output to.
 
           - tb_offset: the number of frames to skip over in the stack, on a
           per-call basis (this overrides temporarily the instance's tb_offset
           given at initialization time.  """
 
         
         if out is None:
             out = self.ostream
         out.flush()
         out.write(self.text(etype, evalue, etb, tb_offset))
         out.write('\n')
         out.flush()
         # FIXME: we should remove the auto pdb behavior from here and leave
         # that to the clients.
         try:
             self.debugger()
         except KeyboardInterrupt:
             print "\nKeyboardInterrupt"
 
     def structured_traceback(self, etype=None, value=None, tb=None,
                              tb_offset=None, context=5):
         if etype is None:
             etype,value,tb = sys.exc_info()
         self.tb = tb
         return FormattedTB.structured_traceback(
             self, etype, value, tb, tb_offset, context)
 
 #---------------------------------------------------------------------------
 
 # A simple class to preserve Nathan's original functionality.
 class ColorTB(FormattedTB):
     """Shorthand to initialize a FormattedTB in Linux colors mode."""
     def __init__(self,color_scheme='Linux',call_pdb=0):
         FormattedTB.__init__(self,color_scheme=color_scheme,
                              call_pdb=call_pdb)
 
 
 class SyntaxTB(ListTB):
     """Extension which holds some state: the last exception value"""
 
     def __init__(self,color_scheme = 'NoColor'):
         ListTB.__init__(self,color_scheme)
         self.last_syntax_error = None
 
     def __call__(self, etype, value, elist):
         self.last_syntax_error = value
         ListTB.__call__(self,etype,value,elist)
 
     def clear_err_state(self):
         """Return the current error state and clear it"""
         e = self.last_syntax_error
         self.last_syntax_error = None
         return e
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return ''.join(stb)
 
 
 #----------------------------------------------------------------------------
 # module testing (minimal)
 if __name__ == "__main__":
     def spam(c, (d, e)):
         x = c + d
         y = c * d
         foo(x, y)
 
     def foo(a, b, bar=1):
         eggs(a, b + bar)
 
     def eggs(f, g, z=globals()):
         h = f + g
         i = f - g
         return h / i
 
     print ''
     print '*** Before ***'
     try:
         print spam(1, (2, 3))
     except:
         traceback.print_exc()
     print ''
     
     handler = ColorTB()
     print '*** ColorTB ***'
     try:
         print spam(1, (2, 3))
     except:
         apply(handler, sys.exc_info() )
     print ''
     
     handler = VerboseTB()
     print '*** VerboseTB ***'
     try:
         print spam(1, (2, 3))
     except:
         apply(handler, sys.exc_info() )
     print ''
     

IPython/external/Itpl.py

@@ -1,276 +1,275 b''
 # -*- coding: utf-8 -*-
 """String interpolation for Python (by Ka-Ping Yee, 14 Feb 2000).
 
 This module lets you quickly and conveniently interpolate values into
 strings (in the flavour of Perl or Tcl, but with less extraneous
 punctuation).  You get a bit more power than in the other languages,
 because this module allows subscripting, slicing, function calls,
 attribute lookup, or arbitrary expressions.  Variables and expressions
 are evaluated in the namespace of the caller.
 
 The itpl() function returns the result of interpolating a string, and
 printpl() prints out an interpolated string.  Here are some examples:
 
     from Itpl import printpl
     printpl("Here is a $string.")
     printpl("Here is a $module.member.")
     printpl("Here is an $object.member.")
     printpl("Here is a $functioncall(with, arguments).")
     printpl("Here is an ${arbitrary + expression}.")
     printpl("Here is an $array[3] member.")
     printpl("Here is a $dictionary['member'].")
 
 The filter() function filters a file object so that output through it
 is interpolated.  This lets you produce the illusion that Python knows
 how to do interpolation:
 
     import Itpl
     sys.stdout = Itpl.filter()
     f = "fancy"
     print "Is this not $f?"
     print "Standard output has been replaced with a $sys.stdout object."
     sys.stdout = Itpl.unfilter()
     print "Okay, back $to $normal."
 
 Under the hood, the Itpl class represents a string that knows how to
 interpolate values.  An instance of the class parses the string once
 upon initialization; the evaluation and substitution can then be done
 each time the instance is evaluated with str(instance).  For example:
 
     from Itpl import Itpl
     s = Itpl("Here is $foo.")
     foo = 5
     print str(s)
     foo = "bar"
     print str(s)
 """
 
 #*****************************************************************************
 #
 # Copyright (c) 2001 Ka-Ping Yee <[e-mail]>
 #
 #
 # Published under the terms of the MIT license, hereby reproduced:
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to
 # deal in the Software without restriction, including without limitation the
 # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 # sell copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
 #
 #*****************************************************************************
 
 __author__  = 'Ka-Ping Yee <[e-mail]>'
 __license__ = 'MIT'
 
 import string
 import sys
 from tokenize import tokenprog
-from types import StringType
 
 class ItplError(ValueError):
     def __init__(self, text, pos):
         self.text = text
         self.pos = pos
     def __str__(self):
         return "unfinished expression in %s at char %d" % (
             repr(self.text), self.pos)
 
 def matchorfail(text, pos):
     match = tokenprog.match(text, pos)
     if match is None:
         raise ItplError(text, pos)
     return match, match.end()
 
 class Itpl:
     """Class representing a string with interpolation abilities.
     
     Upon creation, an instance works out what parts of the format
     string are literal and what parts need to be evaluated.  The
     evaluation and substitution happens in the namespace of the
     caller when str(instance) is called."""
 
     def __init__(self, format,codec='utf_8',encoding_errors='backslashreplace'):
         """The single mandatory argument to this constructor is a format
         string.
 
         The format string is parsed according to the following rules:
 
         1.  A dollar sign and a name, possibly followed by any of: 
               - an open-paren, and anything up to the matching paren 
               - an open-bracket, and anything up to the matching bracket 
               - a period and a name 
             any number of times, is evaluated as a Python expression.
 
         2.  A dollar sign immediately followed by an open-brace, and
             anything up to the matching close-brace, is evaluated as
             a Python expression.
 
         3.  Outside of the expressions described in the above two rules,
             two dollar signs in a row give you one literal dollar sign.
 
         Optional arguments:
 
         - codec('utf_8'): a string containing the name of a valid Python
         codec.
 
         - encoding_errors('backslashreplace'): a string with a valid error handling
         policy.  See the codecs module documentation for details.
 
         These are used to encode the format string if a call to str() fails on
         the expanded result."""
 
         if not isinstance(format,basestring):
             raise TypeError, "needs string initializer"
         self.format = format
         self.codec = codec
         self.encoding_errors = encoding_errors
         
         namechars = "abcdefghijklmnopqrstuvwxyz" \
             "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_";
         chunks = []
         pos = 0
 
         while 1:
-            dollar = string.find(format, "$", pos)
+            dollar = format.find("$", pos)
             if dollar < 0: break
             nextchar = format[dollar+1]
 
             if nextchar == "{":
                 chunks.append((0, format[pos:dollar]))
                 pos, level = dollar+2, 1
                 while level:
                     match, pos = matchorfail(format, pos)
                     tstart, tend = match.regs[3]
                     token = format[tstart:tend]
                     if token == "{": level = level+1
                     elif token == "}": level = level-1
                 chunks.append((1, format[dollar+2:pos-1]))
 
             elif nextchar in namechars:
                 chunks.append((0, format[pos:dollar]))
                 match, pos = matchorfail(format, dollar+1)
                 while pos < len(format):
                     if format[pos] == "." and \
                         pos+1 < len(format) and format[pos+1] in namechars:
                         match, pos = matchorfail(format, pos+1)
                     elif format[pos] in "([":
                         pos, level = pos+1, 1
                         while level:
                             match, pos = matchorfail(format, pos)
                             tstart, tend = match.regs[3]
                             token = format[tstart:tend]
                             if token[0] in "([": level = level+1
                             elif token[0] in ")]": level = level-1
                     else: break
                 chunks.append((1, format[dollar+1:pos]))
 
             else:
                 chunks.append((0, format[pos:dollar+1]))
                 pos = dollar + 1 + (nextchar == "$")
 
         if pos < len(format): chunks.append((0, format[pos:]))
         self.chunks = chunks
 
     def __repr__(self):
         return "<Itpl %s >" % repr(self.format)
 
     def _str(self,glob,loc):
         """Evaluate to a string in the given globals/locals.
 
         The final output is built by calling str(), but if this fails, the
         result is encoded with the instance's codec and error handling policy,
         via a call to out.encode(self.codec,self.encoding_errors)"""
         result = []
         app = result.append
         for live, chunk in self.chunks:
             if live: app(str(eval(chunk,glob,loc)))
             else: app(chunk)
         out = ''.join(result)
         try:
             return str(out)
         except UnicodeError:
             return out.encode(self.codec,self.encoding_errors)
 
     def __str__(self):
         """Evaluate and substitute the appropriate parts of the string."""
 
         # We need to skip enough frames to get to the actual caller outside of
         # Itpl.
         frame = sys._getframe(1)
         while frame.f_globals["__name__"] == __name__: frame = frame.f_back
         loc, glob = frame.f_locals, frame.f_globals
 
         return self._str(glob,loc)
     
 class ItplNS(Itpl):
     """Class representing a string with interpolation abilities.
 
     This inherits from Itpl, but at creation time a namespace is provided
     where the evaluation will occur.  The interpolation becomes a bit more
     efficient, as no traceback needs to be extracte.  It also allows the
     caller to supply a different namespace for the interpolation to occur than
     its own."""
     
     def __init__(self, format,globals,locals=None,
                  codec='utf_8',encoding_errors='backslashreplace'):
         """ItplNS(format,globals[,locals]) -> interpolating string instance.
 
         This constructor, besides a format string, takes a globals dictionary
         and optionally a locals (which defaults to globals if not provided).
 
         For further details, see the Itpl constructor."""
 
         if locals is None:
             locals = globals
         self.globals = globals
         self.locals = locals
         Itpl.__init__(self,format,codec,encoding_errors)
         
     def __str__(self):
         """Evaluate and substitute the appropriate parts of the string."""
         return self._str(self.globals,self.locals)
 
     def __repr__(self):
         return "<ItplNS %s >" % repr(self.format)
 
 # utilities for fast printing
 def itpl(text): return str(Itpl(text))
 def printpl(text): print itpl(text)
 # versions with namespace
 def itplns(text,globals,locals=None): return str(ItplNS(text,globals,locals))
 def printplns(text,globals,locals=None): print itplns(text,globals,locals)
 
 class ItplFile:
     """A file object that filters each write() through an interpolator."""
     def __init__(self, file): self.file = file
     def __repr__(self): return "<interpolated " + repr(self.file) + ">"
     def __getattr__(self, attr): return getattr(self.file, attr)
     def write(self, text): self.file.write(str(Itpl(text)))
 
 def filter(file=sys.stdout):
     """Return an ItplFile that filters writes to the given file object.
     
     'file = filter(file)' replaces 'file' with a filtered object that
     has a write() method.  When called with no argument, this creates
     a filter to sys.stdout."""
     return ItplFile(file)
 
 def unfilter(ifile=None):
     """Return the original file that corresponds to the given ItplFile.
     
     'file = unfilter(file)' undoes the effect of 'file = filter(file)'.
     'sys.stdout = unfilter()' undoes the effect of 'sys.stdout = filter()'."""
     return ifile and ifile.file or sys.stdout.file

IPython/external/configobj.py

@@ -1,2501 +1,2472 b''
+# -*- coding: utf-8 -*-
 # configobj.py
 # A config file reader/writer that supports nested sections in config files.
 # Copyright (C) 2005-2008 Michael Foord, Nicola Larosa
 # E-mail: fuzzyman AT voidspace DOT org DOT uk
 #         nico AT tekNico DOT net
 
 # ConfigObj 4
 # http://www.voidspace.org.uk/python/configobj.html
 
 # Released subject to the BSD License
 # Please see http://www.voidspace.org.uk/python/license.shtml
 
 # Scripts maintained at http://www.voidspace.org.uk/python/index.shtml
 # For information about bugfixes, updates and support, please join the
 # ConfigObj mailing list:
 # http://lists.sourceforge.net/lists/listinfo/configobj-develop
 # Comments, suggestions and bug reports welcome.
 
 from __future__ import generators
 
 import sys
 INTP_VER = sys.version_info[:2]
 if INTP_VER < (2, 2):
     raise RuntimeError("Python v.2.2 or later needed")
 
 import os, re
 compiler = None
 try:
     import compiler
 except ImportError:
     # for IronPython
     pass
 from types import StringTypes
 from warnings import warn
-try:
-    from codecs import BOM_UTF8, BOM_UTF16, BOM_UTF16_BE, BOM_UTF16_LE
-except ImportError:
-    # Python 2.2 does not have these
-    # UTF-8
-    BOM_UTF8 = '\xef\xbb\xbf'
-    # UTF-16, little endian
-    BOM_UTF16_LE = '\xff\xfe'
-    # UTF-16, big endian
-    BOM_UTF16_BE = '\xfe\xff'
-    if sys.byteorder == 'little':
-        # UTF-16, native endianness
-        BOM_UTF16 = BOM_UTF16_LE
-    else:
-        # UTF-16, native endianness
-        BOM_UTF16 = BOM_UTF16_BE
+from codecs import BOM_UTF8, BOM_UTF16, BOM_UTF16_BE, BOM_UTF16_LE
 
 # A dictionary mapping BOM to
 # the encoding to decode with, and what to set the
 # encoding attribute to.
 BOMS = {
     BOM_UTF8: ('utf_8', None),
     BOM_UTF16_BE: ('utf16_be', 'utf_16'),
     BOM_UTF16_LE: ('utf16_le', 'utf_16'),
     BOM_UTF16: ('utf_16', 'utf_16'),
     }
 # All legal variants of the BOM codecs.
 # TODO: the list of aliases is not meant to be exhaustive, is there a
 #   better way ?
 BOM_LIST = {
     'utf_16': 'utf_16',
     'u16': 'utf_16',
     'utf16': 'utf_16',
     'utf-16': 'utf_16',
     'utf16_be': 'utf16_be',
     'utf_16_be': 'utf16_be',
     'utf-16be': 'utf16_be',
     'utf16_le': 'utf16_le',
     'utf_16_le': 'utf16_le',
     'utf-16le': 'utf16_le',
     'utf_8': 'utf_8',
     'u8': 'utf_8',
     'utf': 'utf_8',
     'utf8': 'utf_8',
     'utf-8': 'utf_8',
     }
 
 # Map of encodings to the BOM to write.
 BOM_SET = {
     'utf_8': BOM_UTF8,
     'utf_16': BOM_UTF16,
     'utf16_be': BOM_UTF16_BE,
     'utf16_le': BOM_UTF16_LE,
     None: BOM_UTF8
     }
 
 
 def match_utf8(encoding):
     return BOM_LIST.get(encoding.lower()) == 'utf_8'
 
 
 # Quote strings used for writing values
 squot = "'%s'"
 dquot = '"%s"'
 noquot = "%s"
 wspace_plus = ' \r\t\n\v\t\'"'
 tsquot = '"""%s"""'
 tdquot = "'''%s'''"
 
-try:
-    enumerate
-except NameError:
-    def enumerate(obj):
-        """enumerate for Python 2.2."""
-        i = -1
-        for item in obj:
-            i += 1
-            yield i, item
-
-try:
-    True, False
-except NameError:
-    True, False = 1, 0
-
 
 __version__ = '4.5.2'
 
 __revision__ = '$Id: configobj.py 156 2006-01-31 14:57:08Z fuzzyman $'
 
 __docformat__ = "restructuredtext en"
 
 __all__ = (
     '__version__',
     'DEFAULT_INDENT_TYPE',
     'DEFAULT_INTERPOLATION',
     'ConfigObjError',
     'NestingError',
     'ParseError',
     'DuplicateError',
     'ConfigspecError',
     'ConfigObj',
     'SimpleVal',
     'InterpolationError',
     'InterpolationLoopError',
     'MissingInterpolationOption',
     'RepeatSectionError',
     'ReloadError',
     'UnreprError',
     'UnknownType',
     '__docformat__',
     'flatten_errors',
 )
 
 DEFAULT_INTERPOLATION = 'configparser'
 DEFAULT_INDENT_TYPE = '    '
 MAX_INTERPOL_DEPTH = 10
 
 OPTION_DEFAULTS = {
     'interpolation': True,
     'raise_errors': False,
     'list_values': True,
     'create_empty': False,
     'file_error': False,
     'configspec': None,
     'stringify': True,
     # option may be set to one of ('', ' ', '\t')
     'indent_type': None,
     'encoding': None,
     'default_encoding': None,
     'unrepr': False,
     'write_empty_values': False,
 }
 
 
 
 def getObj(s):
     s = "a=" + s
     if compiler is None:
         raise ImportError('compiler module not available')
     p = compiler.parse(s)
     return p.getChildren()[1].getChildren()[0].getChildren()[1]
 
 
 class UnknownType(Exception):
     pass
 
 
 class Builder(object):
     
     def build(self, o):
         m = getattr(self, 'build_' + o.__class__.__name__, None)
         if m is None:
             raise UnknownType(o.__class__.__name__)
         return m(o)
     
     def build_List(self, o):
         return map(self.build, o.getChildren())
     
     def build_Const(self, o):
         return o.value
     
     def build_Dict(self, o):
         d = {}
         i = iter(map(self.build, o.getChildren()))
         for el in i:
             d[el] = i.next()
         return d
     
     def build_Tuple(self, o):
         return tuple(self.build_List(o))
     
     def build_Name(self, o):
         if o.name == 'None':
             return None
         if o.name == 'True':
             return True
         if o.name == 'False':
             return False
         
         # An undefined Name
         raise UnknownType('Undefined Name')
     
     def build_Add(self, o):
         real, imag = map(self.build_Const, o.getChildren())
         try:
             real = float(real)
         except TypeError:
             raise UnknownType('Add')
         if not isinstance(imag, complex) or imag.real != 0.0:
             raise UnknownType('Add')
         return real+imag
     
     def build_Getattr(self, o):
         parent = self.build(o.expr)
         return getattr(parent, o.attrname)
     
     def build_UnarySub(self, o):
         return -self.build_Const(o.getChildren()[0])
     
     def build_UnaryAdd(self, o):
         return self.build_Const(o.getChildren()[0])
 
 
 _builder = Builder()
 
 
 def unrepr(s):
     if not s:
         return s
     return _builder.build(getObj(s))
 
 
 
 class ConfigObjError(SyntaxError):
     """
     This is the base class for all errors that ConfigObj raises.
     It is a subclass of SyntaxError.
     """
     def __init__(self, message='', line_number=None, line=''):
         self.line = line
         self.line_number = line_number
         self.message = message
         SyntaxError.__init__(self, message)
 
 
 class NestingError(ConfigObjError):
     """
     This error indicates a level of nesting that doesn't match.
     """
 
 
 class ParseError(ConfigObjError):
     """
     This error indicates that a line is badly written.
     It is neither a valid ``key = value`` line,
     nor a valid section marker line.
     """
 
 
 class ReloadError(IOError):
     """
     A 'reload' operation failed.
     This exception is a subclass of ``IOError``.
     """
     def __init__(self):
         IOError.__init__(self, 'reload failed, filename is not set.')
 
 
 class DuplicateError(ConfigObjError):
     """
     The keyword or section specified already exists.
     """
 
 
 class ConfigspecError(ConfigObjError):
     """
     An error occured whilst parsing a configspec.
     """
 
 
 class InterpolationError(ConfigObjError):
     """Base class for the two interpolation errors."""
 
 
 class InterpolationLoopError(InterpolationError):
     """Maximum interpolation depth exceeded in string interpolation."""
 
     def __init__(self, option):
         InterpolationError.__init__(
             self,
             'interpolation loop detected in value "%s".' % option)
 
 
 class RepeatSectionError(ConfigObjError):
     """
     This error indicates additional sections in a section with a
     ``__many__`` (repeated) section.
     """
 
 
 class MissingInterpolationOption(InterpolationError):
     """A value specified for interpolation was missing."""
 
     def __init__(self, option):
         InterpolationError.__init__(
             self,
             'missing option "%s" in interpolation.' % option)
 
 
 class UnreprError(ConfigObjError):
     """An error parsing in unrepr mode."""
 
 
 
 class InterpolationEngine(object):
     """
     A helper class to help perform string interpolation.
 
     This class is an abstract base class; its descendants perform
     the actual work.
     """
 
     # compiled regexp to use in self.interpolate()
     _KEYCRE = re.compile(r"%\(([^)]*)\)s")
 
     def __init__(self, section):
         # the Section instance that "owns" this engine
         self.section = section
 
 
     def interpolate(self, key, value):
         def recursive_interpolate(key, value, section, backtrail):
             """The function that does the actual work.
 
             ``value``: the string we're trying to interpolate.
             ``section``: the section in which that string was found
             ``backtrail``: a dict to keep track of where we've been,
             to detect and prevent infinite recursion loops
 
             This is similar to a depth-first-search algorithm.
             """
             # Have we been here already?
             if backtrail.has_key((key, section.name)):
                 # Yes - infinite loop detected
                 raise InterpolationLoopError(key)
             # Place a marker on our backtrail so we won't come back here again
             backtrail[(key, section.name)] = 1
 
             # Now start the actual work
             match = self._KEYCRE.search(value)
             while match:
                 # The actual parsing of the match is implementation-dependent,
                 # so delegate to our helper function
                 k, v, s = self._parse_match(match)
                 if k is None:
                     # That's the signal that no further interpolation is needed
                     replacement = v
                 else:
                     # Further interpolation may be needed to obtain final value
                     replacement = recursive_interpolate(k, v, s, backtrail)
                 # Replace the matched string with its final value
                 start, end = match.span()
                 value = ''.join((value[:start], replacement, value[end:]))
                 new_search_start = start + len(replacement)
                 # Pick up the next interpolation key, if any, for next time
                 # through the while loop
                 match = self._KEYCRE.search(value, new_search_start)
 
             # Now safe to come back here again; remove marker from backtrail
             del backtrail[(key, section.name)]
 
             return value
 
         # Back in interpolate(), all we have to do is kick off the recursive
         # function with appropriate starting values
         value = recursive_interpolate(key, value, self.section, {})
         return value
 
 
     def _fetch(self, key):
         """Helper function to fetch values from owning section.
 
         Returns a 2-tuple: the value, and the section where it was found.
         """
         # switch off interpolation before we try and fetch anything !
         save_interp = self.section.main.interpolation
         self.section.main.interpolation = False
 
         # Start at section that "owns" this InterpolationEngine
         current_section = self.section
         while True:
             # try the current section first
             val = current_section.get(key)
             if val is not None:
                 break
             # try "DEFAULT" next
             val = current_section.get('DEFAULT', {}).get(key)
             if val is not None:
                 break
             # move up to parent and try again
             # top-level's parent is itself
             if current_section.parent is current_section:
                 # reached top level, time to give up
                 break
             current_section = current_section.parent
 
         # restore interpolation to previous value before returning
         self.section.main.interpolation = save_interp
         if val is None:
             raise MissingInterpolationOption(key)
         return val, current_section
 
 
     def _parse_match(self, match):
         """Implementation-dependent helper function.
 
         Will be passed a match object corresponding to the interpolation
         key we just found (e.g., "%(foo)s" or "$foo"). Should look up that
         key in the appropriate config file section (using the ``_fetch()``
         helper function) and return a 3-tuple: (key, value, section)
 
         ``key`` is the name of the key we're looking for
         ``value`` is the value found for that key
         ``section`` is a reference to the section where it was found
 
         ``key`` and ``section`` should be None if no further
         interpolation should be performed on the resulting value
         (e.g., if we interpolated "$$" and returned "$").
         """
         raise NotImplementedError()
     
 
 
 class ConfigParserInterpolation(InterpolationEngine):
     """Behaves like ConfigParser."""
     _KEYCRE = re.compile(r"%\(([^)]*)\)s")
 
     def _parse_match(self, match):
         key = match.group(1)
         value, section = self._fetch(key)
         return key, value, section
 
 
 
 class TemplateInterpolation(InterpolationEngine):
     """Behaves like string.Template."""
     _delimiter = '$'
     _KEYCRE = re.compile(r"""
         \$(?:
           (?P<escaped>\$)              |   # Two $ signs
           (?P<named>[_a-z][_a-z0-9]*)  |   # $name format
           {(?P<braced>[^}]*)}              # ${name} format
         )
         """, re.IGNORECASE | re.VERBOSE)
 
     def _parse_match(self, match):
         # Valid name (in or out of braces): fetch value from section
         key = match.group('named') or match.group('braced')
         if key is not None:
             value, section = self._fetch(key)
             return key, value, section
         # Escaped delimiter (e.g., $$): return single delimiter
         if match.group('escaped') is not None:
             # Return None for key and section to indicate it's time to stop
             return None, self._delimiter, None
         # Anything else: ignore completely, just return it unchanged
         return None, match.group(), None
 
 
 interpolation_engines = {
     'configparser': ConfigParserInterpolation,
     'template': TemplateInterpolation,
 }
 
 
 
 class Section(dict):
     """
     A dictionary-like object that represents a section in a config file.
     
     It does string interpolation if the 'interpolation' attribute
     of the 'main' object is set to True.
     
     Interpolation is tried first from this object, then from the 'DEFAULT'
     section of this object, next from the parent and its 'DEFAULT' section,
     and so on until the main object is reached.
     
     A Section will behave like an ordered dictionary - following the
     order of the ``scalars`` and ``sections`` attributes.
     You can use this to change the order of members.
     
     Iteration follows the order: scalars, then sections.
     """
 
     def __init__(self, parent, depth, main, indict=None, name=None):
         """
         * parent is the section above
         * depth is the depth level of this section
         * main is the main ConfigObj
         * indict is a dictionary to initialise the section with
         """
         if indict is None:
             indict = {}
         dict.__init__(self)
         # used for nesting level *and* interpolation
         self.parent = parent
         # used for the interpolation attribute
         self.main = main
         # level of nesting depth of this Section
         self.depth = depth
         # purely for information
         self.name = name
         #
         self._initialise()
         # we do this explicitly so that __setitem__ is used properly
         # (rather than just passing to ``dict.__init__``)
         for entry, value in indict.iteritems():
             self[entry] = value
             
             
     def _initialise(self):
         # the sequence of scalar values in this Section
         self.scalars = []
         # the sequence of sections in this Section
         self.sections = []
         # for comments :-)
         self.comments = {}
         self.inline_comments = {}
         # for the configspec
         self.configspec = {}
         self._order = []
         self._configspec_comments = {}
         self._configspec_inline_comments = {}
         self._cs_section_comments = {}
         self._cs_section_inline_comments = {}
         # for defaults
         self.defaults = []
         self.default_values = {}
 
 
     def _interpolate(self, key, value):
         try:
             # do we already have an interpolation engine?
             engine = self._interpolation_engine
         except AttributeError:
             # not yet: first time running _interpolate(), so pick the engine
             name = self.main.interpolation
             if name == True:  # note that "if name:" would be incorrect here
                 # backwards-compatibility: interpolation=True means use default
                 name = DEFAULT_INTERPOLATION
             name = name.lower()  # so that "Template", "template", etc. all work
             class_ = interpolation_engines.get(name, None)
             if class_ is None:
                 # invalid value for self.main.interpolation
                 self.main.interpolation = False
                 return value
             else:
                 # save reference to engine so we don't have to do this again
                 engine = self._interpolation_engine = class_(self)
         # let the engine do the actual work
         return engine.interpolate(key, value)
 
 
     def __getitem__(self, key):
         """Fetch the item and do string interpolation."""
         val = dict.__getitem__(self, key)
         if self.main.interpolation and isinstance(val, StringTypes):
             return self._interpolate(key, val)
         return val
 
 
     def __setitem__(self, key, value, unrepr=False):
         """
         Correctly set a value.
         
         Making dictionary values Section instances.
         (We have to special case 'Section' instances - which are also dicts)
         
         Keys must be strings.
         Values need only be strings (or lists of strings) if
         ``main.stringify`` is set.
         
         `unrepr`` must be set when setting a value to a dictionary, without
         creating a new sub-section.
         """
         if not isinstance(key, StringTypes):
             raise ValueError('The key "%s" is not a string.' % key)
         
         # add the comment
         if not self.comments.has_key(key):
             self.comments[key] = []
             self.inline_comments[key] = ''
         # remove the entry from defaults
         if key in self.defaults:
             self.defaults.remove(key)
         #
         if isinstance(value, Section):
             if not self.has_key(key):
                 self.sections.append(key)
             dict.__setitem__(self, key, value)
         elif isinstance(value, dict) and not unrepr:
             # First create the new depth level,
             # then create the section
             if not self.has_key(key):
                 self.sections.append(key)
             new_depth = self.depth + 1
             dict.__setitem__(
                 self,
                 key,
                 Section(
                     self,
                     new_depth,
                     self.main,
                     indict=value,
                     name=key))
         else:
             if not self.has_key(key):
                 self.scalars.append(key)
             if not self.main.stringify:
                 if isinstance(value, StringTypes):
                     pass
                 elif isinstance(value, (list, tuple)):
                     for entry in value:
                         if not isinstance(entry, StringTypes):
                             raise TypeError('Value is not a string "%s".' % entry)
                 else:
                     raise TypeError('Value is not a string "%s".' % value)
             dict.__setitem__(self, key, value)
 
 
     def __delitem__(self, key):
         """Remove items from the sequence when deleting."""
         dict. __delitem__(self, key)
         if key in self.scalars:
             self.scalars.remove(key)
         else:
             self.sections.remove(key)
         del self.comments[key]
         del self.inline_comments[key]
 
 
     def get(self, key, default=None):
         """A version of ``get`` that doesn't bypass string interpolation."""
         try:
             return self[key]
         except KeyError:
             return default
 
 
     def update(self, indict):
         """
         A version of update that uses our ``__setitem__``.
         """
         for entry in indict:
             self[entry] = indict[entry]
 
 
     def pop(self, key, *args):
         """
         'D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
         If key is not found, d is returned if given, otherwise KeyError is raised'
         """
         val = dict.pop(self, key, *args)
         if key in self.scalars:
             del self.comments[key]
             del self.inline_comments[key]
             self.scalars.remove(key)
         elif key in self.sections:
             del self.comments[key]
             del self.inline_comments[key]
             self.sections.remove(key)
         if self.main.interpolation and isinstance(val, StringTypes):
             return self._interpolate(key, val)
         return val
 
 
     def popitem(self):
         """Pops the first (key,val)"""
         sequence = (self.scalars + self.sections)
         if not sequence:
             raise KeyError(": 'popitem(): dictionary is empty'")
         key = sequence[0]
         val =  self[key]
         del self[key]
         return key, val
 
 
     def clear(self):
         """
         A version of clear that also affects scalars/sections
         Also clears comments and configspec.
         
         Leaves other attributes alone :
             depth/main/parent are not affected
         """
         dict.clear(self)
         self.scalars = []
         self.sections = []
         self.comments = {}
         self.inline_comments = {}
         self.configspec = {}
 
 
     def setdefault(self, key, default=None):
         """A version of setdefault that sets sequence if appropriate."""
         try:
             return self[key]
         except KeyError:
             self[key] = default
             return self[key]
 
 
     def items(self):
         """D.items() -> list of D's (key, value) pairs, as 2-tuples"""
         return zip((self.scalars + self.sections), self.values())
 
 
     def keys(self):
         """D.keys() -> list of D's keys"""
         return (self.scalars + self.sections)
 
 
     def values(self):
         """D.values() -> list of D's values"""
         return [self[key] for key in (self.scalars + self.sections)]
 
 
     def iteritems(self):
         """D.iteritems() -> an iterator over the (key, value) items of D"""
         return iter(self.items())
 
 
     def iterkeys(self):
         """D.iterkeys() -> an iterator over the keys of D"""
         return iter((self.scalars + self.sections))
 
     __iter__ = iterkeys
 
 
     def itervalues(self):
         """D.itervalues() -> an iterator over the values of D"""
         return iter(self.values())
 
 
     def __repr__(self):
         """x.__repr__() <==> repr(x)"""
         return '{%s}' % ', '.join([('%s: %s' % (repr(key), repr(self[key])))
             for key in (self.scalars + self.sections)])
 
     __str__ = __repr__
     __str__.__doc__ = "x.__str__() <==> str(x)"
 
 
     # Extra methods - not in a normal dictionary
 
     def dict(self):
         """
         Return a deepcopy of self as a dictionary.
         
         All members that are ``Section`` instances are recursively turned to
         ordinary dictionaries - by calling their ``dict`` method.
         
         >>> n = a.dict()
         >>> n == a
         1
         >>> n is a
         0
         """
         newdict = {}
         for entry in self:
             this_entry = self[entry]
             if isinstance(this_entry, Section):
                 this_entry = this_entry.dict()
             elif isinstance(this_entry, list):
                 # create a copy rather than a reference
                 this_entry = list(this_entry)
             elif isinstance(this_entry, tuple):
                 # create a copy rather than a reference
                 this_entry = tuple(this_entry)
             newdict[entry] = this_entry
         return newdict
 
 
     def merge(self, indict):
         """
         A recursive update - useful for merging config files.
         
         >>> a = '''[section1]
         ...     option1 = True
         ...     [[subsection]]
         ...     more_options = False
         ...     # end of file'''.splitlines()
         >>> b = '''# File is user.ini
         ...     [section1]
         ...     option1 = False
         ...     # end of file'''.splitlines()
         >>> c1 = ConfigObj(b)
         >>> c2 = ConfigObj(a)
         >>> c2.merge(c1)
         >>> c2
         {'section1': {'option1': 'False', 'subsection': {'more_options': 'False'}}}
         """
-        for key, val in indict.items():
+        for key, val in indict.iteritems():
             if (key in self and isinstance(self[key], dict) and
                                 isinstance(val, dict)):
                 self[key].merge(val)
             else:   
                 self[key] = val
 
 
     def rename(self, oldkey, newkey):
         """
         Change a keyname to another, without changing position in sequence.
         
         Implemented so that transformations can be made on keys,
         as well as on values. (used by encode and decode)
         
         Also renames comments.
         """
         if oldkey in self.scalars:
             the_list = self.scalars
         elif oldkey in self.sections:
             the_list = self.sections
         else:
             raise KeyError('Key "%s" not found.' % oldkey)
         pos = the_list.index(oldkey)
         #
         val = self[oldkey]
         dict.__delitem__(self, oldkey)
         dict.__setitem__(self, newkey, val)
         the_list.remove(oldkey)
         the_list.insert(pos, newkey)
         comm = self.comments[oldkey]
         inline_comment = self.inline_comments[oldkey]
         del self.comments[oldkey]
         del self.inline_comments[oldkey]
         self.comments[newkey] = comm
         self.inline_comments[newkey] = inline_comment
 
 
     def walk(self, function, raise_errors=True,
             call_on_sections=False, **keywargs):
         """
         Walk every member and call a function on the keyword and value.
         
         Return a dictionary of the return values
         
         If the function raises an exception, raise the errror
         unless ``raise_errors=False``, in which case set the return value to
         ``False``.
         
         Any unrecognised keyword arguments you pass to walk, will be pased on
         to the function you pass in.
         
         Note: if ``call_on_sections`` is ``True`` then - on encountering a
         subsection, *first* the function is called for the *whole* subsection,
         and then recurses into it's members. This means your function must be
         able to handle strings, dictionaries and lists. This allows you
         to change the key of subsections as well as for ordinary members. The
         return value when called on the whole subsection has to be discarded.
         
         See  the encode and decode methods for examples, including functions.
         
         .. caution::
         
             You can use ``walk`` to transform the names of members of a section
             but you mustn't add or delete members.
         
         >>> config = '''[XXXXsection]
         ... XXXXkey = XXXXvalue'''.splitlines()
         >>> cfg = ConfigObj(config)
         >>> cfg
         {'XXXXsection': {'XXXXkey': 'XXXXvalue'}}
         >>> def transform(section, key):
         ...     val = section[key]
         ...     newkey = key.replace('XXXX', 'CLIENT1')
         ...     section.rename(key, newkey)
         ...     if isinstance(val, (tuple, list, dict)):
         ...         pass
         ...     else:
         ...         val = val.replace('XXXX', 'CLIENT1')
         ...         section[newkey] = val
         >>> cfg.walk(transform, call_on_sections=True)
         {'CLIENT1section': {'CLIENT1key': None}}
         >>> cfg
         {'CLIENT1section': {'CLIENT1key': 'CLIENT1value'}}
         """
         out = {}
         # scalars first
         for i in range(len(self.scalars)):
             entry = self.scalars[i]
             try:
                 val = function(self, entry, **keywargs)
                 # bound again in case name has changed
                 entry = self.scalars[i]
                 out[entry] = val
             except Exception:
                 if raise_errors:
                     raise
                 else:
                     entry = self.scalars[i]
                     out[entry] = False
         # then sections
         for i in range(len(self.sections)):
             entry = self.sections[i]
             if call_on_sections:
                 try:
                     function(self, entry, **keywargs)
                 except Exception:
                     if raise_errors:
                         raise
                     else:
                         entry = self.sections[i]
                         out[entry] = False
                 # bound again in case name has changed
                 entry = self.sections[i]
             # previous result is discarded
             out[entry] = self[entry].walk(
                 function,
                 raise_errors=raise_errors,
                 call_on_sections=call_on_sections,
                 **keywargs)
         return out
 
 
     def decode(self, encoding):
         """
         Decode all strings and values to unicode, using the specified encoding.
         
         Works with subsections and list values.
         
         Uses the ``walk`` method.
         
         Testing ``encode`` and ``decode``.
         >>> m = ConfigObj(a)
         >>> m.decode('ascii')
         >>> def testuni(val):
         ...     for entry in val:
         ...         if not isinstance(entry, unicode):
         ...             print >> sys.stderr, type(entry)
         ...             raise AssertionError, 'decode failed.'
         ...         if isinstance(val[entry], dict):
         ...             testuni(val[entry])
         ...         elif not isinstance(val[entry], unicode):
         ...             raise AssertionError, 'decode failed.'
         >>> testuni(m)
         >>> m.encode('ascii')
         >>> a == m
         1
         """
         warn('use of ``decode`` is deprecated.', DeprecationWarning)
         def decode(section, key, encoding=encoding, warn=True):
             """ """
             val = section[key]
             if isinstance(val, (list, tuple)):
                 newval = []
                 for entry in val:
                     newval.append(entry.decode(encoding))
             elif isinstance(val, dict):
                 newval = val
             else:
                 newval = val.decode(encoding)
             newkey = key.decode(encoding)
             section.rename(key, newkey)
             section[newkey] = newval
         # using ``call_on_sections`` allows us to modify section names
         self.walk(decode, call_on_sections=True)
 
 
     def encode(self, encoding):
         """
         Encode all strings and values from unicode,
         using the specified encoding.
         
         Works with subsections and list values.
         Uses the ``walk`` method.
         """
         warn('use of ``encode`` is deprecated.', DeprecationWarning)
         def encode(section, key, encoding=encoding):
             """ """
             val = section[key]
             if isinstance(val, (list, tuple)):
                 newval = []
                 for entry in val:
                     newval.append(entry.encode(encoding))
             elif isinstance(val, dict):
                 newval = val
             else:
                 newval = val.encode(encoding)
             newkey = key.encode(encoding)
             section.rename(key, newkey)
             section[newkey] = newval
         self.walk(encode, call_on_sections=True)
 
 
     def istrue(self, key):
         """A deprecated version of ``as_bool``."""
         warn('use of ``istrue`` is deprecated. Use ``as_bool`` method '
                 'instead.', DeprecationWarning)
         return self.as_bool(key)
 
 
     def as_bool(self, key):
         """
         Accepts a key as input. The corresponding value must be a string or
         the objects (``True`` or 1) or (``False`` or 0). We allow 0 and 1 to
         retain compatibility with Python 2.2.
         
         If the string is one of  ``True``, ``On``, ``Yes``, or ``1`` it returns 
         ``True``.
         
         If the string is one of  ``False``, ``Off``, ``No``, or ``0`` it returns 
         ``False``.
         
         ``as_bool`` is not case sensitive.
         
         Any other input will raise a ``ValueError``.
         
         >>> a = ConfigObj()
         >>> a['a'] = 'fish'
         >>> a.as_bool('a')
         Traceback (most recent call last):
         ValueError: Value "fish" is neither True nor False
         >>> a['b'] = 'True'
         >>> a.as_bool('b')
         1
         >>> a['b'] = 'off'
         >>> a.as_bool('b')
         0
         """
         val = self[key]
         if val == True:
             return True
         elif val == False:
             return False
         else:
             try:
                 if not isinstance(val, StringTypes):
                     # TODO: Why do we raise a KeyError here?
                     raise KeyError()
                 else:
                     return self.main._bools[val.lower()]
             except KeyError:
                 raise ValueError('Value "%s" is neither True nor False' % val)
 
 
     def as_int(self, key):
         """
         A convenience method which coerces the specified value to an integer.
         
         If the value is an invalid literal for ``int``, a ``ValueError`` will
         be raised.
         
         >>> a = ConfigObj()
         >>> a['a'] = 'fish'
         >>> a.as_int('a')
         Traceback (most recent call last):
         ValueError: invalid literal for int(): fish
         >>> a['b'] = '1'
         >>> a.as_int('b')
         1
         >>> a['b'] = '3.2'
         >>> a.as_int('b')
         Traceback (most recent call last):
         ValueError: invalid literal for int(): 3.2
         """
         return int(self[key])
 
 
     def as_float(self, key):
         """
         A convenience method which coerces the specified value to a float.
         
         If the value is an invalid literal for ``float``, a ``ValueError`` will
         be raised.
         
         >>> a = ConfigObj()
         >>> a['a'] = 'fish'
         >>> a.as_float('a')
         Traceback (most recent call last):
         ValueError: invalid literal for float(): fish
         >>> a['b'] = '1'
         >>> a.as_float('b')
         1.0
         >>> a['b'] = '3.2'
         >>> a.as_float('b')
         3.2000000000000002
         """
         return float(self[key])
 
 
     def restore_default(self, key):
         """
         Restore (and return) default value for the specified key.
         
         This method will only work for a ConfigObj that was created
         with a configspec and has been validated.
         
         If there is no default value for this key, ``KeyError`` is raised.
         """
         default = self.default_values[key]
         dict.__setitem__(self, key, default)
         if key not in self.defaults:
             self.defaults.append(key)
         return default
 
     
     def restore_defaults(self):
         """
         Recursively restore default values to all members
         that have them.
         
         This method will only work for a ConfigObj that was created
         with a configspec and has been validated.
         
         It doesn't delete or modify entries without default values.
         """
         for key in self.default_values:
             self.restore_default(key)
             
         for section in self.sections:
             self[section].restore_defaults()
 
 
 class ConfigObj(Section):
     """An object to read, create, and write config files."""
 
     _keyword = re.compile(r'''^ # line start
         (\s*)                   # indentation
         (                       # keyword
             (?:".*?")|          # double quotes
             (?:'.*?')|          # single quotes
             (?:[^'"=].*?)       # no quotes
         )
         \s*=\s*                 # divider
         (.*)                    # value (including list values and comments)
         $   # line end
         ''',
         re.VERBOSE)
 
     _sectionmarker = re.compile(r'''^
         (\s*)                     # 1: indentation
         ((?:\[\s*)+)              # 2: section marker open
         (                         # 3: section name open
             (?:"\s*\S.*?\s*")|    # at least one non-space with double quotes
             (?:'\s*\S.*?\s*')|    # at least one non-space with single quotes
             (?:[^'"\s].*?)        # at least one non-space unquoted
         )                         # section name close
         ((?:\s*\])+)              # 4: section marker close
         \s*(\#.*)?                # 5: optional comment
         $''',
         re.VERBOSE)
 
     # this regexp pulls list values out as a single string
     # or single values and comments
     # FIXME: this regex adds a '' to the end of comma terminated lists
     #   workaround in ``_handle_value``
     _valueexp = re.compile(r'''^
         (?:
             (?:
                 (
                     (?:
                         (?:
                             (?:".*?")|              # double quotes
                             (?:'.*?')|              # single quotes
                             (?:[^'",\#][^,\#]*?)    # unquoted
                         )
                         \s*,\s*                     # comma
                     )*      # match all list items ending in a comma (if any)
                 )
                 (
                     (?:".*?")|                      # double quotes
                     (?:'.*?')|                      # single quotes
                     (?:[^'",\#\s][^,]*?)|           # unquoted
                     (?:(?<!,))                      # Empty value
                 )?          # last item in a list - or string value
             )|
             (,)             # alternatively a single comma - empty list
         )
         \s*(\#.*)?          # optional comment
         $''',
         re.VERBOSE)
 
     # use findall to get the members of a list value
     _listvalueexp = re.compile(r'''
         (
             (?:".*?")|          # double quotes
             (?:'.*?')|          # single quotes
             (?:[^'",\#].*?)       # unquoted
         )
         \s*,\s*                 # comma
         ''',
         re.VERBOSE)
 
     # this regexp is used for the value
     # when lists are switched off
     _nolistvalue = re.compile(r'''^
         (
             (?:".*?")|          # double quotes
             (?:'.*?')|          # single quotes
             (?:[^'"\#].*?)|     # unquoted
             (?:)                # Empty value
         )
         \s*(\#.*)?              # optional comment
         $''',
         re.VERBOSE)
 
     # regexes for finding triple quoted values on one line
     _single_line_single = re.compile(r"^'''(.*?)'''\s*(#.*)?$")
     _single_line_double = re.compile(r'^"""(.*?)"""\s*(#.*)?$')
     _multi_line_single = re.compile(r"^(.*?)'''\s*(#.*)?$")
     _multi_line_double = re.compile(r'^(.*?)"""\s*(#.*)?$')
 
     _triple_quote = {
         "'''": (_single_line_single, _multi_line_single),
         '"""': (_single_line_double, _multi_line_double),
     }
 
     # Used by the ``istrue`` Section method
     _bools = {
         'yes': True, 'no': False,
         'on': True, 'off': False,
         '1': True, '0': False,
         'true': True, 'false': False,
         }
 
 
     def __init__(self, infile=None, options=None, **kwargs):
         """
         Parse a config file or create a config file object.
         
         ``ConfigObj(infile=None, options=None, **kwargs)``
         """
         # init the superclass
         Section.__init__(self, self, 0, self)
         
         if infile is None:
             infile = []
         if options is None:
             options = {}
         else:
             options = dict(options)
             
         # keyword arguments take precedence over an options dictionary
         options.update(kwargs)
         
         defaults = OPTION_DEFAULTS.copy()
         # TODO: check the values too.
         for entry in options:
             if entry not in defaults:
                 raise TypeError('Unrecognised option "%s".' % entry)
         
         # Add any explicit options to the defaults
         defaults.update(options)
         self._initialise(defaults)
         configspec = defaults['configspec']
         self._original_configspec = configspec
         self._load(infile, configspec)
         
         
     def _load(self, infile, configspec):
         if isinstance(infile, StringTypes):
             self.filename = infile
             if os.path.isfile(infile):
                 h = open(infile, 'rb')
                 infile = h.read() or []
                 h.close()
             elif self.file_error:
                 # raise an error if the file doesn't exist
                 raise IOError('Config file not found: "%s".' % self.filename)
             else:
                 # file doesn't already exist
                 if self.create_empty:
                     # this is a good test that the filename specified
                     # isn't impossible - like on a non-existent device
                     h = open(infile, 'w')
                     h.write('')
                     h.close()
                 infile = []
                 
         elif isinstance(infile, (list, tuple)):
             infile = list(infile)
             
         elif isinstance(infile, dict):
             # initialise self
             # the Section class handles creating subsections
             if isinstance(infile, ConfigObj):
                 # get a copy of our ConfigObj
                 infile = infile.dict()
                 
             for entry in infile:
                 self[entry] = infile[entry]
             del self._errors
             
             if configspec is not None:
                 self._handle_configspec(configspec)
             else:
                 self.configspec = None
             return
         
         elif hasattr(infile, 'read'):
             # This supports file like objects
             infile = infile.read() or []
             # needs splitting into lines - but needs doing *after* decoding
             # in case it's not an 8 bit encoding
         else:
             raise TypeError('infile must be a filename, file like object, or list of lines.')
         
         if infile:
             # don't do it for the empty ConfigObj
             infile = self._handle_bom(infile)
             # infile is now *always* a list
             #
             # Set the newlines attribute (first line ending it finds)
             # and strip trailing '\n' or '\r' from lines
             for line in infile:
                 if (not line) or (line[-1] not in ('\r', '\n', '\r\n')):
                     continue
                 for end in ('\r\n', '\n', '\r'):
                     if line.endswith(end):
                         self.newlines = end
                         break
                 break
 
             infile = [line.rstrip('\r\n') for line in infile]
             
         self._parse(infile)
         # if we had any errors, now is the time to raise them
         if self._errors:
             info = "at line %s." % self._errors[0].line_number
             if len(self._errors) > 1:
                 msg = "Parsing failed with several errors.\nFirst error %s" % info
                 error = ConfigObjError(msg)
             else:
                 error = self._errors[0]
             # set the errors attribute; it's a list of tuples:
             # (error_type, message, line_number)
             error.errors = self._errors
             # set the config attribute
             error.config = self
             raise error
         # delete private attributes
         del self._errors
         
         if configspec is None:
             self.configspec = None
         else:
             self._handle_configspec(configspec)
     
     
     def _initialise(self, options=None):
         if options is None:
             options = OPTION_DEFAULTS
             
         # initialise a few variables
         self.filename = None
         self._errors = []
         self.raise_errors = options['raise_errors']
         self.interpolation = options['interpolation']
         self.list_values = options['list_values']
         self.create_empty = options['create_empty']
         self.file_error = options['file_error']
         self.stringify = options['stringify']
         self.indent_type = options['indent_type']
         self.encoding = options['encoding']
         self.default_encoding = options['default_encoding']
         self.BOM = False
         self.newlines = None
         self.write_empty_values = options['write_empty_values']
         self.unrepr = options['unrepr']
         
         self.initial_comment = []
         self.final_comment = []
         self.configspec = {}
         
         # Clear section attributes as well
         Section._initialise(self)
         
         
     def __repr__(self):
         return ('ConfigObj({%s})' % 
                 ', '.join([('%s: %s' % (repr(key), repr(self[key]))) 
                 for key in (self.scalars + self.sections)]))
     
     
     def _handle_bom(self, infile):
         """
         Handle any BOM, and decode if necessary.
         
         If an encoding is specified, that *must* be used - but the BOM should
         still be removed (and the BOM attribute set).
         
         (If the encoding is wrongly specified, then a BOM for an alternative
         encoding won't be discovered or removed.)
         
         If an encoding is not specified, UTF8 or UTF16 BOM will be detected and
         removed. The BOM attribute will be set. UTF16 will be decoded to
         unicode.
         
         NOTE: This method must not be called with an empty ``infile``.
         
         Specifying the *wrong* encoding is likely to cause a
         ``UnicodeDecodeError``.
         
         ``infile`` must always be returned as a list of lines, but may be
         passed in as a single string.
         """
         if ((self.encoding is not None) and
             (self.encoding.lower() not in BOM_LIST)):
             # No need to check for a BOM
             # the encoding specified doesn't have one
             # just decode
             return self._decode(infile, self.encoding)
         
         if isinstance(infile, (list, tuple)):
             line = infile[0]
         else:
             line = infile
         if self.encoding is not None:
             # encoding explicitly supplied
             # And it could have an associated BOM
             # TODO: if encoding is just UTF16 - we ought to check for both
             # TODO: big endian and little endian versions.
             enc = BOM_LIST[self.encoding.lower()]
             if enc == 'utf_16':
                 # For UTF16 we try big endian and little endian
-                for BOM, (encoding, final_encoding) in BOMS.items():
+                for BOM, (encoding, final_encoding) in BOMS.iteritems():
                     if not final_encoding:
                         # skip UTF8
                         continue
                     if infile.startswith(BOM):
                         ### BOM discovered
                         ##self.BOM = True
                         # Don't need to remove BOM
                         return self._decode(infile, encoding)
                     
                 # If we get this far, will *probably* raise a DecodeError
                 # As it doesn't appear to start with a BOM
                 return self._decode(infile, self.encoding)
             
             # Must be UTF8
             BOM = BOM_SET[enc]
             if not line.startswith(BOM):
                 return self._decode(infile, self.encoding)
             
             newline = line[len(BOM):]
             
             # BOM removed
             if isinstance(infile, (list, tuple)):
                 infile[0] = newline
             else:
                 infile = newline
             self.BOM = True
             return self._decode(infile, self.encoding)
         
         # No encoding specified - so we need to check for UTF8/UTF16
-        for BOM, (encoding, final_encoding) in BOMS.items():
+        for BOM, (encoding, final_encoding) in BOMS.iteritems():
             if not line.startswith(BOM):
                 continue
             else:
                 # BOM discovered
                 self.encoding = final_encoding
                 if not final_encoding:
                     self.BOM = True
                     # UTF8
                     # remove BOM
                     newline = line[len(BOM):]
                     if isinstance(infile, (list, tuple)):
                         infile[0] = newline
                     else:
                         infile = newline
                     # UTF8 - don't decode
                     if isinstance(infile, StringTypes):
                         return infile.splitlines(True)
                     else:
                         return infile
                 # UTF16 - have to decode
                 return self._decode(infile, encoding)
             
         # No BOM discovered and no encoding specified, just return
         if isinstance(infile, StringTypes):
             # infile read from a file will be a single string
             return infile.splitlines(True)
         return infile
 
 
     def _a_to_u(self, aString):
         """Decode ASCII strings to unicode if a self.encoding is specified."""
         if self.encoding:
             return aString.decode('ascii')
         else:
             return aString
 
 
     def _decode(self, infile, encoding):
         """
         Decode infile to unicode. Using the specified encoding.
         
         if is a string, it also needs converting to a list.
         """
         if isinstance(infile, StringTypes):
             # can't be unicode
             # NOTE: Could raise a ``UnicodeDecodeError``
             return infile.decode(encoding).splitlines(True)
         for i, line in enumerate(infile):
             if not isinstance(line, unicode):
                 # NOTE: The isinstance test here handles mixed lists of unicode/string
                 # NOTE: But the decode will break on any non-string values
                 # NOTE: Or could raise a ``UnicodeDecodeError``
                 infile[i] = line.decode(encoding)
         return infile
 
 
     def _decode_element(self, line):
         """Decode element to unicode if necessary."""
         if not self.encoding:
             return line
         if isinstance(line, str) and self.default_encoding:
             return line.decode(self.default_encoding)
         return line
 
 
     def _str(self, value):
         """
         Used by ``stringify`` within validate, to turn non-string values
         into strings.
         """
         if not isinstance(value, StringTypes):
             return str(value)
         else:
             return value
 
 
     def _parse(self, infile):
         """Actually parse the config file."""
         temp_list_values = self.list_values
         if self.unrepr:
             self.list_values = False
             
         comment_list = []
         done_start = False
         this_section = self
         maxline = len(infile) - 1
         cur_index = -1
         reset_comment = False
         
         while cur_index < maxline:
             if reset_comment:
                 comment_list = []
             cur_index += 1
             line = infile[cur_index]
             sline = line.strip()
             # do we have anything on the line ?
             if not sline or sline.startswith('#'):
                 reset_comment = False
                 comment_list.append(line)
                 continue
             
             if not done_start:
                 # preserve initial comment
                 self.initial_comment = comment_list
                 comment_list = []
                 done_start = True
                 
             reset_comment = True
             # first we check if it's a section marker
             mat = self._sectionmarker.match(line)
             if mat is not None:
                 # is a section line
                 (indent, sect_open, sect_name, sect_close, comment) = mat.groups()
                 if indent and (self.indent_type is None):
                     self.indent_type = indent
                 cur_depth = sect_open.count('[')
                 if cur_depth != sect_close.count(']'):
                     self._handle_error("Cannot compute the section depth at line %s.",
                                        NestingError, infile, cur_index)
                     continue
                 
                 if cur_depth < this_section.depth:
                     # the new section is dropping back to a previous level
                     try:
                         parent = self._match_depth(this_section,
                                                    cur_depth).parent
                     except SyntaxError:
                         self._handle_error("Cannot compute nesting level at line %s.",
                                            NestingError, infile, cur_index)
                         continue
                 elif cur_depth == this_section.depth:
                     # the new section is a sibling of the current section
                     parent = this_section.parent
                 elif cur_depth == this_section.depth + 1:
                     # the new section is a child the current section
                     parent = this_section
                 else:
                     self._handle_error("Section too nested at line %s.",
                                        NestingError, infile, cur_index)
                     
                 sect_name = self._unquote(sect_name)
                 if parent.has_key(sect_name):
                     self._handle_error('Duplicate section name at line %s.',
                                        DuplicateError, infile, cur_index)
                     continue
                 
                 # create the new section
                 this_section = Section(
                     parent,
                     cur_depth,
                     self,
                     name=sect_name)
                 parent[sect_name] = this_section
                 parent.inline_comments[sect_name] = comment
                 parent.comments[sect_name] = comment_list
                 continue
             #
             # it's not a section marker,
             # so it should be a valid ``key = value`` line
             mat = self._keyword.match(line)
             if mat is None:
                 # it neither matched as a keyword
                 # or a section marker
                 self._handle_error(
                     'Invalid line at line "%s".',
                     ParseError, infile, cur_index)
             else:
                 # is a keyword value
                 # value will include any inline comment
                 (indent, key, value) = mat.groups()
                 if indent and (self.indent_type is None):
                     self.indent_type = indent
                 # check for a multiline value
                 if value[:3] in ['"""', "'''"]:
                     try:
                         (value, comment, cur_index) = self._multiline(
                             value, infile, cur_index, maxline)
                     except SyntaxError:
                         self._handle_error(
                             'Parse error in value at line %s.',
                             ParseError, infile, cur_index)
                         continue
                     else:
                         if self.unrepr:
                             comment = ''
                             try:
                                 value = unrepr(value)
                             except Exception, e:
                                 if type(e) == UnknownType:
                                     msg = 'Unknown name or type in value at line %s.'
                                 else:
                                     msg = 'Parse error in value at line %s.'
                                 self._handle_error(msg, UnreprError, infile,
                                     cur_index)
                                 continue
                 else:
                     if self.unrepr:
                         comment = ''
                         try:
                             value = unrepr(value)
                         except Exception, e:
                             if isinstance(e, UnknownType):
                                 msg = 'Unknown name or type in value at line %s.'
                             else:
                                 msg = 'Parse error in value at line %s.'
                             self._handle_error(msg, UnreprError, infile,
                                 cur_index)
                             continue
                     else:
                         # extract comment and lists
                         try:
                             (value, comment) = self._handle_value(value)
                         except SyntaxError:
                             self._handle_error(
                                 'Parse error in value at line %s.',
                                 ParseError, infile, cur_index)
                             continue
                 #
                 key = self._unquote(key)
                 if this_section.has_key(key):
                     self._handle_error(
                         'Duplicate keyword name at line %s.',
                         DuplicateError, infile, cur_index)
                     continue
                 # add the key.
                 # we set unrepr because if we have got this far we will never
                 # be creating a new section
                 this_section.__setitem__(key, value, unrepr=True)
                 this_section.inline_comments[key] = comment
                 this_section.comments[key] = comment_list
                 continue
         #
         if self.indent_type is None:
             # no indentation used, set the type accordingly
             self.indent_type = ''
 
         # preserve the final comment
         if not self and not self.initial_comment:
             self.initial_comment = comment_list
         elif not reset_comment:
             self.final_comment = comment_list
         self.list_values = temp_list_values
 
 
     def _match_depth(self, sect, depth):
         """
         Given a section and a depth level, walk back through the sections
         parents to see if the depth level matches a previous section.
         
         Return a reference to the right section,
         or raise a SyntaxError.
         """
         while depth < sect.depth:
             if sect is sect.parent:
                 # we've reached the top level already
                 raise SyntaxError()
             sect = sect.parent
         if sect.depth == depth:
             return sect
         # shouldn't get here
         raise SyntaxError()
 
 
     def _handle_error(self, text, ErrorClass, infile, cur_index):
         """
         Handle an error according to the error settings.
         
         Either raise the error or store it.
         The error will have occured at ``cur_index``
         """
         line = infile[cur_index]
         cur_index += 1
         message = text % cur_index
         error = ErrorClass(message, cur_index, line)
         if self.raise_errors:
             # raise the error - parsing stops here
             raise error
         # store the error
         # reraise when parsing has finished
         self._errors.append(error)
 
 
     def _unquote(self, value):
         """Return an unquoted version of a value"""
         if (value[0] == value[-1]) and (value[0] in ('"', "'")):
             value = value[1:-1]
         return value
 
 
     def _quote(self, value, multiline=True):
         """
         Return a safely quoted version of a value.
         
         Raise a ConfigObjError if the value cannot be safely quoted.
         If multiline is ``True`` (default) then use triple quotes
         if necessary.
         
         Don't quote values that don't need it.
         Recursively quote members of a list and return a comma joined list.
         Multiline is ``False`` for lists.
         Obey list syntax for empty and single member lists.
         
         If ``list_values=False`` then the value is only quoted if it contains
         a ``\n`` (is multiline) or '#'.
         
         If ``write_empty_values`` is set, and the value is an empty string, it
         won't be quoted.
         """
         if multiline and self.write_empty_values and value == '':
             # Only if multiline is set, so that it is used for values not
             # keys, and not values that are part of a list
             return ''
         
         if multiline and isinstance(value, (list, tuple)):
             if not value:
                 return ','
             elif len(value) == 1:
                 return self._quote(value[0], multiline=False) + ','
             return ', '.join([self._quote(val, multiline=False)
                 for val in value])
         if not isinstance(value, StringTypes):
             if self.stringify:
                 value = str(value)
             else:
                 raise TypeError('Value "%s" is not a string.' % value)
 
         if not value:
             return '""'
         
         no_lists_no_quotes = not self.list_values and '\n' not in value and '#' not in value
         need_triple = multiline and ((("'" in value) and ('"' in value)) or ('\n' in value ))
         hash_triple_quote = multiline and not need_triple and ("'" in value) and ('"' in value) and ('#' in value)
         check_for_single = (no_lists_no_quotes or not need_triple) and not hash_triple_quote
         
         if check_for_single:
             if not self.list_values:
                 # we don't quote if ``list_values=False``
                 quot = noquot
             # for normal values either single or double quotes will do
             elif '\n' in value:
                 # will only happen if multiline is off - e.g. '\n' in key
                 raise ConfigObjError('Value "%s" cannot be safely quoted.' % value)
             elif ((value[0] not in wspace_plus) and
                     (value[-1] not in wspace_plus) and
                     (',' not in value)):
                 quot = noquot
             else:
                 quot = self._get_single_quote(value)
         else:
             # if value has '\n' or "'" *and* '"', it will need triple quotes
             quot = self._get_triple_quote(value)
         
         if quot == noquot and '#' in value and self.list_values:
             quot = self._get_single_quote(value)
                 
         return quot % value
     
     
     def _get_single_quote(self, value):
         if ("'" in value) and ('"' in value):
             raise ConfigObjError('Value "%s" cannot be safely quoted.' % value)
         elif '"' in value:
             quot = squot
         else:
             quot = dquot
         return quot
     
     
     def _get_triple_quote(self, value):
         if (value.find('"""') != -1) and (value.find("'''") != -1):
             raise ConfigObjError('Value "%s" cannot be safely quoted.' % value)
         if value.find('"""') == -1:
             quot = tdquot
         else:
             quot = tsquot 
         return quot
 
 
     def _handle_value(self, value):
         """
         Given a value string, unquote, remove comment,
         handle lists. (including empty and single member lists)
         """
         # do we look for lists in values ?
         if not self.list_values:
             mat = self._nolistvalue.match(value)
             if mat is None:
                 raise SyntaxError()
             # NOTE: we don't unquote here
             return mat.groups()
         #
         mat = self._valueexp.match(value)
         if mat is None:
             # the value is badly constructed, probably badly quoted,
             # or an invalid list
             raise SyntaxError()
         (list_values, single, empty_list, comment) = mat.groups()
         if (list_values == '') and (single is None):
             # change this if you want to accept empty values
             raise SyntaxError()
         # NOTE: note there is no error handling from here if the regex
         # is wrong: then incorrect values will slip through
         if empty_list is not None:
             # the single comma - meaning an empty list
             return ([], comment)
         if single is not None:
             # handle empty values
             if list_values and not single:
                 # FIXME: the '' is a workaround because our regex now matches
                 #   '' at the end of a list if it has a trailing comma
                 single = None
             else:
                 single = single or '""'
                 single = self._unquote(single)
         if list_values == '':
             # not a list value
             return (single, comment)
         the_list = self._listvalueexp.findall(list_values)
         the_list = [self._unquote(val) for val in the_list]
         if single is not None:
             the_list += [single]
         return (the_list, comment)
 
 
     def _multiline(self, value, infile, cur_index, maxline):
         """Extract the value, where we are in a multiline situation."""
         quot = value[:3]
         newvalue = value[3:]
         single_line = self._triple_quote[quot][0]
         multi_line = self._triple_quote[quot][1]
         mat = single_line.match(value)
         if mat is not None:
             retval = list(mat.groups())
             retval.append(cur_index)
             return retval
         elif newvalue.find(quot) != -1:
             # somehow the triple quote is missing
             raise SyntaxError()
         #
         while cur_index < maxline:
             cur_index += 1
             newvalue += '\n'
             line = infile[cur_index]
             if line.find(quot) == -1:
                 newvalue += line
             else:
                 # end of multiline, process it
                 break
         else:
             # we've got to the end of the config, oops...
             raise SyntaxError()
         mat = multi_line.match(line)
         if mat is None:
             # a badly formed line
             raise SyntaxError()
         (value, comment) = mat.groups()
         return (newvalue + value, comment, cur_index)
 
 
     def _handle_configspec(self, configspec):
         """Parse the configspec."""
         # FIXME: Should we check that the configspec was created with the 
         #        correct settings ? (i.e. ``list_values=False``)
         if not isinstance(configspec, ConfigObj):
             try:
                 configspec = ConfigObj(configspec,
                                        raise_errors=True,
                                        file_error=True,
                                        list_values=False)
             except ConfigObjError, e:
                 # FIXME: Should these errors have a reference
                 #        to the already parsed ConfigObj ?
                 raise ConfigspecError('Parsing configspec failed: %s' % e)
             except IOError, e:
                 raise IOError('Reading configspec failed: %s' % e)
         
         self._set_configspec_value(configspec, self)
 
 
     def _set_configspec_value(self, configspec, section):
         """Used to recursively set configspec values."""
         if '__many__' in configspec.sections:
             section.configspec['__many__'] = configspec['__many__']
             if len(configspec.sections) > 1:
                 # FIXME: can we supply any useful information here ?
                 raise RepeatSectionError()
             
         if hasattr(configspec, 'initial_comment'):
             section._configspec_initial_comment = configspec.initial_comment
             section._configspec_final_comment = configspec.final_comment
             section._configspec_encoding = configspec.encoding
             section._configspec_BOM = configspec.BOM
             section._configspec_newlines = configspec.newlines
             section._configspec_indent_type = configspec.indent_type
             
         for entry in configspec.scalars:
             section._configspec_comments[entry] = configspec.comments[entry]
             section._configspec_inline_comments[entry] = configspec.inline_comments[entry]
             section.configspec[entry] = configspec[entry]
             section._order.append(entry)
             
         for entry in configspec.sections:
             if entry == '__many__':
                 continue
             
             section._cs_section_comments[entry] = configspec.comments[entry]
             section._cs_section_inline_comments[entry] = configspec.inline_comments[entry]
             if not section.has_key(entry):
                 section[entry] = {}
             self._set_configspec_value(configspec[entry], section[entry])
 
 
     def _handle_repeat(self, section, configspec):
         """Dynamically assign configspec for repeated section."""
         try:
             section_keys = configspec.sections
             scalar_keys = configspec.scalars
         except AttributeError:
             section_keys = [entry for entry in configspec 
                                 if isinstance(configspec[entry], dict)]
             scalar_keys = [entry for entry in configspec 
                                 if not isinstance(configspec[entry], dict)]
             
         if '__many__' in section_keys and len(section_keys) > 1:
             # FIXME: can we supply any useful information here ?
             raise RepeatSectionError()
         
         scalars = {}
         sections = {}
         for entry in scalar_keys:
             val = configspec[entry]
             scalars[entry] = val
         for entry in section_keys:
             val = configspec[entry]
             if entry == '__many__':
                 scalars[entry] = val
                 continue
             sections[entry] = val
             
         section.configspec = scalars
         for entry in sections:
             if not section.has_key(entry):
                 section[entry] = {}
             self._handle_repeat(section[entry], sections[entry])
 
 
     def _write_line(self, indent_string, entry, this_entry, comment):
         """Write an individual line, for the write method"""
         # NOTE: the calls to self._quote here handles non-StringType values.
         if not self.unrepr:
             val = self._decode_element(self._quote(this_entry))
         else:
             val = repr(this_entry)
         return '%s%s%s%s%s' % (indent_string,
                                self._decode_element(self._quote(entry, multiline=False)),
                                self._a_to_u(' = '),
                                val,
                                self._decode_element(comment))
 
 
     def _write_marker(self, indent_string, depth, entry, comment):
         """Write a section marker line"""
         return '%s%s%s%s%s' % (indent_string,
                                self._a_to_u('[' * depth),
                                self._quote(self._decode_element(entry), multiline=False),
                                self._a_to_u(']' * depth),
                                self._decode_element(comment))
 
 
     def _handle_comment(self, comment):
         """Deal with a comment."""
         if not comment:
             return ''
         start = self.indent_type
         if not comment.startswith('#'):
             start += self._a_to_u(' # ')
         return (start + comment)
 
 
     # Public methods
 
     def write(self, outfile=None, section=None):
         """
         Write the current ConfigObj as a file
         
         tekNico: FIXME: use StringIO instead of real files
         
         >>> filename = a.filename
         >>> a.filename = 'test.ini'
         >>> a.write()
         >>> a.filename = filename
         >>> a == ConfigObj('test.ini', raise_errors=True)
         1
         """
         if self.indent_type is None:
             # this can be true if initialised from a dictionary
             self.indent_type = DEFAULT_INDENT_TYPE
             
         out = []
         cs = self._a_to_u('#')
         csp = self._a_to_u('# ')
         if section is None:
             int_val = self.interpolation
             self.interpolation = False
             section = self
             for line in self.initial_comment:
                 line = self._decode_element(line)
                 stripped_line = line.strip()
                 if stripped_line and not stripped_line.startswith(cs):
                     line = csp + line
                 out.append(line)
                 
         indent_string = self.indent_type * section.depth
         for entry in (section.scalars + section.sections):
             if entry in section.defaults:
                 # don't write out default values
                 continue
             for comment_line in section.comments[entry]:
                 comment_line = self._decode_element(comment_line.lstrip())
                 if comment_line and not comment_line.startswith(cs):
                     comment_line = csp + comment_line
                 out.append(indent_string + comment_line)
             this_entry = section[entry]
             comment = self._handle_comment(section.inline_comments[entry])
             
             if isinstance(this_entry, dict):
                 # a section
                 out.append(self._write_marker(
                     indent_string,
                     this_entry.depth,
                     entry,
                     comment))
                 out.extend(self.write(section=this_entry))
             else:
                 out.append(self._write_line(
                     indent_string,
                     entry,
                     this_entry,
                     comment))
                 
         if section is self:
             for line in self.final_comment:
                 line = self._decode_element(line)
                 stripped_line = line.strip()
                 if stripped_line and not stripped_line.startswith(cs):
                     line = csp + line
                 out.append(line)
             self.interpolation = int_val
             
         if section is not self:
             return out
         
         if (self.filename is None) and (outfile is None):
             # output a list of lines
             # might need to encode
             # NOTE: This will *screw* UTF16, each line will start with the BOM
             if self.encoding:
                 out = [l.encode(self.encoding) for l in out]
             if (self.BOM and ((self.encoding is None) or
                 (BOM_LIST.get(self.encoding.lower()) == 'utf_8'))):
                 # Add the UTF8 BOM
                 if not out:
                     out.append('')
                 out[0] = BOM_UTF8 + out[0]
             return out
         
         # Turn the list to a string, joined with correct newlines
         newline = self.newlines or os.linesep
         output = self._a_to_u(newline).join(out)
         if self.encoding:
             output = output.encode(self.encoding)
         if self.BOM and ((self.encoding is None) or match_utf8(self.encoding)):
             # Add the UTF8 BOM
             output = BOM_UTF8 + output
             
         if not output.endswith(newline):
             output += newline
         if outfile is not None:
             outfile.write(output)
         else:
             h = open(self.filename, 'wb')
             h.write(output)
             h.close()
 
 
     def validate(self, validator, preserve_errors=False, copy=False,
                  section=None):
         """
         Test the ConfigObj against a configspec.
         
         It uses the ``validator`` object from *validate.py*.
         
         To run ``validate`` on the current ConfigObj, call: ::
         
             test = config.validate(validator)
         
         (Normally having previously passed in the configspec when the ConfigObj
         was created - you can dynamically assign a dictionary of checks to the
         ``configspec`` attribute of a section though).
         
         It returns ``True`` if everything passes, or a dictionary of
         pass/fails (True/False). If every member of a subsection passes, it
         will just have the value ``True``. (It also returns ``False`` if all
         members fail).
         
         In addition, it converts the values from strings to their native
         types if their checks pass (and ``stringify`` is set).
         
         If ``preserve_errors`` is ``True`` (``False`` is default) then instead
         of a marking a fail with a ``False``, it will preserve the actual
         exception object. This can contain info about the reason for failure.
         For example the ``VdtValueTooSmallError`` indicates that the value
         supplied was too small. If a value (or section) is missing it will
         still be marked as ``False``.
         
         You must have the validate module to use ``preserve_errors=True``.
         
         You can then use the ``flatten_errors`` function to turn your nested
         results dictionary into a flattened list of failures - useful for
         displaying meaningful error messages.
         """
         if section is None:
             if self.configspec is None:
                 raise ValueError('No configspec supplied.')
             if preserve_errors:
                 # We do this once to remove a top level dependency on the validate module
                 # Which makes importing configobj faster
                 from validate import VdtMissingValue
                 self._vdtMissingValue = VdtMissingValue
             section = self
         #
         spec_section = section.configspec
         if copy and hasattr(section, '_configspec_initial_comment'):
             section.initial_comment = section._configspec_initial_comment
             section.final_comment = section._configspec_final_comment
             section.encoding = section._configspec_encoding
             section.BOM = section._configspec_BOM
             section.newlines = section._configspec_newlines
             section.indent_type = section._configspec_indent_type
             
         if '__many__' in section.configspec:
             many = spec_section['__many__']
             # dynamically assign the configspecs
             # for the sections below
             for entry in section.sections:
                 self._handle_repeat(section[entry], many)
         #
         out = {}
         ret_true = True
         ret_false = True
         order = [k for k in section._order if k in spec_section]
         order += [k for k in spec_section if k not in order]
         for entry in order:
             if entry == '__many__':
                 continue
             if (not entry in section.scalars) or (entry in section.defaults):
                 # missing entries
                 # or entries from defaults
                 missing = True
                 val = None
                 if copy and not entry in section.scalars:
                     # copy comments
                     section.comments[entry] = (
                         section._configspec_comments.get(entry, []))
                     section.inline_comments[entry] = (
                         section._configspec_inline_comments.get(entry, ''))
                 #
             else:
                 missing = False
                 val = section[entry]
             try:
                 check = validator.check(spec_section[entry],
                                         val,
                                         missing=missing
                                         )
             except validator.baseErrorClass, e:
                 if not preserve_errors or isinstance(e, self._vdtMissingValue):
                     out[entry] = False
                 else:
                     # preserve the error
                     out[entry] = e
                     ret_false = False
                 ret_true = False
             else:
                 try: 
                     section.default_values.pop(entry, None)
                 except AttributeError: 
                     # For Python 2.2 compatibility
                     try:
                         del section.default_values[entry]
                     except KeyError:
                         pass
                     
                 if hasattr(validator, 'get_default_value'):
                     try: 
                         section.default_values[entry] = validator.get_default_value(spec_section[entry])
                     except KeyError:
                         # No default
                         pass
                     
                 ret_false = False
                 out[entry] = True
                 if self.stringify or missing:
                     # if we are doing type conversion
                     # or the value is a supplied default
                     if not self.stringify:
                         if isinstance(check, (list, tuple)):
                             # preserve lists
                             check = [self._str(item) for item in check]
                         elif missing and check is None:
                             # convert the None from a default to a ''
                             check = ''
                         else:
                             check = self._str(check)
                     if (check != val) or missing:
                         section[entry] = check
                 if not copy and missing and entry not in section.defaults:
                     section.defaults.append(entry)
         # Missing sections will have been created as empty ones when the
         # configspec was read.
         for entry in section.sections:
             # FIXME: this means DEFAULT is not copied in copy mode
             if section is self and entry == 'DEFAULT':
                 continue
             if copy:
                 section.comments[entry] = section._cs_section_comments[entry]
                 section.inline_comments[entry] = (
                     section._cs_section_inline_comments[entry])
             check = self.validate(validator, preserve_errors=preserve_errors,
                 copy=copy, section=section[entry])
             out[entry] = check
             if check == False:
                 ret_true = False
             elif check == True:
                 ret_false = False
             else:
                 ret_true = False
                 ret_false = False
         #
         if ret_true:
             return True
         elif ret_false:
             return False
         return out
 
 
     def reset(self):
         """Clear ConfigObj instance and restore to 'freshly created' state."""
         self.clear()
         self._initialise()
         # FIXME: Should be done by '_initialise', but ConfigObj constructor (and reload)
         #        requires an empty dictionary
         self.configspec = None
         # Just to be sure ;-)
         self._original_configspec = None
         
         
     def reload(self):
         """
         Reload a ConfigObj from file.
         
         This method raises a ``ReloadError`` if the ConfigObj doesn't have
         a filename attribute pointing to a file.
         """
         if not isinstance(self.filename, StringTypes):
             raise ReloadError()
 
         filename = self.filename
         current_options = {}
         for entry in OPTION_DEFAULTS:
             if entry == 'configspec':
                 continue
             current_options[entry] = getattr(self, entry)
             
         configspec = self._original_configspec
         current_options['configspec'] = configspec
             
         self.clear()
         self._initialise(current_options)
         self._load(filename, configspec)
         
 
 
 class SimpleVal(object):
     """
     A simple validator.
     Can be used to check that all members expected are present.
     
     To use it, provide a configspec with all your members in (the value given
     will be ignored). Pass an instance of ``SimpleVal`` to the ``validate``
     method of your ``ConfigObj``. ``validate`` will return ``True`` if all
     members are present, or a dictionary with True/False meaning
     present/missing. (Whole missing sections will be replaced with ``False``)
     """
     
     def __init__(self):
         self.baseErrorClass = ConfigObjError
     
     def check(self, check, member, missing=False):
         """A dummy check method, always returns the value unchanged."""
         if missing:
             raise self.baseErrorClass()
         return member
 
 
 # Check / processing functions for options
 def flatten_errors(cfg, res, levels=None, results=None):
     """
     An example function that will turn a nested dictionary of results
     (as returned by ``ConfigObj.validate``) into a flat list.
     
     ``cfg`` is the ConfigObj instance being checked, ``res`` is the results
     dictionary returned by ``validate``.
     
     (This is a recursive function, so you shouldn't use the ``levels`` or
     ``results`` arguments - they are used by the function.
     
     Returns a list of keys that failed. Each member of the list is a tuple :
     ::
     
         ([list of sections...], key, result)
     
     If ``validate`` was called with ``preserve_errors=False`` (the default)
     then ``result`` will always be ``False``.
 
     *list of sections* is a flattened list of sections that the key was found
     in.
     
     If the section was missing then key will be ``None``.
     
     If the value (or section) was missing then ``result`` will be ``False``.
     
     If ``validate`` was called with ``preserve_errors=True`` and a value
     was present, but failed the check, then ``result`` will be the exception
     object returned. You can use this as a string that describes the failure.
     
     For example *The value "3" is of the wrong type*.
     
     >>> import validate
     >>> vtor = validate.Validator()
     >>> my_ini = '''
     ...     option1 = True
     ...     [section1]
     ...     option1 = True
     ...     [section2]
     ...     another_option = Probably
     ...     [section3]
     ...     another_option = True
     ...     [[section3b]]
     ...     value = 3
     ...     value2 = a
     ...     value3 = 11
     ...     '''
     >>> my_cfg = '''
     ...     option1 = boolean()
     ...     option2 = boolean()
     ...     option3 = boolean(default=Bad_value)
     ...     [section1]
     ...     option1 = boolean()
     ...     option2 = boolean()
     ...     option3 = boolean(default=Bad_value)
     ...     [section2]
     ...     another_option = boolean()
     ...     [section3]
     ...     another_option = boolean()
     ...     [[section3b]]
     ...     value = integer
     ...     value2 = integer
     ...     value3 = integer(0, 10)
     ...         [[[section3b-sub]]]
     ...         value = string
     ...     [section4]
     ...     another_option = boolean()
     ...     '''
     >>> cs = my_cfg.split('\\n')
     >>> ini = my_ini.split('\\n')
     >>> cfg = ConfigObj(ini, configspec=cs)
     >>> res = cfg.validate(vtor, preserve_errors=True)
     >>> errors = []
     >>> for entry in flatten_errors(cfg, res):
     ...     section_list, key, error = entry
     ...     section_list.insert(0, '[root]')
     ...     if key is not None:
     ...        section_list.append(key)
     ...     else:
     ...         section_list.append('[missing]')
     ...     section_string = ', '.join(section_list)
     ...     errors.append((section_string, ' = ', error))
     >>> errors.sort()
     >>> for entry in errors:
     ...     print entry[0], entry[1], (entry[2] or 0)
     [root], option2  =  0
     [root], option3  =  the value "Bad_value" is of the wrong type.
     [root], section1, option2  =  0
     [root], section1, option3  =  the value "Bad_value" is of the wrong type.
     [root], section2, another_option  =  the value "Probably" is of the wrong type.
     [root], section3, section3b, section3b-sub, [missing]  =  0
     [root], section3, section3b, value2  =  the value "a" is of the wrong type.
     [root], section3, section3b, value3  =  the value "11" is too big.
     [root], section4, [missing]  =  0
     """
     if levels is None:
         # first time called
         levels = []
         results = []
     if res is True:
         return results
     if res is False:
         results.append((levels[:], None, False))
         if levels:
             levels.pop()
         return results
-    for (key, val) in res.items():
+    for (key, val) in res.iteritems():
         if val == True:
             continue
         if isinstance(cfg.get(key), dict):
             # Go down one level
             levels.append(key)
             flatten_errors(cfg[key], val, levels, results)
             continue
         results.append((levels[:], key, val))
     #
     # Go up one level
     if levels:
         levels.pop()
     #
     return results
 
 
 """*A programming language is a medium of expression.* - Paul Graham"""

IPython/external/path.py

@@ -1,973 +1,944 b''
 """ path.py - An object representing a path to a file or directory.
 
 Example:
 
 from IPython.external.path import path
 d = path('/home/guido/bin')
 for f in d.files('*.py'):
     f.chmod(0755)
 
-This module requires Python 2.2 or later.
+This module requires Python 2.5 or later.
 
 
 URL:     http://www.jorendorff.com/articles/python/path
 Author:  Jason Orendorff <jason.orendorff\x40gmail\x2ecom> (and others - see the url!)
 Date:    9 Mar 2007
 """
 
 
 # TODO
 #   - Tree-walking functions don't avoid symlink loops.  Matt Harrison
 #     sent me a patch for this.
 #   - Bug in write_text().  It doesn't support Universal newline mode.
 #   - Better error message in listdir() when self isn't a
 #     directory. (On Windows, the error message really sucks.)
 #   - Make sure everything has a good docstring.
 #   - Add methods for regex find and replace.
 #   - guess_content_type() method?
 #   - Perhaps support arguments to touch().
 
 from __future__ import generators
 
 import sys, warnings, os, fnmatch, glob, shutil, codecs
-# deprecated in python 2.6
-warnings.filterwarnings('ignore', r'.*md5.*')
-import md5
+from hashlib import md5
 
 __version__ = '2.2'
 __all__ = ['path']
 
 # Platform-specific support for path.owner
 if os.name == 'nt':
     try:
         import win32security
     except ImportError:
         win32security = None
 else:
     try:
         import pwd
     except ImportError:
         pwd = None
 
-# Pre-2.3 support.  Are unicode filenames supported?
-_base = str
-_getcwd = os.getcwd
-try:
-    if os.path.supports_unicode_filenames:
-        _base = unicode
-        _getcwd = os.getcwdu
-except AttributeError:
-    pass
-
-# Pre-2.3 workaround for booleans
-try:
-    True, False
-except NameError:
-    True, False = 1, 0
-
-# Pre-2.3 workaround for basestring.
-try:
-    basestring
-except NameError:
-    basestring = (str, unicode)
-
-# Universal newline support
-_textmode = 'r'
-if hasattr(file, 'newlines'):
-    _textmode = 'U'
-
 
 class TreeWalkWarning(Warning):
     pass
 
-class path(_base):
+class path(unicode):
     """ Represents a filesystem path.
 
     For documentation on individual methods, consult their
     counterparts in os.path.
     """
 
     # --- Special Python methods.
 
     def __repr__(self):
-        return 'path(%s)' % _base.__repr__(self)
+        return 'path(%s)' % unicode.__repr__(self)
 
     # Adding a path and a string yields a path.
     def __add__(self, more):
         try:
-            resultStr = _base.__add__(self, more)
+            resultStr = unicode.__add__(self, more)
         except TypeError:  #Python bug
             resultStr = NotImplemented
         if resultStr is NotImplemented:
             return resultStr
         return self.__class__(resultStr)
 
     def __radd__(self, other):
         if isinstance(other, basestring):
             return self.__class__(other.__add__(self))
         else:
             return NotImplemented
 
     # The / operator joins paths.
     def __div__(self, rel):
         """ fp.__div__(rel) == fp / rel == fp.joinpath(rel)
 
         Join two path components, adding a separator character if
         needed.
         """
         return self.__class__(os.path.join(self, rel))
 
     # Make the / operator work even when true division is enabled.
     __truediv__ = __div__
 
     def getcwd(cls):
         """ Return the current working directory as a path object. """
-        return cls(_getcwd())
+        return cls(os.getcwdu())
     getcwd = classmethod(getcwd)
 
 
     # --- Operations on path strings.
 
     isabs = os.path.isabs
     def abspath(self):       return self.__class__(os.path.abspath(self))
     def normcase(self):      return self.__class__(os.path.normcase(self))
     def normpath(self):      return self.__class__(os.path.normpath(self))
     def realpath(self):      return self.__class__(os.path.realpath(self))
     def expanduser(self):    return self.__class__(os.path.expanduser(self))
     def expandvars(self):    return self.__class__(os.path.expandvars(self))
     def dirname(self):       return self.__class__(os.path.dirname(self))
     basename = os.path.basename
 
     def expand(self):
         """ Clean up a filename by calling expandvars(),
         expanduser(), and normpath() on it.
 
         This is commonly everything needed to clean up a filename
         read from a configuration file, for example.
         """
         return self.expandvars().expanduser().normpath()
 
     def _get_namebase(self):
         base, ext = os.path.splitext(self.name)
         return base
 
     def _get_ext(self):
-        f, ext = os.path.splitext(_base(self))
+        f, ext = os.path.splitext(unicode(self))
         return ext
 
     def _get_drive(self):
         drive, r = os.path.splitdrive(self)
         return self.__class__(drive)
 
     parent = property(
         dirname, None, None,
         """ This path's parent directory, as a new path object.
 
         For example, path('/usr/local/lib/libpython.so').parent == path('/usr/local/lib')
         """)
 
     name = property(
         basename, None, None,
         """ The name of this file or directory without the full path.
 
         For example, path('/usr/local/lib/libpython.so').name == 'libpython.so'
         """)
 
     namebase = property(
         _get_namebase, None, None,
         """ The same as path.name, but with one file extension stripped off.
 
         For example, path('/home/guido/python.tar.gz').name     == 'python.tar.gz',
         but          path('/home/guido/python.tar.gz').namebase == 'python.tar'
         """)
 
     ext = property(
         _get_ext, None, None,
         """ The file extension, for example '.py'. """)
 
     drive = property(
         _get_drive, None, None,
         """ The drive specifier, for example 'C:'.
         This is always empty on systems that don't use drive specifiers.
         """)
 
     def splitpath(self):
         """ p.splitpath() -> Return (p.parent, p.name). """
         parent, child = os.path.split(self)
         return self.__class__(parent), child
 
     def splitdrive(self):
         """ p.splitdrive() -> Return (p.drive, <the rest of p>).
 
         Split the drive specifier from this path.  If there is
         no drive specifier, p.drive is empty, so the return value
         is simply (path(''), p).  This is always the case on Unix.
         """
         drive, rel = os.path.splitdrive(self)
         return self.__class__(drive), rel
 
     def splitext(self):
         """ p.splitext() -> Return (p.stripext(), p.ext).
 
         Split the filename extension from this path and return
         the two parts.  Either part may be empty.
 
         The extension is everything from '.' to the end of the
         last path segment.  This has the property that if
         (a, b) == p.splitext(), then a + b == p.
         """
         filename, ext = os.path.splitext(self)
         return self.__class__(filename), ext
 
     def stripext(self):
         """ p.stripext() -> Remove one file extension from the path.
 
         For example, path('/home/guido/python.tar.gz').stripext()
         returns path('/home/guido/python.tar').
         """
         return self.splitext()[0]
 
     if hasattr(os.path, 'splitunc'):
         def splitunc(self):
             unc, rest = os.path.splitunc(self)
             return self.__class__(unc), rest
 
         def _get_uncshare(self):
             unc, r = os.path.splitunc(self)
             return self.__class__(unc)
 
         uncshare = property(
             _get_uncshare, None, None,
             """ The UNC mount point for this path.
             This is empty for paths on local drives. """)
 
     def joinpath(self, *args):
         """ Join two or more path components, adding a separator
         character (os.sep) if needed.  Returns a new path
         object.
         """
         return self.__class__(os.path.join(self, *args))
 
     def splitall(self):
         r""" Return a list of the path components in this path.
 
         The first item in the list will be a path.  Its value will be
         either os.curdir, os.pardir, empty, or the root directory of
         this path (for example, '/' or 'C:\\').  The other items in
         the list will be strings.
 
         path.path.joinpath(*result) will yield the original path.
         """
         parts = []
         loc = self
         while loc != os.curdir and loc != os.pardir:
             prev = loc
             loc, child = prev.splitpath()
             if loc == prev:
                 break
             parts.append(child)
         parts.append(loc)
         parts.reverse()
         return parts
 
     def relpath(self):
         """ Return this path as a relative path,
         based from the current working directory.
         """
         cwd = self.__class__(os.getcwd())
         return cwd.relpathto(self)
 
     def relpathto(self, dest):
         """ Return a relative path from self to dest.
 
         If there is no relative path from self to dest, for example if
         they reside on different drives in Windows, then this returns
         dest.abspath().
         """
         origin = self.abspath()
         dest = self.__class__(dest).abspath()
 
         orig_list = origin.normcase().splitall()
         # Don't normcase dest!  We want to preserve the case.
         dest_list = dest.splitall()
 
         if orig_list[0] != os.path.normcase(dest_list[0]):
             # Can't get here from there.
             return dest
 
         # Find the location where the two paths start to differ.
         i = 0
         for start_seg, dest_seg in zip(orig_list, dest_list):
             if start_seg != os.path.normcase(dest_seg):
                 break
             i += 1
 
         # Now i is the point where the two paths diverge.
         # Need a certain number of "os.pardir"s to work up
         # from the origin to the point of divergence.
         segments = [os.pardir] * (len(orig_list) - i)
         # Need to add the diverging part of dest_list.
         segments += dest_list[i:]
         if len(segments) == 0:
             # If they happen to be identical, use os.curdir.
             relpath = os.curdir
         else:
             relpath = os.path.join(*segments)
         return self.__class__(relpath)
 
     # --- Listing, searching, walking, and matching
 
     def listdir(self, pattern=None):
         """ D.listdir() -> List of items in this directory.
 
         Use D.files() or D.dirs() instead if you want a listing
         of just files or just subdirectories.
 
         The elements of the list are path objects.
 
         With the optional 'pattern' argument, this only lists
         items whose names match the given pattern.
         """
         names = os.listdir(self)
         if pattern is not None:
             names = fnmatch.filter(names, pattern)
         return [self / child for child in names]
 
     def dirs(self, pattern=None):
         """ D.dirs() -> List of this directory's subdirectories.
 
         The elements of the list are path objects.
         This does not walk recursively into subdirectories
         (but see path.walkdirs).
 
         With the optional 'pattern' argument, this only lists
         directories whose names match the given pattern.  For
         example, d.dirs('build-*').
         """
         return [p for p in self.listdir(pattern) if p.isdir()]
 
     def files(self, pattern=None):
         """ D.files() -> List of the files in this directory.
 
         The elements of the list are path objects.
         This does not walk into subdirectories (see path.walkfiles).
 
         With the optional 'pattern' argument, this only lists files
         whose names match the given pattern.  For example,
         d.files('*.pyc').
         """
         
         return [p for p in self.listdir(pattern) if p.isfile()]
 
     def walk(self, pattern=None, errors='strict'):
         """ D.walk() -> iterator over files and subdirs, recursively.
 
         The iterator yields path objects naming each child item of
         this directory and its descendants.  This requires that
         D.isdir().
 
         This performs a depth-first traversal of the directory tree.
         Each directory is returned just before all its children.
 
         The errors= keyword argument controls behavior when an
         error occurs.  The default is 'strict', which causes an
         exception.  The other allowed values are 'warn', which
         reports the error via warnings.warn(), and 'ignore'.
         """
         if errors not in ('strict', 'warn', 'ignore'):
             raise ValueError("invalid errors parameter")
 
         try:
             childList = self.listdir()
         except Exception:
             if errors == 'ignore':
                 return
             elif errors == 'warn':
                 warnings.warn(
                     "Unable to list directory '%s': %s"
                     % (self, sys.exc_info()[1]),
                     TreeWalkWarning)
                 return
             else:
                 raise
 
         for child in childList:
             if pattern is None or child.fnmatch(pattern):
                 yield child
             try:
                 isdir = child.isdir()
             except Exception:
                 if errors == 'ignore':
                     isdir = False
                 elif errors == 'warn':
                     warnings.warn(
                         "Unable to access '%s': %s"
                         % (child, sys.exc_info()[1]),
                         TreeWalkWarning)
                     isdir = False
                 else:
                     raise
 
             if isdir:
                 for item in child.walk(pattern, errors):
                     yield item
 
     def walkdirs(self, pattern=None, errors='strict'):
         """ D.walkdirs() -> iterator over subdirs, recursively.
 
         With the optional 'pattern' argument, this yields only
         directories whose names match the given pattern.  For
         example, mydir.walkdirs('*test') yields only directories
         with names ending in 'test'.
 
         The errors= keyword argument controls behavior when an
         error occurs.  The default is 'strict', which causes an
         exception.  The other allowed values are 'warn', which
         reports the error via warnings.warn(), and 'ignore'.
         """
         if errors not in ('strict', 'warn', 'ignore'):
             raise ValueError("invalid errors parameter")
 
         try:
             dirs = self.dirs()
         except Exception:
             if errors == 'ignore':
                 return
             elif errors == 'warn':
                 warnings.warn(
                     "Unable to list directory '%s': %s"
                     % (self, sys.exc_info()[1]),
                     TreeWalkWarning)
                 return
             else:
                 raise
 
         for child in dirs:
             if pattern is None or child.fnmatch(pattern):
                 yield child
             for subsubdir in child.walkdirs(pattern, errors):
                 yield subsubdir
 
     def walkfiles(self, pattern=None, errors='strict'):
         """ D.walkfiles() -> iterator over files in D, recursively.
 
         The optional argument, pattern, limits the results to files
         with names that match the pattern.  For example,
         mydir.walkfiles('*.tmp') yields only files with the .tmp
         extension.
         """
         if errors not in ('strict', 'warn', 'ignore'):
             raise ValueError("invalid errors parameter")
 
         try:
             childList = self.listdir()
         except Exception:
             if errors == 'ignore':
                 return
             elif errors == 'warn':
                 warnings.warn(
                     "Unable to list directory '%s': %s"
                     % (self, sys.exc_info()[1]),
                     TreeWalkWarning)
                 return
             else:
                 raise
 
         for child in childList:
             try:
                 isfile = child.isfile()
                 isdir = not isfile and child.isdir()
             except:
                 if errors == 'ignore':
                     continue
                 elif errors == 'warn':
                     warnings.warn(
                         "Unable to access '%s': %s"
                         % (self, sys.exc_info()[1]),
                         TreeWalkWarning)
                     continue
                 else:
                     raise
 
             if isfile:
                 if pattern is None or child.fnmatch(pattern):
                     yield child
             elif isdir:
                 for f in child.walkfiles(pattern, errors):
                     yield f
 
     def fnmatch(self, pattern):
         """ Return True if self.name matches the given pattern.
 
         pattern - A filename pattern with wildcards,
             for example '*.py'.
         """
         return fnmatch.fnmatch(self.name, pattern)
 
     def glob(self, pattern):
         """ Return a list of path objects that match the pattern.
 
         pattern - a path relative to this directory, with wildcards.
 
         For example, path('/users').glob('*/bin/*') returns a list
         of all the files users have in their bin directories.
         """
         cls = self.__class__
-        return [cls(s) for s in glob.glob(_base(self / pattern))]
+        return [cls(s) for s in glob.glob(unicode(self / pattern))]
 
 
     # --- Reading or writing an entire file at once.
 
     def open(self, mode='r'):
         """ Open this file.  Return a file object. """
-        return file(self, mode)
+        return open(self, mode)
 
     def bytes(self):
         """ Open this file, read all bytes, return them as a string. """
         f = self.open('rb')
         try:
             return f.read()
         finally:
             f.close()
 
     def write_bytes(self, bytes, append=False):
         """ Open this file and write the given bytes to it.
 
         Default behavior is to overwrite any existing file.
         Call p.write_bytes(bytes, append=True) to append instead.
         """
         if append:
             mode = 'ab'
         else:
             mode = 'wb'
         f = self.open(mode)
         try:
             f.write(bytes)
         finally:
             f.close()
 
     def text(self, encoding=None, errors='strict'):
         r""" Open this file, read it in, return the content as a string.
 
         This uses 'U' mode in Python 2.3 and later, so '\r\n' and '\r'
         are automatically translated to '\n'.
 
         Optional arguments:
 
         encoding - The Unicode encoding (or character set) of
             the file.  If present, the content of the file is
             decoded and returned as a unicode object; otherwise
             it is returned as an 8-bit str.
         errors - How to handle Unicode errors; see help(str.decode)
             for the options.  Default is 'strict'.
         """
         if encoding is None:
             # 8-bit
-            f = self.open(_textmode)
+            f = self.open('U')
             try:
                 return f.read()
             finally:
                 f.close()
         else:
             # Unicode
             f = codecs.open(self, 'r', encoding, errors)
             # (Note - Can't use 'U' mode here, since codecs.open
             # doesn't support 'U' mode, even in Python 2.3.)
             try:
                 t = f.read()
             finally:
                 f.close()
             return (t.replace(u'\r\n', u'\n')
                      .replace(u'\r\x85', u'\n')
                      .replace(u'\r', u'\n')
                      .replace(u'\x85', u'\n')
                      .replace(u'\u2028', u'\n'))
 
     def write_text(self, text, encoding=None, errors='strict', linesep=os.linesep, append=False):
         r""" Write the given text to this file.
 
         The default behavior is to overwrite any existing file;
         to append instead, use the 'append=True' keyword argument.
 
         There are two differences between path.write_text() and
         path.write_bytes(): newline handling and Unicode handling.
         See below.
 
         Parameters:
 
           - text - str/unicode - The text to be written.
 
           - encoding - str - The Unicode encoding that will be used.
             This is ignored if 'text' isn't a Unicode string.
 
           - errors - str - How to handle Unicode encoding errors.
             Default is 'strict'.  See help(unicode.encode) for the
             options.  This is ignored if 'text' isn't a Unicode
             string.
 
           - linesep - keyword argument - str/unicode - The sequence of
             characters to be used to mark end-of-line.  The default is
             os.linesep.  You can also specify None; this means to
             leave all newlines as they are in 'text'.
 
           - append - keyword argument - bool - Specifies what to do if
             the file already exists (True: append to the end of it;
             False: overwrite it.)  The default is False.
 
 
         --- Newline handling.
 
         write_text() converts all standard end-of-line sequences
         ('\n', '\r', and '\r\n') to your platform's default end-of-line
         sequence (see os.linesep; on Windows, for example, the
         end-of-line marker is '\r\n').
 
         If you don't like your platform's default, you can override it
         using the 'linesep=' keyword argument.  If you specifically want
         write_text() to preserve the newlines as-is, use 'linesep=None'.
 
         This applies to Unicode text the same as to 8-bit text, except
         there are three additional standard Unicode end-of-line sequences:
         u'\x85', u'\r\x85', and u'\u2028'.
 
         (This is slightly different from when you open a file for
         writing with fopen(filename, "w") in C or file(filename, 'w')
         in Python.)
 
 
         --- Unicode
 
         If 'text' isn't Unicode, then apart from newline handling, the
         bytes are written verbatim to the file.  The 'encoding' and
         'errors' arguments are not used and must be omitted.
 
         If 'text' is Unicode, it is first converted to bytes using the
         specified 'encoding' (or the default encoding if 'encoding'
         isn't specified).  The 'errors' argument applies only to this
         conversion.
 
         """
         if isinstance(text, unicode):
             if linesep is not None:
                 # Convert all standard end-of-line sequences to
                 # ordinary newline characters.
                 text = (text.replace(u'\r\n', u'\n')
                             .replace(u'\r\x85', u'\n')
                             .replace(u'\r', u'\n')
                             .replace(u'\x85', u'\n')
                             .replace(u'\u2028', u'\n'))
                 text = text.replace(u'\n', linesep)
             if encoding is None:
                 encoding = sys.getdefaultencoding()
             bytes = text.encode(encoding, errors)
         else:
             # It is an error to specify an encoding if 'text' is
             # an 8-bit string.
             assert encoding is None
 
             if linesep is not None:
                 text = (text.replace('\r\n', '\n')
                             .replace('\r', '\n'))
                 bytes = text.replace('\n', linesep)
 
         self.write_bytes(bytes, append)
 
     def lines(self, encoding=None, errors='strict', retain=True):
         r""" Open this file, read all lines, return them in a list.
 
         Optional arguments:
             encoding - The Unicode encoding (or character set) of
                 the file.  The default is None, meaning the content
                 of the file is read as 8-bit characters and returned
                 as a list of (non-Unicode) str objects.
             errors - How to handle Unicode errors; see help(str.decode)
                 for the options.  Default is 'strict'
             retain - If true, retain newline characters; but all newline
                 character combinations ('\r', '\n', '\r\n') are
                 translated to '\n'.  If false, newline characters are
                 stripped off.  Default is True.
 
         This uses 'U' mode in Python 2.3 and later.
         """
         if encoding is None and retain:
-            f = self.open(_textmode)
+            f = self.open('U')
             try:
                 return f.readlines()
             finally:
                 f.close()
         else:
             return self.text(encoding, errors).splitlines(retain)
 
     def write_lines(self, lines, encoding=None, errors='strict',
                     linesep=os.linesep, append=False):
         r""" Write the given lines of text to this file.
 
         By default this overwrites any existing file at this path.
 
         This puts a platform-specific newline sequence on every line.
         See 'linesep' below.
 
         lines - A list of strings.
 
         encoding - A Unicode encoding to use.  This applies only if
             'lines' contains any Unicode strings.
 
         errors - How to handle errors in Unicode encoding.  This
             also applies only to Unicode strings.
 
         linesep - The desired line-ending.  This line-ending is
             applied to every line.  If a line already has any
             standard line ending ('\r', '\n', '\r\n', u'\x85',
             u'\r\x85', u'\u2028'), that will be stripped off and
             this will be used instead.  The default is os.linesep,
             which is platform-dependent ('\r\n' on Windows, '\n' on
             Unix, etc.)  Specify None to write the lines as-is,
             like file.writelines().
 
         Use the keyword argument append=True to append lines to the
         file.  The default is to overwrite the file.  Warning:
         When you use this with Unicode data, if the encoding of the
         existing data in the file is different from the encoding
         you specify with the encoding= parameter, the result is
         mixed-encoding data, which can really confuse someone trying
         to read the file later.
         """
         if append:
             mode = 'ab'
         else:
             mode = 'wb'
         f = self.open(mode)
         try:
             for line in lines:
                 isUnicode = isinstance(line, unicode)
                 if linesep is not None:
                     # Strip off any existing line-end and add the
                     # specified linesep string.
                     if isUnicode:
                         if line[-2:] in (u'\r\n', u'\x0d\x85'):
                             line = line[:-2]
                         elif line[-1:] in (u'\r', u'\n',
                                            u'\x85', u'\u2028'):
                             line = line[:-1]
                     else:
                         if line[-2:] == '\r\n':
                             line = line[:-2]
                         elif line[-1:] in ('\r', '\n'):
                             line = line[:-1]
                     line += linesep
                 if isUnicode:
                     if encoding is None:
                         encoding = sys.getdefaultencoding()
                     line = line.encode(encoding, errors)
                 f.write(line)
         finally:
             f.close()
 
     def read_md5(self):
         """ Calculate the md5 hash for this file.
 
         This reads through the entire file.
         """
         f = self.open('rb')
         try:
-            m = md5.new()
+            m = md5()
             while True:
                 d = f.read(8192)
                 if not d:
                     break
                 m.update(d)
         finally:
             f.close()
         return m.digest()
 
     # --- Methods for querying the filesystem.
 
     exists = os.path.exists
     isdir = os.path.isdir
     isfile = os.path.isfile
     islink = os.path.islink
     ismount = os.path.ismount
 
     if hasattr(os.path, 'samefile'):
         samefile = os.path.samefile
 
     getatime = os.path.getatime
     atime = property(
         getatime, None, None,
         """ Last access time of the file. """)
 
     getmtime = os.path.getmtime
     mtime = property(
         getmtime, None, None,
         """ Last-modified time of the file. """)
 
     if hasattr(os.path, 'getctime'):
         getctime = os.path.getctime
         ctime = property(
             getctime, None, None,
             """ Creation time of the file. """)
 
     getsize = os.path.getsize
     size = property(
         getsize, None, None,
         """ Size of the file, in bytes. """)
 
     if hasattr(os, 'access'):
         def access(self, mode):
             """ Return true if current user has access to this path.
 
             mode - One of the constants os.F_OK, os.R_OK, os.W_OK, os.X_OK
             """
             return os.access(self, mode)
 
     def stat(self):
         """ Perform a stat() system call on this path. """
         return os.stat(self)
 
     def lstat(self):
         """ Like path.stat(), but do not follow symbolic links. """
         return os.lstat(self)
 
     def get_owner(self):
         r""" Return the name of the owner of this file or directory.
 
         This follows symbolic links.
 
         On Windows, this returns a name of the form ur'DOMAIN\User Name'.
         On Windows, a group can own a file or directory.
         """
         if os.name == 'nt':
             if win32security is None:
                 raise Exception("path.owner requires win32all to be installed")
             desc = win32security.GetFileSecurity(
                 self, win32security.OWNER_SECURITY_INFORMATION)
             sid = desc.GetSecurityDescriptorOwner()
             account, domain, typecode = win32security.LookupAccountSid(None, sid)
             return domain + u'\\' + account
         else:
             if pwd is None:
                 raise NotImplementedError("path.owner is not implemented on this platform.")
             st = self.stat()
             return pwd.getpwuid(st.st_uid).pw_name
 
     owner = property(
         get_owner, None, None,
         """ Name of the owner of this file or directory. """)
 
     if hasattr(os, 'statvfs'):
         def statvfs(self):
             """ Perform a statvfs() system call on this path. """
             return os.statvfs(self)
 
     if hasattr(os, 'pathconf'):
         def pathconf(self, name):
             return os.pathconf(self, name)
 
 
     # --- Modifying operations on files and directories
 
     def utime(self, times):
         """ Set the access and modified times of this file. """
         os.utime(self, times)
 
     def chmod(self, mode):
         os.chmod(self, mode)
 
     if hasattr(os, 'chown'):
         def chown(self, uid, gid):
             os.chown(self, uid, gid)
 
     def rename(self, new):
         os.rename(self, new)
 
     def renames(self, new):
         os.renames(self, new)
 
 
     # --- Create/delete operations on directories
 
     def mkdir(self, mode=0777):
         os.mkdir(self, mode)
 
     def makedirs(self, mode=0777):
         os.makedirs(self, mode)
 
     def rmdir(self):
         os.rmdir(self)
 
     def removedirs(self):
         os.removedirs(self)
 
 
     # --- Modifying operations on files
 
     def touch(self):
         """ Set the access/modified times of this file to the current time.
         Create the file if it does not exist.
         """
         fd = os.open(self, os.O_WRONLY | os.O_CREAT, 0666)
         os.close(fd)
         os.utime(self, None)
 
     def remove(self):
         os.remove(self)
 
     def unlink(self):
         os.unlink(self)
 
 
     # --- Links
 
     if hasattr(os, 'link'):
         def link(self, newpath):
             """ Create a hard link at 'newpath', pointing to this file. """
             os.link(self, newpath)
 
     if hasattr(os, 'symlink'):
         def symlink(self, newlink):
             """ Create a symbolic link at 'newlink', pointing here. """
             os.symlink(self, newlink)
 
     if hasattr(os, 'readlink'):
         def readlink(self):
             """ Return the path to which this symbolic link points.
 
             The result may be an absolute or a relative path.
             """
             return self.__class__(os.readlink(self))
 
         def readlinkabs(self):
             """ Return the path to which this symbolic link points.
 
             The result is always an absolute path.
             """
             p = self.readlink()
             if p.isabs():
                 return p
             else:
                 return (self.parent / p).abspath()
 
 
     # --- High-level functions from shutil
 
     copyfile = shutil.copyfile
     copymode = shutil.copymode
     copystat = shutil.copystat
     copy = shutil.copy
     copy2 = shutil.copy2
     copytree = shutil.copytree
     if hasattr(shutil, 'move'):
         move = shutil.move
     rmtree = shutil.rmtree
 
 
     # --- Special stuff from os
 
     if hasattr(os, 'chroot'):
         def chroot(self):
             os.chroot(self)
 
     if hasattr(os, 'startfile'):
         def startfile(self):
             os.startfile(self)
 

IPython/external/pretty.py

@@ -1,705 +1,690 b''
 # -*- coding: utf-8 -*-
 """
     pretty
     ~~
 
     Python advanced pretty printer.  This pretty printer is intended to
     replace the old `pprint` python module which does not allow developers
     to provide their own pretty print callbacks.
 
     This module is based on ruby's `prettyprint.rb` library by `Tanaka Akira`.
 
 
     Example Usage
     =============
 
     To directly print the representation of an object use `pprint`::
 
         from pretty import pprint
         pprint(complex_object)
 
     To get a string of the output use `pretty`::
 
         from pretty import pretty
         string = pretty(complex_object)
 
 
     Extending
     =========
 
     The pretty library allows developers to add pretty printing rules for their
     own objects.  This process is straightforward.  All you have to do is to
     add a `__pretty__` method to your object and call the methods on the
     pretty printer passed::
 
         class MyObject(object):
 
             def __pretty__(self, p, cycle):
                 ...
 
     Depending on the python version you want to support you have two
     possibilities.  The following list shows the python 2.5 version and the
     compatibility one.
 
 
     Here the example implementation of a `__pretty__` method for a list
     subclass for python 2.5 and higher (python 2.5 requires the with statement
     __future__ import)::
 
         class MyList(list):
 
             def __pretty__(self, p, cycle):
                 if cycle:
                     p.text('MyList(...)')
                 else:
                     with p.group(8, 'MyList([', '])'):
                         for idx, item in enumerate(self):
                             if idx:
                                 p.text(',')
                                 p.breakable()
                             p.pretty(item)
 
     The `cycle` parameter is `True` if pretty detected a cycle.  You *have* to
     react to that or the result is an infinite loop.  `p.text()` just adds
     non breaking text to the output, `p.breakable()` either adds a whitespace
     or breaks here.  If you pass it an argument it's used instead of the
     default space.  `p.pretty` prettyprints another object using the pretty print
     method.
 
     The first parameter to the `group` function specifies the extra indentation
     of the next line.  In this example the next item will either be not
     breaked (if the items are short enough) or aligned with the right edge of
     the opening bracked of `MyList`.
 
     If you want to support python 2.4 and lower you can use this code::
 
         class MyList(list):
 
             def __pretty__(self, p, cycle):
                 if cycle:
                     p.text('MyList(...)')
                 else:
                     p.begin_group(8, 'MyList([')
                     for idx, item in enumerate(self):
                         if idx:
                             p.text(',')
                             p.breakable()
                         p.pretty(item)
                     p.end_group(8, '])')
 
     If you just want to indent something you can use the group function
     without open / close parameters.  Under python 2.5 you can also use this
     code::
 
         with p.indent(2):
             ...
 
     Or under python2.4 you might want to modify ``p.indentation`` by hand but
     this is rather ugly.
 
     :copyright: 2007 by Armin Ronacher.
                 Portions (c) 2009 by Robert Kern.
     :license: BSD License.
 """
-import __future__
+from __future__ import with_statement
+from contextlib import contextmanager
 import sys
 import types
 import re
 import datetime
 from StringIO import StringIO
 from collections import deque
 
 
 __all__ = ['pretty', 'pprint', 'PrettyPrinter', 'RepresentationPrinter',
     'for_type', 'for_type_by_name']
 
 
 _re_pattern_type = type(re.compile(''))
 
 
 def pretty(obj, verbose=False, max_width=79, newline='\n'):
     """
     Pretty print the object's representation.
     """
     stream = StringIO()
     printer = RepresentationPrinter(stream, verbose, max_width, newline)
     printer.pretty(obj)
     printer.flush()
     return stream.getvalue()
 
 
 def pprint(obj, verbose=False, max_width=79, newline='\n'):
     """
     Like `pretty` but print to stdout.
     """
     printer = RepresentationPrinter(sys.stdout, verbose, max_width, newline)
     printer.pretty(obj)
     printer.flush()
     sys.stdout.write(newline)
     sys.stdout.flush()
 
-
-# add python2.5 context managers if we have the with statement feature
-if hasattr(__future__, 'with_statement'): exec '''
-from __future__ import with_statement
-from contextlib import contextmanager
-
 class _PrettyPrinterBase(object):
 
     @contextmanager
     def indent(self, indent):
         """with statement support for indenting/dedenting."""
         self.indentation += indent
         try:
             yield
         finally:
             self.indentation -= indent
 
     @contextmanager
     def group(self, indent=0, open='', close=''):
         """like begin_group / end_group but for the with statement."""
         self.begin_group(indent, open)
         try:
             with self.indent(indent):
                 yield
         finally:
             self.end_group(indent, close)
-'''
-else:
-    class _PrettyPrinterBase(object):
-
-        def _unsupported(self, *a, **kw):
-            """unsupported operation"""
-            raise RuntimeError('not available in this python version')
-        group = indent = _unsupported
-        del _unsupported
-
 
 class PrettyPrinter(_PrettyPrinterBase):
     """
     Baseclass for the `RepresentationPrinter` prettyprinter that is used to
     generate pretty reprs of objects.  Contrary to the `RepresentationPrinter`
     this printer knows nothing about the default pprinters or the `__pretty__`
     callback method.
     """
 
     def __init__(self, output, max_width=79, newline='\n'):
         self.output = output
         self.max_width = max_width
         self.newline = newline
         self.output_width = 0
         self.buffer_width = 0
         self.buffer = deque()
 
         root_group = Group(0)
         self.group_stack = [root_group]
         self.group_queue = GroupQueue(root_group)
         self.indentation = 0
 
     def _break_outer_groups(self):
         while self.max_width < self.output_width + self.buffer_width:
             group = self.group_queue.deq()
             if not group:
                 return
             while group.breakables:
                 x = self.buffer.popleft()
                 self.output_width = x.output(self.output, self.output_width)
                 self.buffer_width -= x.width
             while self.buffer and isinstance(self.buffer[0], Text):
                 x = self.buffer.popleft()
                 self.output_width = x.output(self.output, self.output_width)
                 self.buffer_width -= x.width
 
     def text(self, obj):
         """Add literal text to the output."""
         width = len(obj)
         if self.buffer:
             text = self.buffer[-1]
             if not isinstance(text, Text):
                 text = Text()
                 self.buffer.append(text)
             text.add(obj, width)
             self.buffer_width += width
             self._break_outer_groups()
         else:
             self.output.write(obj)
             self.output_width += width
 
     def breakable(self, sep=' '):
         """
         Add a breakable separator to the output.  This does not mean that it
         will automatically break here.  If no breaking on this position takes
         place the `sep` is inserted which default to one space.
         """
         width = len(sep)
         group = self.group_stack[-1]
         if group.want_break:
             self.flush()
             self.output.write(self.newline)
             self.output.write(' ' * self.indentation)
             self.output_width = self.indentation
             self.buffer_width = 0
         else:
             self.buffer.append(Breakable(sep, width, self))
             self.buffer_width += width
             self._break_outer_groups()
 
 
     def begin_group(self, indent=0, open=''):
         """
         Begin a group.  If you want support for python < 2.5 which doesn't has
         the with statement this is the preferred way:
 
             p.begin_group(1, '{')
             ...
             p.end_group(1, '}')
 
         The python 2.5 expression would be this:
 
             with p.group(1, '{', '}'):
                 ...
 
         The first parameter specifies the indentation for the next line (usually
         the width of the opening text), the second the opening text.  All
         parameters are optional.
         """
         if open:
             self.text(open)
         group = Group(self.group_stack[-1].depth + 1)
         self.group_stack.append(group)
         self.group_queue.enq(group)
         self.indentation += indent
 
     def end_group(self, dedent=0, close=''):
         """End a group. See `begin_group` for more details."""
         self.indentation -= dedent
         group = self.group_stack.pop()
         if not group.breakables:
             self.group_queue.remove(group)
         if close:
             self.text(close)
 
     def flush(self):
         """Flush data that is left in the buffer."""
         for data in self.buffer:
             self.output_width += data.output(self.output, self.output_width)
         self.buffer.clear()
         self.buffer_width = 0
 
 
 def _get_mro(obj_class):
     """ Get a reasonable method resolution order of a class and its superclasses
     for both old-style and new-style classes.
     """
     if not hasattr(obj_class, '__mro__'):
         # Old-style class. Mix in object to make a fake new-style class.
         try:
             obj_class = type(obj_class.__name__, (obj_class, object), {})
         except TypeError:
             # Old-style extension type that does not descend from object.
             # FIXME: try to construct a more thorough MRO.
             mro = [obj_class]
         else:
             mro = obj_class.__mro__[1:-1]
     else:
         mro = obj_class.__mro__
     return mro
 
 
 class RepresentationPrinter(PrettyPrinter):
     """
     Special pretty printer that has a `pretty` method that calls the pretty
     printer for a python object.
 
     This class stores processing data on `self` so you must *never* use
     this class in a threaded environment.  Always lock it or reinstanciate
     it.
 
     Instances also have a verbose flag callbacks can access to control their
     output.  For example the default instance repr prints all attributes and
     methods that are not prefixed by an underscore if the printer is in
     verbose mode.
     """
 
     def __init__(self, output, verbose=False, max_width=79, newline='\n'):
         PrettyPrinter.__init__(self, output, max_width, newline)
         self.verbose = verbose
         self.stack = []
 
     def pretty(self, obj):
         """Pretty print the given object."""
         obj_id = id(obj)
         cycle = obj_id in self.stack
         self.stack.append(obj_id)
         self.begin_group()
         try:
             obj_class = getattr(obj, '__class__', None) or type(obj)
             if hasattr(obj_class, '__pretty__'):
                 return obj_class.__pretty__(obj, self, cycle)
             try:
                 printer = _singleton_pprinters[obj_id]
             except (TypeError, KeyError):
                 pass
             else:
                 return printer(obj, self, cycle)
             for cls in _get_mro(obj_class):
                 if cls in _type_pprinters:
                     return _type_pprinters[cls](obj, self, cycle)
                 else:
                     printer = self._in_deferred_types(cls)
                     if printer is not None:
                         return printer(obj, self, cycle)
             return _default_pprint(obj, self, cycle)
         finally:
             self.end_group()
             self.stack.pop()
 
     def _in_deferred_types(self, cls):
         """
         Check if the given class is specified in the deferred type registry.
 
         Returns the printer from the registry if it exists, and None if the
         class is not in the registry. Successful matches will be moved to the
         regular type registry for future use.
         """
         mod = getattr(cls, '__module__', None)
         name = getattr(cls, '__name__', None)
         key = (mod, name)
         printer = None
         if key in _deferred_type_pprinters:
             # Move the printer over to the regular registry.
             printer = _deferred_type_pprinters.pop(key)
             _type_pprinters[cls] = printer
         return printer
 
 
 
 class Printable(object):
 
     def output(self, stream, output_width):
         return output_width
 
 
 class Text(Printable):
 
     def __init__(self):
         self.objs = []
         self.width = 0
 
     def output(self, stream, output_width):
         for obj in self.objs:
             stream.write(obj)
         return output_width + self.width
 
     def add(self, obj, width):
         self.objs.append(obj)
         self.width += width
 
 
 class Breakable(Printable):
 
     def __init__(self, seq, width, pretty):
         self.obj = seq
         self.width = width
         self.pretty = pretty
         self.indentation = pretty.indentation
         self.group = pretty.group_stack[-1]
         self.group.breakables.append(self)
 
     def output(self, stream, output_width):
         self.group.breakables.popleft()
         if self.group.want_break:
             stream.write(self.pretty.newline)
             stream.write(' ' * self.indentation)
             return self.indentation
         if not self.group.breakables:
             self.pretty.group_queue.remove(self.group)
         stream.write(self.obj)
         return output_width + self.width
 
 
 class Group(Printable):
 
     def __init__(self, depth):
         self.depth = depth
         self.breakables = deque()
         self.want_break = False
 
 
 class GroupQueue(object):
 
     def __init__(self, *groups):
         self.queue = []
         for group in groups:
             self.enq(group)
 
     def enq(self, group):
         depth = group.depth
         while depth > len(self.queue) - 1:
             self.queue.append([])
         self.queue[depth].append(group)
 
     def deq(self):
         for stack in self.queue:
             for idx, group in enumerate(reversed(stack)):
                 if group.breakables:
                     del stack[idx]
                     group.want_break = True
                     return group
             for group in stack:
                 group.want_break = True
             del stack[:]
 
     def remove(self, group):
         try:
             self.queue[group.depth].remove(group)
         except ValueError:
             pass
 
 
 _baseclass_reprs = (object.__repr__, types.InstanceType.__repr__)
 
 
 def _default_pprint(obj, p, cycle):
     """
     The default print function.  Used if an object does not provide one and
     it's none of the builtin objects.
     """
     klass = getattr(obj, '__class__', None) or type(obj)
     if getattr(klass, '__repr__', None) not in _baseclass_reprs:
         # A user-provided repr.
         p.text(repr(obj))
         return
     p.begin_group(1, '<')
     p.pretty(klass)
     p.text(' at 0x%x' % id(obj))
     if cycle:
         p.text(' ...')
     elif p.verbose:
         first = True
         for key in dir(obj):
             if not key.startswith('_'):
                 try:
                     value = getattr(obj, key)
                 except AttributeError:
                     continue
                 if isinstance(value, types.MethodType):
                     continue
                 if not first:
                     p.text(',')
                 p.breakable()
                 p.text(key)
                 p.text('=')
                 step = len(key) + 1
                 p.indentation += step
                 p.pretty(value)
                 p.indentation -= step
                 first = False
     p.end_group(1, '>')
 
 
 def _seq_pprinter_factory(start, end):
     """
     Factory that returns a pprint function useful for sequences.  Used by
     the default pprint for tuples, dicts, lists, sets and frozensets.
     """
     def inner(obj, p, cycle):
         if cycle:
             return p.text(start + '...' + end)
         step = len(start)
         p.begin_group(step, start)
         for idx, x in enumerate(obj):
             if idx:
                 p.text(',')
                 p.breakable()
             p.pretty(x)
         if len(obj) == 1 and type(obj) is tuple:
             # Special case for 1-item tuples.
             p.text(',')
         p.end_group(step, end)
     return inner
 
 
 def _dict_pprinter_factory(start, end):
     """
     Factory that returns a pprint function used by the default pprint of
     dicts and dict proxies.
     """
     def inner(obj, p, cycle):
         if cycle:
             return p.text('{...}')
         p.begin_group(1, start)
         keys = obj.keys()
         try:
             keys.sort()
         except Exception, e:
             # Sometimes the keys don't sort.
             pass
         for idx, key in enumerate(keys):
             if idx:
                 p.text(',')
                 p.breakable()
             p.pretty(key)
             p.text(': ')
             p.pretty(obj[key])
         p.end_group(1, end)
     return inner
 
 
 def _super_pprint(obj, p, cycle):
     """The pprint for the super type."""
     p.begin_group(8, '<super: ')
     p.pretty(obj.__self_class__)
     p.text(',')
     p.breakable()
     p.pretty(obj.__self__)
     p.end_group(8, '>')
 
 
 def _re_pattern_pprint(obj, p, cycle):
     """The pprint function for regular expression patterns."""
     p.text('re.compile(')
     pattern = repr(obj.pattern)
     if pattern[:1] in 'uU':
         pattern = pattern[1:]
         prefix = 'ur'
     else:
         prefix = 'r'
     pattern = prefix + pattern.replace('\\\\', '\\')
     p.text(pattern)
     if obj.flags:
         p.text(',')
         p.breakable()
         done_one = False
         for flag in ('TEMPLATE', 'IGNORECASE', 'LOCALE', 'MULTILINE', 'DOTALL',
             'UNICODE', 'VERBOSE', 'DEBUG'):
             if obj.flags & getattr(re, flag):
                 if done_one:
                     p.text('|')
                 p.text('re.' + flag)
                 done_one = True
     p.text(')')
 
 
 def _type_pprint(obj, p, cycle):
     """The pprint for classes and types."""
     if obj.__module__ in ('__builtin__', 'exceptions'):
         name = obj.__name__
     else:
         name = obj.__module__ + '.' + obj.__name__
     p.text(name)
 
 
 def _repr_pprint(obj, p, cycle):
     """A pprint that just redirects to the normal repr function."""
     p.text(repr(obj))
 
 
 def _function_pprint(obj, p, cycle):
     """Base pprint for all functions and builtin functions."""
     if obj.__module__ in ('__builtin__', 'exceptions') or not obj.__module__:
         name = obj.__name__
     else:
         name = obj.__module__ + '.' + obj.__name__
     p.text('<function %s>' % name)
 
 
 def _exception_pprint(obj, p, cycle):
     """Base pprint for all exceptions."""
     if obj.__class__.__module__ == 'exceptions':
         name = obj.__class__.__name__
     else:
         name = '%s.%s' % (
             obj.__class__.__module__,
             obj.__class__.__name__
         )
     step = len(name) + 1
     p.begin_group(step, '(')
     for idx, arg in enumerate(getattr(obj, 'args', ())):
         if idx:
             p.text(',')
             p.breakable()
         p.pretty(arg)
     p.end_group(step, ')')
 
 
 #: the exception base
 try:
     _exception_base = BaseException
 except NameError:
     _exception_base = Exception
 
 
 #: printers for builtin types
 _type_pprinters = {
     int:                        _repr_pprint,
     long:                       _repr_pprint,
     float:                      _repr_pprint,
     str:                        _repr_pprint,
     unicode:                    _repr_pprint,
     tuple:                      _seq_pprinter_factory('(', ')'),
     list:                       _seq_pprinter_factory('[', ']'),
     dict:                       _dict_pprinter_factory('{', '}'),
     types.DictProxyType:        _dict_pprinter_factory('<dictproxy {', '}>'),
     set:                        _seq_pprinter_factory('set([', '])'),
     frozenset:                  _seq_pprinter_factory('frozenset([', '])'),
     super:                      _super_pprint,
     _re_pattern_type:           _re_pattern_pprint,
     type:                       _type_pprint,
     types.ClassType:            _type_pprint,
     types.FunctionType:         _function_pprint,
     types.BuiltinFunctionType:  _function_pprint,
     types.SliceType:            _repr_pprint,
     types.MethodType:           _repr_pprint,
     xrange:                     _repr_pprint,
     datetime.datetime:          _repr_pprint,
     datetime.timedelta:         _repr_pprint,
     _exception_base:            _exception_pprint
 }
 
 #: printers for types specified by name
 _deferred_type_pprinters = {
 }
 
 def for_type(typ, func):
     """
     Add a pretty printer for a given type.
     """
     oldfunc = _type_pprinters.get(typ, None)
     if func is not None:
         # To support easy restoration of old pprinters, we need to ignore Nones.
         _type_pprinters[typ] = func
     return oldfunc
 
 def for_type_by_name(type_module, type_name, func):
     """
     Add a pretty printer for a type specified by the module and name of a type
     rather than the type object itself.
     """
     key = (type_module, type_name)
     oldfunc = _deferred_type_pprinters.get(key, None)
     if func is not None:
         # To support easy restoration of old pprinters, we need to ignore Nones.
         _deferred_type_pprinters[key] = func
     return oldfunc
 
 
 #: printers for the default singletons
 _singleton_pprinters = dict.fromkeys(map(id, [None, True, False, Ellipsis,
                                       NotImplemented]), _repr_pprint)
 
 
 if __name__ == '__main__':
     from random import randrange
     class Foo(object):
         def __init__(self):
             self.foo = 1
             self.bar = re.compile(r'\s+')
             self.blub = dict.fromkeys(range(30), randrange(1, 40))
             self.hehe = 23424.234234
             self.list = ["blub", "blah", self]
 
         def get_foo(self):
             print "foo"
 
     pprint(Foo(), verbose=True)