Replacing some .items() calls with .iteritems() for cleaner conversion with 2to3.

IPython/__init__.py

@@ -1,54 +1,54
 #!/usr/bin/env python
 # encoding: utf-8
 """
 IPython.
 
 IPython is a set of tools for interactive and exploratory computing in Python.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import absolute_import
 
 import os
 import sys
 
 #-----------------------------------------------------------------------------
 # Setup everything
 #-----------------------------------------------------------------------------
 
 if sys.version[0:3] < '2.6':
     raise ImportError('Python Version 2.6 or above is required for IPython.')
 
 
 # Make it easy to import extensions - they are always directly on pythonpath.
 # Therefore, non-IPython modules can be added to extensions directory.
 # This should probably be in ipapp.py.
 sys.path.append(os.path.join(os.path.dirname(__file__), "extensions"))
 
 #-----------------------------------------------------------------------------
 # Setup the top level names
 #-----------------------------------------------------------------------------
 
 from .config.loader import Config
 from .core import release
 from .core.application import Application
 from .frontend.terminal.embed import embed
 from .core.error import TryNext
 from .core.interactiveshell import InteractiveShell
 from .testing import test
 
 # Release data
 __author__ = ''
-for author, email in release.authors.values():
+for author, email in release.authors.itervalues():
     __author__ += author + ' <' + email + '>\n'
 __license__  = release.license
 __version__  = release.version
 __revision__ = release.revision

IPython/config/configurable.py

@@ -1,139 +1,139
 #!/usr/bin/env python
 # encoding: utf-8
 """
 A base class for objects that are configurable.
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from copy import deepcopy
 import datetime
 from weakref import WeakValueDictionary
 
 from IPython.utils.importstring import import_item
 from loader import Config
 from IPython.utils.traitlets import HasTraits, Instance
 
 
 #-----------------------------------------------------------------------------
 # Helper classes for Configurables
 #-----------------------------------------------------------------------------
 
 
 class ConfigurableError(Exception):
     pass
 
 
 #-----------------------------------------------------------------------------
 # Configurable implementation
 #-----------------------------------------------------------------------------
 
 
 class Configurable(HasTraits):
 
     config = Instance(Config,(),{})
     created = None
 
     def __init__(self, **kwargs):
         """Create a conigurable given a config config.
 
         Parameters
         ----------
         config : Config
             If this is empty, default values are used. If config is a 
             :class:`Config` instance, it will be used to configure the
             instance.
         
         Notes
         -----
         Subclasses of Configurable must call the :meth:`__init__` method of
         :class:`Configurable` *before* doing anything else and using 
         :func:`super`::
         
             class MyConfigurable(Configurable):
                 def __init__(self, config=None):
                     super(MyConfigurable, self).__init__(config)
                     # Then any other code you need to finish initialization.
 
         This ensures that instances will be configured properly.
         """
         config = kwargs.pop('config', None)
         if config is not None:
             # We used to deepcopy, but for now we are trying to just save
             # by reference.  This *could* have side effects as all components
             # will share config. In fact, I did find such a side effect in
             # _config_changed below. If a config attribute value was a mutable type
             # all instances of a component were getting the same copy, effectively
             # making that a class attribute.
             # self.config = deepcopy(config)
             self.config = config
         # This should go second so individual keyword arguments override 
         # the values in config.
         super(Configurable, self).__init__(**kwargs)
         self.created = datetime.datetime.now()
 
     #-------------------------------------------------------------------------
     # Static trait notifiations
     #-------------------------------------------------------------------------
 
     def _config_changed(self, name, old, new):
         """Update all the class traits having ``config=True`` as metadata.
 
         For any class trait with a ``config`` metadata attribute that is
         ``True``, we update the trait with the value of the corresponding
         config entry.
         """
         # Get all traits with a config metadata entry that is True
         traits = self.traits(config=True)
 
         # We auto-load config section for this class as well as any parent
         # classes that are Configurable subclasses.  This starts with Configurable
         # and works down the mro loading the config for each section.
         section_names = [cls.__name__ for cls in \
             reversed(self.__class__.__mro__) if 
             issubclass(cls, Configurable) and issubclass(self.__class__, cls)]
 
         for sname in section_names:
             # Don't do a blind getattr as that would cause the config to 
             # dynamically create the section with name self.__class__.__name__.
             if new._has_section(sname):
                 my_config = new[sname]
-                for k, v in traits.items():
+                for k, v in traits.iteritems():
                     # Don't allow traitlets with config=True to start with
                     # uppercase.  Otherwise, they are confused with Config
                     # subsections.  But, developers shouldn't have uppercase
                     # attributes anyways! (PEP 6)
                     if k[0].upper()==k[0] and not k.startswith('_'):
                         raise ConfigurableError('Configurable traitlets with '
                         'config=True must start with a lowercase so they are '
                         'not confused with Config subsections: %s.%s' % \
                         (self.__class__.__name__, k))
                     try:
                         # Here we grab the value from the config
                         # If k has the naming convention of a config
                         # section, it will be auto created.
                         config_value = my_config[k]
                     except KeyError:
                         pass
                     else:
                         # print "Setting %s.%s from %s.%s=%r" % \
                         #     (self.__class__.__name__,k,sname,k,config_value)
                         # We have to do a deepcopy here if we don't deepcopy the entire
                         # config object. If we don't, a mutable config_value will be
                         # shared by all instances, effectively making it a class attribute.
                         setattr(self, k, deepcopy(config_value))
 

IPython/config/loader.py

@@ -1,372 +1,373
+# -*- coding: utf-8 -*-
 # coding: utf-8
 """A simple configuration system.
 
 Authors
 -------
 * Brian Granger
 * Fernando Perez
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import __builtin__
 import os
 import sys
 
 from IPython.external import argparse
 from IPython.utils.path import filefind
 
 #-----------------------------------------------------------------------------
 # Exceptions
 #-----------------------------------------------------------------------------
 
 
 class ConfigError(Exception):
     pass
 
 
 class ConfigLoaderError(ConfigError):
     pass
 
 #-----------------------------------------------------------------------------
 # Argparse fix
 #-----------------------------------------------------------------------------
 
 # Unfortunately argparse by default prints help messages to stderr instead of
 # stdout.  This makes it annoying to capture long help screens at the command
 # line, since one must know how to pipe stderr, which many users don't know how
 # to do.  So we override the print_help method with one that defaults to
 # stdout and use our class instead.
 
 class ArgumentParser(argparse.ArgumentParser):
     """Simple argparse subclass that prints help to stdout by default."""
     
     def print_help(self, file=None):
         if file is None:
             file = sys.stdout
         return super(ArgumentParser, self).print_help(file)
     
     print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
     
 #-----------------------------------------------------------------------------
 # Config class for holding config information
 #-----------------------------------------------------------------------------
 
 
 class Config(dict):
     """An attribute based dict that can do smart merges."""
 
     def __init__(self, *args, **kwds):
         dict.__init__(self, *args, **kwds)
         # This sets self.__dict__ = self, but it has to be done this way
         # because we are also overriding __setattr__.
         dict.__setattr__(self, '__dict__', self)
 
     def _merge(self, other):
         to_update = {}
-        for k, v in other.items():
+        for k, v in other.iteritems():
             if not self.has_key(k):
                 to_update[k] = v
             else: # I have this key
                 if isinstance(v, Config):
                     # Recursively merge common sub Configs
                     self[k]._merge(v)
                 else:
                     # Plain updates for non-Configs
                     to_update[k] = v
 
         self.update(to_update)
 
     def _is_section_key(self, key):
         if key[0].upper()==key[0] and not key.startswith('_'):
             return True
         else:
             return False
 
     def __contains__(self, key):
         if self._is_section_key(key):
             return True
         else:
             return super(Config, self).__contains__(key)
     # .has_key is deprecated for dictionaries.
     has_key = __contains__
 
     def _has_section(self, key):
         if self._is_section_key(key):
             if super(Config, self).__contains__(key):
                 return True
         return False
 
     def copy(self):
         return type(self)(dict.copy(self))
 
     def __copy__(self):
         return self.copy()
 
     def __deepcopy__(self, memo):
         import copy
         return type(self)(copy.deepcopy(self.items()))
 
     def __getitem__(self, key):
         # Because we use this for an exec namespace, we need to delegate
         # the lookup of names in __builtin__ to itself.  This means
         # that you can't have section or attribute names that are 
         # builtins.
         try:
             return getattr(__builtin__, key)
         except AttributeError:
             pass
         if self._is_section_key(key):
             try:
                 return dict.__getitem__(self, key)
             except KeyError:
                 c = Config()
                 dict.__setitem__(self, key, c)
                 return c
         else:
             return dict.__getitem__(self, key)
 
     def __setitem__(self, key, value):
         # Don't allow names in __builtin__ to be modified.
         if hasattr(__builtin__, key):
             raise ConfigError('Config variable names cannot have the same name '
                               'as a Python builtin: %s' % key)
         if self._is_section_key(key):
             if not isinstance(value, Config):
                 raise ValueError('values whose keys begin with an uppercase '
                                  'char must be Config instances: %r, %r' % (key, value))
         else:
             dict.__setitem__(self, key, value)
 
     def __getattr__(self, key):
         try:
             return self.__getitem__(key)
         except KeyError, e:
             raise AttributeError(e)
 
     def __setattr__(self, key, value):
         try:
             self.__setitem__(key, value)
         except KeyError, e:
             raise AttributeError(e)
 
     def __delattr__(self, key):
         try:
             dict.__delitem__(self, key)
         except KeyError, e:
             raise AttributeError(e)
 
 
 #-----------------------------------------------------------------------------
 # Config loading classes
 #-----------------------------------------------------------------------------
 
 
 class ConfigLoader(object):
     """A object for loading configurations from just about anywhere.
     
     The resulting configuration is packaged as a :class:`Struct`.
     
     Notes
     -----
     A :class:`ConfigLoader` does one thing: load a config from a source 
     (file, command line arguments) and returns the data as a :class:`Struct`.
     There are lots of things that :class:`ConfigLoader` does not do.  It does
     not implement complex logic for finding config files.  It does not handle
     default values or merge multiple configs.  These things need to be 
     handled elsewhere.
     """
 
     def __init__(self):
         """A base class for config loaders.
         
         Examples
         --------
         
         >>> cl = ConfigLoader()
         >>> config = cl.load_config()
         >>> config
         {}
         """
         self.clear()
 
     def clear(self):
         self.config = Config()
 
     def load_config(self):
         """Load a config from somewhere, return a :class:`Config` instance.
         
         Usually, this will cause self.config to be set and then returned.
         However, in most cases, :meth:`ConfigLoader.clear` should be called
         to erase any previous state.
         """
         self.clear()
         return self.config
 
 
 class FileConfigLoader(ConfigLoader):
     """A base class for file based configurations.
 
     As we add more file based config loaders, the common logic should go
     here.
     """
     pass
 
 
 class PyFileConfigLoader(FileConfigLoader):
     """A config loader for pure python files.
     
     This calls execfile on a plain python file and looks for attributes
     that are all caps.  These attribute are added to the config Struct.
     """
 
     def __init__(self, filename, path=None):
         """Build a config loader for a filename and path.
 
         Parameters
         ----------
         filename : str
             The file name of the config file.
         path : str, list, tuple
             The path to search for the config file on, or a sequence of
             paths to try in order.
         """
         super(PyFileConfigLoader, self).__init__()
         self.filename = filename
         self.path = path
         self.full_filename = ''
         self.data = None
 
     def load_config(self):
         """Load the config from a file and return it as a Struct."""
         self.clear()
         self._find_file()
         self._read_file_as_dict()
         self._convert_to_config()
         return self.config
 
     def _find_file(self):
         """Try to find the file by searching the paths."""
         self.full_filename = filefind(self.filename, self.path)
 
     def _read_file_as_dict(self):
         """Load the config file into self.config, with recursive loading."""
         # This closure is made available in the namespace that is used
         # to exec the config file.  This allows users to call
         # load_subconfig('myconfig.py') to load config files recursively.
         # It needs to be a closure because it has references to self.path
         # and self.config.  The sub-config is loaded with the same path
         # as the parent, but it uses an empty config which is then merged
         # with the parents.
         def load_subconfig(fname):
             loader = PyFileConfigLoader(fname, self.path)
             try:
                 sub_config = loader.load_config()
             except IOError:
                 # Pass silently if the sub config is not there. This happens
                 # when a user us using a profile, but not the default config.
                 pass
             else:
                 self.config._merge(sub_config)
 
         # Again, this needs to be a closure and should be used in config
         # files to get the config being loaded.
         def get_config():
             return self.config
 
         namespace = dict(load_subconfig=load_subconfig, get_config=get_config)
         execfile(self.full_filename, namespace)
 
     def _convert_to_config(self):
         if self.data is None:
             ConfigLoaderError('self.data does not exist')
 
 
 class CommandLineConfigLoader(ConfigLoader):
     """A config loader for command line arguments.
 
     As we add more command line based loaders, the common logic should go
     here.
     """
 
 
 class ArgParseConfigLoader(CommandLineConfigLoader):
 
     def __init__(self, argv=None, *parser_args, **parser_kw):
         """Create a config loader for use with argparse.
 
         Parameters
         ----------
 
         argv : optional, list
           If given, used to read command-line arguments from, otherwise
           sys.argv[1:] is used.
 
         parser_args : tuple
           A tuple of positional arguments that will be passed to the
           constructor of :class:`argparse.ArgumentParser`.
 
         parser_kw : dict
           A tuple of keyword arguments that will be passed to the
           constructor of :class:`argparse.ArgumentParser`.
         """
         super(CommandLineConfigLoader, self).__init__()
         if argv == None:
             argv = sys.argv[1:]
         self.argv = argv
         self.parser_args = parser_args
         kwargs = dict(argument_default=argparse.SUPPRESS)
         kwargs.update(parser_kw)
         self.parser_kw = kwargs
 
     def load_config(self, args=None):
         """Parse command line arguments and return as a Struct.
 
         Parameters
         ----------
 
         args : optional, list
           If given, a list with the structure of sys.argv[1:] to parse
           arguments from. If not given, the instance's self.argv attribute
           (given at construction time) is used."""
         self.clear()
         if args is None:
             args = self.argv
         self._create_parser()
         self._parse_args(args)
         self._convert_to_config()
         return self.config
 
     def get_extra_args(self):
         if hasattr(self, 'extra_args'):
             return self.extra_args
         else:
             return []
 
     def _create_parser(self):
         self.parser = ArgumentParser(*self.parser_args, **self.parser_kw)
         self._add_arguments()
 
     def _add_arguments(self):
         raise NotImplementedError("subclasses must implement _add_arguments")
 
     def _parse_args(self, args):
         """self.parser->self.parsed_data""" 
         self.parsed_data, self.extra_args = self.parser.parse_known_args(args)
 
     def _convert_to_config(self):
         """self.parsed_data->self.config"""
-        for k, v in vars(self.parsed_data).items():
+        for k, v in vars(self.parsed_data).iteritems():
             exec_str = 'self.config.' + k + '= v'
             exec exec_str in locals(), globals()
 
 

IPython/core/completerlib.py

@@ -1,346 +1,346
 """Implementations for various useful completers.
 
 These are all loaded by default by IPython.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010 The IPython Development Team.
 #
 #  Distributed under the terms of the BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Stdlib imports
 import glob
 import inspect
 import os
 import re
 import shlex
 import sys
 
 # Third-party imports
 from time import time
 from zipimport import zipimporter
 
 # Our own imports
 from IPython.core.completer import expand_user, compress_user
 from IPython.core.error import TryNext
 
 # FIXME: this should be pulled in with the right call via the component system
 from IPython.core.ipapi import get as get_ipython
 
 #-----------------------------------------------------------------------------
 # Globals and constants
 #-----------------------------------------------------------------------------
 
 # Time in seconds after which the rootmodules will be stored permanently in the
 # ipython ip.db database (kept in the user's .ipython dir).
 TIMEOUT_STORAGE = 2
 
 # Time in seconds after which we give up
 TIMEOUT_GIVEUP = 20
 
 # Regular expression for the python import statement
 import_re = re.compile(r'.*(\.so|\.py[cod]?)$')
 
 # RE for the ipython %run command (python + ipython scripts)
 magic_run_re = re.compile(r'.*(\.ipy|\.py[w]?)$')
 
 #-----------------------------------------------------------------------------
 # Local utilities
 #-----------------------------------------------------------------------------
 
 def shlex_split(x):
     """Helper function to split lines into segments.
     """
     # shlex.split raises an exception if there is a syntax error in sh syntax
     # for example if no closing " is found. This function keeps dropping the
     # last character of the line until shlex.split does not raise
     # an exception. It adds end of the line to the result of shlex.split
     #
     # Example:
     # %run "c:/python -> ['%run','"c:/python']
 
     # shlex.split has unicode bugs, so encode first to str
     if isinstance(x, unicode):
         x = x.encode(sys.stdin.encoding)
 
     endofline = []
     while x != '':
         try:
             comps = shlex.split(x)
             if len(endofline) >= 1:
                 comps.append(''.join(endofline))
             return comps
         
         except ValueError:
             endofline = [x[-1:]]+endofline
             x = x[:-1]
 
     return [''.join(endofline)]
 
 def module_list(path):
     """
     Return the list containing the names of the modules available in the given
     folder.
     """
 
     if os.path.isdir(path):
         folder_list = os.listdir(path)
     elif path.endswith('.egg'):
         try:
             folder_list = [f for f in zipimporter(path)._files]
         except:
             folder_list = []
     else:
         folder_list = []
 
     if not folder_list:
         return []
 
     # A few local constants to be used in loops below
     isfile = os.path.isfile
     pjoin = os.path.join
     basename = os.path.basename
     
     # Now find actual path matches for packages or modules
     folder_list = [p for p in folder_list
                    if isfile(pjoin(path, p,'__init__.py'))
                    or import_re.match(p) ]
 
     return [basename(p).split('.')[0] for p in folder_list]
 
 def get_root_modules():
     """
     Returns a list containing the names of all the modules available in the
     folders of the pythonpath.
     """
     ip = get_ipython()
 
     if 'rootmodules' in ip.db:
         return ip.db['rootmodules']
     
     t = time()
     store = False
     modules = list(sys.builtin_module_names)
     for path in sys.path:
         modules += module_list(path)        
         if time() - t >= TIMEOUT_STORAGE and not store:
             store = True
             print("\nCaching the list of root modules, please wait!")
             print("(This will only be done once - type '%rehashx' to "
                   "reset cache!)\n")
             sys.stdout.flush()
         if time() - t > TIMEOUT_GIVEUP:
             print("This is taking too long, we give up.\n")
             ip.db['rootmodules'] = []
             return []
     
     modules = set(modules)
     if '__init__' in modules:
         modules.remove('__init__')
     modules = list(modules)
     if store:
         ip.db['rootmodules'] = modules
     return modules
 
 
 def is_importable(module, attr, only_modules):
     if only_modules:
         return inspect.ismodule(getattr(module, attr))
     else:
         return not(attr[:2] == '__' and attr[-2:] == '__')
 
 
 def try_import(mod, only_modules=False):
     try:
         m = __import__(mod)
     except:
         return []
     mods = mod.split('.')
     for module in mods[1:]:
         m = getattr(m, module)
 
     m_is_init = hasattr(m, '__file__') and '__init__' in m.__file__
 
     completions = []
     if (not hasattr(m, '__file__')) or (not only_modules) or m_is_init:
         completions.extend( [attr for attr in dir(m) if
                              is_importable(m, attr, only_modules)])
         
     completions.extend(getattr(m, '__all__', []))
     if m_is_init:
         completions.extend(module_list(os.path.dirname(m.__file__)))
     completions = set(completions)
     if '__init__' in completions:
         completions.remove('__init__')
     return list(completions)
 
 
 #-----------------------------------------------------------------------------
 # Completion-related functions.
 #-----------------------------------------------------------------------------
 
 def quick_completer(cmd, completions):
     """ Easily create a trivial completer for a command.
 
     Takes either a list of completions, or all completions in string (that will
     be split on whitespace).
     
     Example::
     
         [d:\ipython]|1> import ipy_completers
         [d:\ipython]|2> ipy_completers.quick_completer('foo', ['bar','baz'])
         [d:\ipython]|3> foo b<TAB>
         bar baz
         [d:\ipython]|3> foo ba
     """
     
     if isinstance(completions, basestring):
         completions = completions.split()
 
     def do_complete(self, event):
         return completions
     
     get_ipython().set_hook('complete_command',do_complete, str_key = cmd)
 
 
 def module_completion(line):
     """
     Returns a list containing the completion possibilities for an import line.
 
     The line looks like this :
     'import xml.d'
     'from xml.dom import'
     """
 
     words = line.split(' ')
     nwords = len(words)
 
     # from whatever <tab> -> 'import '
     if nwords == 3 and words[0] == 'from':
         return ['import ']
     
     # 'from xy<tab>' or 'import xy<tab>'
     if nwords < 3 and (words[0] in ['import','from']) :
         if nwords == 1:
             return get_root_modules()
         mod = words[1].split('.')
         if len(mod) < 2:
             return get_root_modules()
         completion_list = try_import('.'.join(mod[:-1]), True)
         return ['.'.join(mod[:-1] + [el]) for el in completion_list]
     
     # 'from xyz import abc<tab>'
     if nwords >= 3 and words[0] == 'from':
         mod = words[1]
         return try_import(mod)
 
 #-----------------------------------------------------------------------------
 # Completers
 #-----------------------------------------------------------------------------
 # These all have the func(self, event) signature to be used as custom
 # completers
 
 def module_completer(self,event):
     """Give completions after user has typed 'import ...' or 'from ...'"""
 
     # This works in all versions of python.  While 2.5 has
     # pkgutil.walk_packages(), that particular routine is fairly dangerous,
     # since it imports *EVERYTHING* on sys.path.  That is: a) very slow b) full
     # of possibly problematic side effects.
     # This search the folders in the sys.path for available modules.
 
     return module_completion(event.line)
 
 # FIXME: there's a lot of logic common to the run, cd and builtin file
 # completers, that is currently reimplemented in each.
 
 def magic_run_completer(self, event):
     """Complete files that end in .py or .ipy for the %run command.
     """
     comps = shlex_split(event.line)
     relpath = (len(comps) > 1 and comps[-1] or '').strip("'\"")
 
     #print("\nev=", event)  # dbg
     #print("rp=", relpath)  # dbg
     #print('comps=', comps)  # dbg
 
     lglob = glob.glob
     isdir = os.path.isdir
     relpath, tilde_expand, tilde_val = expand_user(relpath)
         
     dirs = [f.replace('\\','/') + "/" for f in lglob(relpath+'*') if isdir(f)]
 
     # Find if the user has already typed the first filename, after which we
     # should complete on all files, since after the first one other files may
     # be arguments to the input script.
 
     if filter(magic_run_re.match, comps):
         pys =  [f.replace('\\','/') for f in lglob('*')]
     else:
         pys =  [f.replace('\\','/')
                 for f in lglob(relpath+'*.py') + lglob(relpath+'*.ipy') +
                 lglob(relpath + '*.pyw')]
     #print('run comp:', dirs+pys) # dbg
     return [compress_user(p, tilde_expand, tilde_val) for p in dirs+pys]
 
 
 def cd_completer(self, event):
     """Completer function for cd, which only returns directories."""
     ip = get_ipython()
     relpath = event.symbol
 
     #print(event) # dbg
     if event.line.endswith('-b') or ' -b ' in event.line:
         # return only bookmark completions
         bkms = self.db.get('bookmarks', None)
         if bkms:
             return bkms.keys()
         else:
             return []
     
     if event.symbol == '-':
         width_dh = str(len(str(len(ip.user_ns['_dh']) + 1)))
         # jump in directory history by number
         fmt = '-%0' + width_dh +'d [%s]'
         ents = [ fmt % (i,s) for i,s in enumerate(ip.user_ns['_dh'])]
         if len(ents) > 1:
             return ents
         return []
 
     if event.symbol.startswith('--'):
         return ["--" + os.path.basename(d) for d in ip.user_ns['_dh']]
 
     # Expand ~ in path and normalize directory separators.
     relpath, tilde_expand, tilde_val = expand_user(relpath)
     relpath = relpath.replace('\\','/')
 
     found = []
     for d in [f.replace('\\','/') + '/' for f in glob.glob(relpath+'*')
               if os.path.isdir(f)]:
         if ' ' in d:
             # we don't want to deal with any of that, complex code
             # for this is elsewhere
             raise TryNext
         
         found.append(d)
 
     if not found:
         if os.path.isdir(relpath):
             return [compress_user(relpath, tilde_expand, tilde_val)]
 
         # if no completions so far, try bookmarks
-        bks = self.db.get('bookmarks',{}).keys()
+        bks = self.db.get('bookmarks',{}).iterkeys()
         bkmatches = [s for s in bks if s.startswith(event.symbol)]
         if bkmatches:
             return bkmatches
         
         raise TryNext
 
     return [compress_user(p, tilde_expand, tilde_val) for p in found]

IPython/core/history.py

@@ -1,283 +1,283
 # -*- coding: utf-8 -*-
 """ History related magics and functionality """
 
 # Stdlib imports
 import fnmatch
 import os
 
 import IPython.utils.io
 from IPython.utils.io import ask_yes_no
 from IPython.utils.warn import warn
 from IPython.core import ipapi
 
 def magic_history(self, parameter_s = ''):
     """Print input history (_i<n> variables), with most recent last.
     
     %history       -> print at most 40 inputs (some may be multi-line)\\
     %history n     -> print at most n inputs\\
     %history n1 n2 -> print inputs between n1 and n2 (n2 not included)\\
 
     By default, input history is printed without line numbers so it can be
     directly pasted into an editor.
 
     With -n, each input's number <n> is shown, and is accessible as the
     automatically generated variable _i<n> as well as In[<n>].  Multi-line
     statements are printed starting at a new line for easy copy/paste.
 
     Options:
 
       -n: print line numbers for each input.
       This feature is only available if numbered prompts are in use.
 
       -o: also print outputs for each input.
 
       -p: print classic '>>>' python prompts before each input.  This is useful
        for making documentation, and in conjunction with -o, for producing
        doctest-ready output.
 
       -r: (default) print the 'raw' history, i.e. the actual commands you typed.
       
       -t: print the 'translated' history, as IPython understands it.  IPython
       filters your input and converts it all into valid Python source before
       executing it (things like magics or aliases are turned into function
       calls, for example). With this option, you'll see the native history
       instead of the user-entered version: '%cd /' will be seen as
       'get_ipython().magic("%cd /")' instead of '%cd /'.
       
       -g: treat the arg as a pattern to grep for in (full) history.
       This includes the "shadow history" (almost all commands ever written).
       Use '%hist -g' to show full shadow history (may be very long).
       In shadow history, every index nuwber starts with 0.
 
       -f FILENAME: instead of printing the output to the screen, redirect it to
        the given file.  The file is always overwritten, though IPython asks for
        confirmation first if it already exists.
     """
 
     if not self.shell.displayhook.do_full_cache:
         print 'This feature is only available if numbered prompts are in use.'
         return
     opts,args = self.parse_options(parameter_s,'gnoptsrf:',mode='list')
 
     # Check if output to specific file was requested.
     try:
         outfname = opts['f']
     except KeyError:
         outfile = IPython.utils.io.Term.cout  # default
         # We don't want to close stdout at the end!
         close_at_end = False
     else:
         if os.path.exists(outfname):
             if not ask_yes_no("File %r exists. Overwrite?" % outfname): 
                 print 'Aborting.'
                 return
 
         outfile = open(outfname,'w')
         close_at_end = True
 
     if 't' in opts:
         input_hist = self.shell.input_hist
     elif 'r' in opts:
         input_hist = self.shell.input_hist_raw
     else:
         # Raw history is the default
         input_hist = self.shell.input_hist_raw
             
     default_length = 40
     pattern = None
     if 'g' in opts:
         init = 1
         final = len(input_hist)
         parts = parameter_s.split(None, 1)
         if len(parts) == 1:
             parts += '*'
         head, pattern = parts
         pattern = "*" + pattern + "*"
     elif len(args) == 0:
         final = len(input_hist)-1
         init = max(1,final-default_length)
     elif len(args) == 1:
         final = len(input_hist)
         init = max(1, final-int(args[0]))
     elif len(args) == 2:
         init, final = map(int, args)
     else:
         warn('%hist takes 0, 1 or 2 arguments separated by spaces.')
         print >> IPython.utils.io.Term.cout, self.magic_hist.__doc__
         return
     
     width = len(str(final))
     line_sep = ['','\n']
     print_nums = 'n' in opts
     print_outputs = 'o' in opts
     pyprompts = 'p' in opts
     
     found = False
     if pattern is not None:
         sh = self.shell.shadowhist.all()
         for idx, s in sh:
             if fnmatch.fnmatch(s, pattern):
                 print >> outfile, "0%d: %s" %(idx, s.expandtabs(4))
                 found = True
     
     if found:
         print >> outfile, "==="
         print >> outfile, \
               "shadow history ends, fetch by %rep <number> (must start with 0)"
         print >> outfile, "=== start of normal history ==="
         
     for in_num in range(init, final):
         # Print user history with tabs expanded to 4 spaces.  The GUI clients
         # use hard tabs for easier usability in auto-indented code, but we want
         # to produce PEP-8 compliant history for safe pasting into an editor.
         inline = input_hist[in_num].expandtabs(4)
 
         if pattern is not None and not fnmatch.fnmatch(inline, pattern):
             continue
             
         multiline = int(inline.count('\n') > 1)
         if print_nums:
             print >> outfile, \
                   '%s:%s' % (str(in_num).ljust(width), line_sep[multiline]),
         if pyprompts:
             print >> outfile, '>>>',
             if multiline:
                 lines = inline.splitlines()
                 print >> outfile, '\n... '.join(lines)
                 print >> outfile, '... '
             else:
                 print >> outfile, inline,
         else:
             print >> outfile, inline,
         if print_outputs:
             output = self.shell.output_hist.get(in_num)
             if output is not None:
                 print >> outfile, repr(output)
 
     if close_at_end:
         outfile.close()
 
 
 def magic_hist(self, parameter_s=''):
     """Alternate name for %history."""
     return self.magic_history(parameter_s)
 
 
 def rep_f(self, arg):
     r""" Repeat a command, or get command to input line for editing
 
     - %rep (no arguments):
     
     Place a string version of last computation result (stored in the special '_'
     variable) to the next input prompt. Allows you to create elaborate command
     lines without using copy-paste::
     
         $ l = ["hei", "vaan"]       
         $ "".join(l)        
         ==> heivaan        
         $ %rep        
         $ heivaan_ <== cursor blinking    
     
     %rep 45
     
     Place history line 45 to next input prompt. Use %hist to find out the
     number.
     
     %rep 1-4 6-7 3
     
     Repeat the specified lines immediately. Input slice syntax is the same as
     in %macro and %save.
     
     %rep foo
     
     Place the most recent line that has the substring "foo" to next input.
     (e.g. 'svn ci -m foobar').    
     """
     
     opts,args = self.parse_options(arg,'',mode='list')
     if not args:
         self.set_next_input(str(self.shell.user_ns["_"]))
         return
 
     if len(args) == 1 and not '-' in args[0]:
         arg = args[0]
         if len(arg) > 1 and arg.startswith('0'):
             # get from shadow hist
             num = int(arg[1:])
             line = self.shell.shadowhist.get(num)
             self.set_next_input(str(line))
             return
         try:
             num = int(args[0])
             self.set_next_input(str(self.shell.input_hist_raw[num]).rstrip())
             return
         except ValueError:
             pass
         
         for h in reversed(self.shell.input_hist_raw):
             if 'rep' in h:
                 continue
             if fnmatch.fnmatch(h,'*' + arg + '*'):
                 self.set_next_input(str(h).rstrip())
                 return
         
     try:
         lines = self.extract_input_slices(args, True)
         print "lines",lines
         self.runlines(lines)
     except ValueError:
         print "Not found in recent history:", args
         
 
 _sentinel = object()
 
 class ShadowHist(object):
     def __init__(self, db):
         # cmd => idx mapping
         self.curidx = 0
         self.db = db
         self.disabled = False
     
     def inc_idx(self):
         idx = self.db.get('shadowhist_idx', 1)
         self.db['shadowhist_idx'] = idx + 1
         return idx
         
     def add(self, ent):
         if self.disabled:
             return
         try:
             old = self.db.hget('shadowhist', ent, _sentinel)
             if old is not _sentinel:
                 return
             newidx = self.inc_idx()
             #print "new",newidx # dbg
             self.db.hset('shadowhist',ent, newidx)
         except:
             ipapi.get().showtraceback()
             print "WARNING: disabling shadow history"
             self.disabled = True
     
     def all(self):
         d = self.db.hdict('shadowhist')
-        items = [(i,s) for (s,i) in d.items()]
+        items = [(i,s) for (s,i) in d.iteritems()]
         items.sort()
         return items
 
     def get(self, idx):
         all = self.all()
         
         for k, v in all:
             #print k,v
             if k == idx:
                 return v
 
 
 def init_ipython(ip):
     ip.define_magic("rep",rep_f)        
     ip.define_magic("hist",magic_hist)            
     ip.define_magic("history",magic_history)
 
     # XXX - ipy_completers are in quarantine, need to be updated to new apis
     #import ipy_completers
     #ipy_completers.quick_completer('%hist' ,'-g -t -r -n')

IPython/core/interactiveshell.py

@@ -1,2584 +1,2584
 # -*- coding: utf-8 -*-
 """Main IPython class."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001 Janko Hauser <[e-mail]>
 #  Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #  Copyright (C) 2008-2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
 from __future__ import absolute_import
 
 import __builtin__
 import __future__
 import abc
 import atexit
 import codeop
 import os
 import re
 import string
 import sys
 import tempfile
 import types
 from contextlib import nested
 
 from IPython.config.configurable import Configurable
 from IPython.core import debugger, oinspect
 from IPython.core import history as ipcorehist
 from IPython.core import page
 from IPython.core import prefilter
 from IPython.core import shadowns
 from IPython.core import ultratb
 from IPython.core.alias import AliasManager
 from IPython.core.builtin_trap import BuiltinTrap
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.displayhook import DisplayHook
 from IPython.core.error import TryNext, UsageError
 from IPython.core.extensions import ExtensionManager
 from IPython.core.fakemodule import FakeModule, init_fakemod_dict
 from IPython.core.inputlist import InputList
 from IPython.core.inputsplitter import IPythonInputSplitter
 from IPython.core.logger import Logger
 from IPython.core.magic import Magic
 from IPython.core.payload import PayloadManager
 from IPython.core.plugin import PluginManager
 from IPython.core.prefilter import PrefilterManager, ESC_MAGIC
 from IPython.external.Itpl import ItplNS
 from IPython.utils import PyColorize
 from IPython.utils import io
 from IPython.utils import pickleshare
 from IPython.utils.doctestreload import doctest_reload
 from IPython.utils.io import ask_yes_no, rprint
 from IPython.utils.ipstruct import Struct
 from IPython.utils.path import get_home_dir, get_ipython_dir, HomeDirError
 from IPython.utils.process import system, getoutput
 from IPython.utils.strdispatch import StrDispatch
 from IPython.utils.syspathcontext import prepended_to_syspath
 from IPython.utils.text import num_ini_spaces, format_screen, LSString, SList
 from IPython.utils.traitlets import (Int, Str, CBool, CaselessStrEnum, Enum,
                                      List, Unicode, Instance, Type)
 from IPython.utils.warn import warn, error, fatal
 import IPython.core.hooks
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # compiled regexps for autoindent management
 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
 
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
 # store the builtin raw_input globally, and use this always, in case user code
 # overwrites it (like wx.py.PyShell does)
 raw_input_original = raw_input
 
 def softspace(file, newvalue):
     """Copied from code.py, to remove the dependency"""
 
     oldvalue = 0
     try:
         oldvalue = file.softspace
     except AttributeError:
         pass
     try:
         file.softspace = newvalue
     except (AttributeError, TypeError):
         # "attribute-less object" or "read-only attributes"
         pass
     return oldvalue
 
 
 def no_op(*a, **kw): pass
 
 class SpaceInInput(Exception): pass
 
 class Bunch: pass
 
 
 def get_default_colors():
     if sys.platform=='darwin':
         return "LightBG"
     elif os.name=='nt':
         return 'Linux'
     else:
         return 'Linux'
 
 
 class SeparateStr(Str):
     """A Str subclass to validate separate_in, separate_out, etc.
 
     This is a Str based trait that converts '0'->'' and '\\n'->'\n'.
     """
 
     def validate(self, obj, value):
         if value == '0': value = ''
         value = value.replace('\\n','\n')
         return super(SeparateStr, self).validate(obj, value)
 
 class MultipleInstanceError(Exception):
     pass
 
 
 #-----------------------------------------------------------------------------
 # Main IPython class
 #-----------------------------------------------------------------------------
 
 
 class InteractiveShell(Configurable, Magic):
     """An enhanced, interactive shell for Python."""
 
     _instance = None
     autocall = Enum((0,1,2), default_value=1, config=True)
     # TODO: remove all autoindent logic and put into frontends.
     # We can't do this yet because even runlines uses the autoindent.
     autoindent = CBool(True, config=True)
     automagic = CBool(True, config=True)
     cache_size = Int(1000, config=True)
     color_info = CBool(True, config=True)
     colors = CaselessStrEnum(('NoColor','LightBG','Linux'), 
                              default_value=get_default_colors(), config=True)
     debug = CBool(False, config=True)
     deep_reload = CBool(False, config=True)
     displayhook_class = Type(DisplayHook)
     exit_now = CBool(False)
     filename = Str("<ipython console>")
     ipython_dir= Unicode('', config=True) # Set to get_ipython_dir() in __init__
 
     # Input splitter, to split entire cells of input into either individual
     # interactive statements or whole blocks.
     input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
                               (), {})
     logstart = CBool(False, config=True)
     logfile = Str('', config=True)
     logappend = Str('', config=True)
     object_info_string_level = Enum((0,1,2), default_value=0,
                                     config=True)
     pdb = CBool(False, config=True)
 
     pprint = CBool(True, config=True)
     profile = Str('', config=True)
     prompt_in1 = Str('In [\\#]: ', config=True)
     prompt_in2 = Str('   .\\D.: ', config=True)
     prompt_out = Str('Out[\\#]: ', config=True)
     prompts_pad_left = CBool(True, config=True)
     quiet = CBool(False, config=True)
 
     # The readline stuff will eventually be moved to the terminal subclass
     # but for now, we can't do that as readline is welded in everywhere.
     readline_use = CBool(True, config=True)
     readline_merge_completions = CBool(True, config=True)
     readline_omit__names = Enum((0,1,2), default_value=0, config=True)
     readline_remove_delims = Str('-/~', config=True)
     readline_parse_and_bind = List([
             'tab: complete',
             '"\C-l": clear-screen',
             'set show-all-if-ambiguous on',
             '"\C-o": tab-insert',
             '"\M-i": "    "',
             '"\M-o": "\d\d\d\d"',
             '"\M-I": "\d\d\d\d"',
             '"\C-r": reverse-search-history',
             '"\C-s": forward-search-history',
             '"\C-p": history-search-backward',
             '"\C-n": history-search-forward',
             '"\e[A": history-search-backward',
             '"\e[B": history-search-forward',
             '"\C-k": kill-line',
             '"\C-u": unix-line-discard',
         ], allow_none=False, config=True)
 
     # TODO: this part of prompt management should be moved to the frontends.
     # Use custom TraitTypes that convert '0'->'' and '\\n'->'\n'
     separate_in = SeparateStr('\n', config=True)
     separate_out = SeparateStr('', config=True)
     separate_out2 = SeparateStr('', config=True)
     wildcards_case_sensitive = CBool(True, config=True)
     xmode = CaselessStrEnum(('Context','Plain', 'Verbose'), 
                             default_value='Context', config=True)
 
     # Subcomponents of InteractiveShell
     alias_manager = Instance('IPython.core.alias.AliasManager')
     prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager')
     builtin_trap = Instance('IPython.core.builtin_trap.BuiltinTrap')
     display_trap = Instance('IPython.core.display_trap.DisplayTrap')
     extension_manager = Instance('IPython.core.extensions.ExtensionManager')
     plugin_manager = Instance('IPython.core.plugin.PluginManager')
     payload_manager = Instance('IPython.core.payload.PayloadManager')
 
     # Private interface
     _post_execute = set()
 
     def __init__(self, config=None, ipython_dir=None,
                  user_ns=None, user_global_ns=None,
                  custom_exceptions=((), None)):
 
         # This is where traits with a config_key argument are updated
         # from the values on config.
         super(InteractiveShell, self).__init__(config=config)
 
         # These are relatively independent and stateless
         self.init_ipython_dir(ipython_dir)
         self.init_instance_attrs()
         self.init_environment()
 
         # Create namespaces (user_ns, user_global_ns, etc.)
         self.init_create_namespaces(user_ns, user_global_ns)
         # This has to be done after init_create_namespaces because it uses
         # something in self.user_ns, but before init_sys_modules, which
         # is the first thing to modify sys.
         # TODO: When we override sys.stdout and sys.stderr before this class
         # is created, we are saving the overridden ones here. Not sure if this
         # is what we want to do.
         self.save_sys_module_state()
         self.init_sys_modules()
 
         self.init_history()
         self.init_encoding()
         self.init_prefilter()
 
         Magic.__init__(self, self)
 
         self.init_syntax_highlighting()
         self.init_hooks()
         self.init_pushd_popd_magic()
         # self.init_traceback_handlers use to be here, but we moved it below
         # because it and init_io have to come after init_readline.
         self.init_user_ns()
         self.init_logger()
         self.init_alias()
         self.init_builtins()
 
         # pre_config_initialization
         self.init_shadow_hist()
 
         # The next section should contain everything that was in ipmaker.
         self.init_logstart()
 
         # The following was in post_config_initialization
         self.init_inspector()
         # init_readline() must come before init_io(), because init_io uses
         # readline related things.
         self.init_readline()
         # init_completer must come after init_readline, because it needs to
         # know whether readline is present or not system-wide to configure the
         # completers, since the completion machinery can now operate
         # independently of readline (e.g. over the network)
         self.init_completer()
         # TODO: init_io() needs to happen before init_traceback handlers
         # because the traceback handlers hardcode the stdout/stderr streams.
         # This logic in in debugger.Pdb and should eventually be changed.
         self.init_io()
         self.init_traceback_handlers(custom_exceptions)
         self.init_prompts()
         self.init_displayhook()
         self.init_reload_doctest()
         self.init_magics()
         self.init_pdb()
         self.init_extension_manager()
         self.init_plugin_manager()
         self.init_payload()
         self.hooks.late_startup_hook()
         atexit.register(self.atexit_operations)
 
     @classmethod
     def instance(cls, *args, **kwargs):
         """Returns a global InteractiveShell instance."""
         if cls._instance is None:
             inst = cls(*args, **kwargs)
             # Now make sure that the instance will also be returned by
             # the subclasses instance attribute.
             for subclass in cls.mro():
                 if issubclass(cls, subclass) and \
                        issubclass(subclass, InteractiveShell):
                     subclass._instance = inst
                 else:
                     break
         if isinstance(cls._instance, cls):
             return cls._instance
         else:
             raise MultipleInstanceError(
                 'Multiple incompatible subclass instances of '
                 'InteractiveShell are being created.'
             )
 
     @classmethod
     def initialized(cls):
         return hasattr(cls, "_instance")
 
     def get_ipython(self):
         """Return the currently running IPython instance."""
         return self
 
     #-------------------------------------------------------------------------
     # Trait changed handlers
     #-------------------------------------------------------------------------
 
     def _ipython_dir_changed(self, name, new):
         if not os.path.isdir(new):
             os.makedirs(new, mode = 0777)
 
     def set_autoindent(self,value=None):
         """Set the autoindent flag, checking for readline support.
 
         If called with no arguments, it acts as a toggle."""
 
         if not self.has_readline:
             if os.name == 'posix':
                 warn("The auto-indent feature requires the readline library")
             self.autoindent = 0
             return
         if value is None:
             self.autoindent = not self.autoindent
         else:
             self.autoindent = value
 
     #-------------------------------------------------------------------------
     # init_* methods called by __init__
     #-------------------------------------------------------------------------
 
     def init_ipython_dir(self, ipython_dir):
         if ipython_dir is not None:
             self.ipython_dir = ipython_dir
             self.config.Global.ipython_dir = self.ipython_dir
             return
 
         if hasattr(self.config.Global, 'ipython_dir'):
             self.ipython_dir = self.config.Global.ipython_dir
         else:
             self.ipython_dir = get_ipython_dir()
 
         # All children can just read this
         self.config.Global.ipython_dir = self.ipython_dir
 
     def init_instance_attrs(self):
         self.more = False
 
         # command compiler
         self.compile = codeop.CommandCompiler()
 
         # User input buffer
         self.buffer = []
 
         # Make an empty namespace, which extension writers can rely on both
         # existing and NEVER being used by ipython itself.  This gives them a
         # convenient location for storing additional information and state
         # their extensions may require, without fear of collisions with other
         # ipython names that may develop later.
         self.meta = Struct()
 
         # Object variable to store code object waiting execution.  This is
         # used mainly by the multithreaded shells, but it can come in handy in
         # other situations.  No need to use a Queue here, since it's a single
         # item which gets cleared once run.
         self.code_to_run = None
 
         # Temporary files used for various purposes.  Deleted at exit.
         self.tempfiles = []
 
         # Keep track of readline usage (later set by init_readline)
         self.has_readline = False
 
         # keep track of where we started running (mainly for crash post-mortem)
         # This is not being used anywhere currently.
         self.starting_dir = os.getcwd()
 
         # Indentation management
         self.indent_current_nsp = 0
 
     def init_environment(self):
         """Any changes we need to make to the user's environment."""
         pass
 
     def init_encoding(self):
         # Get system encoding at startup time.  Certain terminals (like Emacs
         # under Win32 have it set to None, and we need to have a known valid
         # encoding to use in the raw_input() method
         try:
             self.stdin_encoding = sys.stdin.encoding or 'ascii'
         except AttributeError:
             self.stdin_encoding = 'ascii'
 
     def init_syntax_highlighting(self):
         # Python source parser/formatter for syntax highlighting
         pyformat = PyColorize.Parser().format
         self.pycolorize = lambda src: pyformat(src,'str',self.colors)
 
     def init_pushd_popd_magic(self):
         # for pushd/popd management
         try:
             self.home_dir = get_home_dir()
         except HomeDirError, msg:
             fatal(msg)
 
         self.dir_stack = []
 
     def init_logger(self):
         self.logger = Logger(self, logfname='ipython_log.py', logmode='rotate')
         # local shortcut, this is used a LOT
         self.log = self.logger.log
 
     def init_logstart(self):
         if self.logappend:
             self.magic_logstart(self.logappend + ' append')
         elif self.logfile:
             self.magic_logstart(self.logfile)
         elif self.logstart:
             self.magic_logstart()
 
     def init_builtins(self):
         self.builtin_trap = BuiltinTrap(shell=self)
 
     def init_inspector(self):
         # Object inspector
         self.inspector = oinspect.Inspector(oinspect.InspectColors,
                                             PyColorize.ANSICodeColors,
                                             'NoColor',
                                             self.object_info_string_level)
 
     def init_io(self):
         # This will just use sys.stdout and sys.stderr. If you want to
         # override sys.stdout and sys.stderr themselves, you need to do that
         # *before* instantiating this class, because Term holds onto 
         # references to the underlying streams.
         if sys.platform == 'win32' and self.has_readline:
             Term = io.IOTerm(cout=self.readline._outputfile,
                              cerr=self.readline._outputfile)
         else:
             Term = io.IOTerm()
         io.Term = Term
 
     def init_prompts(self):
         # TODO: This is a pass for now because the prompts are managed inside
         # the DisplayHook. Once there is a separate prompt manager, this 
         # will initialize that object and all prompt related information.
         pass
 
     def init_displayhook(self):
         # Initialize displayhook, set in/out prompts and printing system
         self.displayhook = self.displayhook_class(
             shell=self,
             cache_size=self.cache_size,
             input_sep = self.separate_in,
             output_sep = self.separate_out,
             output_sep2 = self.separate_out2,
             ps1 = self.prompt_in1,
             ps2 = self.prompt_in2,
             ps_out = self.prompt_out,
             pad_left = self.prompts_pad_left
         )
         # This is a context manager that installs/revmoes the displayhook at
         # the appropriate time.
         self.display_trap = DisplayTrap(hook=self.displayhook)
 
     def init_reload_doctest(self):
         # Do a proper resetting of doctest, including the necessary displayhook
         # monkeypatching
         try:
             doctest_reload()
         except ImportError:
             warn("doctest module does not exist.")
 
     #-------------------------------------------------------------------------
     # Things related to injections into the sys module
     #-------------------------------------------------------------------------
 
     def save_sys_module_state(self):
         """Save the state of hooks in the sys module.
 
         This has to be called after self.user_ns is created.
         """
         self._orig_sys_module_state = {}
         self._orig_sys_module_state['stdin'] = sys.stdin
         self._orig_sys_module_state['stdout'] = sys.stdout
         self._orig_sys_module_state['stderr'] = sys.stderr
         self._orig_sys_module_state['excepthook'] = sys.excepthook
         try:
             self._orig_sys_modules_main_name = self.user_ns['__name__']
         except KeyError:
             pass
 
     def restore_sys_module_state(self):
         """Restore the state of the sys module."""
         try:
-            for k, v in self._orig_sys_module_state.items():
+            for k, v in self._orig_sys_module_state.iteritems():
                 setattr(sys, k, v)
         except AttributeError:
             pass
         # Reset what what done in self.init_sys_modules
         try:
             sys.modules[self.user_ns['__name__']] = self._orig_sys_modules_main_name
         except (AttributeError, KeyError):
             pass
 
     #-------------------------------------------------------------------------
     # Things related to hooks
     #-------------------------------------------------------------------------
 
     def init_hooks(self):
         # hooks holds pointers used for user-side customizations
         self.hooks = Struct()
 
         self.strdispatchers = {}
 
         # Set all default hooks, defined in the IPython.hooks module.
         hooks = IPython.core.hooks
         for hook_name in hooks.__all__:
             # default hooks have priority 100, i.e. low; user hooks should have
             # 0-100 priority
             self.set_hook(hook_name,getattr(hooks,hook_name), 100)
 
     def set_hook(self,name,hook, priority = 50, str_key = None, re_key = None):
         """set_hook(name,hook) -> sets an internal IPython hook.
 
         IPython exposes some of its internal API as user-modifiable hooks.  By
         adding your function to one of these hooks, you can modify IPython's 
         behavior to call at runtime your own routines."""
 
         # At some point in the future, this should validate the hook before it
         # accepts it.  Probably at least check that the hook takes the number
         # of args it's supposed to.
         
         f = types.MethodType(hook, self)
 
         # check if the hook is for strdispatcher first
         if str_key is not None:
             sdp = self.strdispatchers.get(name, StrDispatch())
             sdp.add_s(str_key, f, priority )
             self.strdispatchers[name] = sdp
             return
         if re_key is not None:
             sdp = self.strdispatchers.get(name, StrDispatch())
             sdp.add_re(re.compile(re_key), f, priority )
             self.strdispatchers[name] = sdp
             return
             
         dp = getattr(self.hooks, name, None)
         if name not in IPython.core.hooks.__all__:
             print "Warning! Hook '%s' is not one of %s" % \
                   (name, IPython.core.hooks.__all__ )
         if not dp:
             dp = IPython.core.hooks.CommandChainDispatcher()
         
         try:
             dp.add(f,priority)
         except AttributeError:
             # it was not commandchain, plain old func - replace
             dp = f
 
         setattr(self.hooks,name, dp)
 
     def register_post_execute(self, func):
         """Register a function for calling after code execution.
         """
         if not callable(func):
             raise ValueError('argument %s must be callable' % func)
         self._post_execute.add(func)
 
     #-------------------------------------------------------------------------
     # Things related to the "main" module
     #-------------------------------------------------------------------------
 
     def new_main_mod(self,ns=None):
         """Return a new 'main' module object for user code execution.
         """
         main_mod = self._user_main_module
         init_fakemod_dict(main_mod,ns)
         return main_mod
 
     def cache_main_mod(self,ns,fname):
         """Cache a main module's namespace.
 
         When scripts are executed via %run, we must keep a reference to the
         namespace of their __main__ module (a FakeModule instance) around so
         that Python doesn't clear it, rendering objects defined therein
         useless.
 
         This method keeps said reference in a private dict, keyed by the
         absolute path of the module object (which corresponds to the script
         path).  This way, for multiple executions of the same script we only
         keep one copy of the namespace (the last one), thus preventing memory
         leaks from old references while allowing the objects from the last
         execution to be accessible.
 
         Note: we can not allow the actual FakeModule instances to be deleted,
         because of how Python tears down modules (it hard-sets all their
         references to None without regard for reference counts).  This method
         must therefore make a *copy* of the given namespace, to allow the
         original module's __dict__ to be cleared and reused.
 
         
         Parameters
         ----------
           ns : a namespace (a dict, typically)
 
           fname : str
             Filename associated with the namespace.
 
         Examples
         --------
 
         In [10]: import IPython
 
         In [11]: _ip.cache_main_mod(IPython.__dict__,IPython.__file__)
 
         In [12]: IPython.__file__ in _ip._main_ns_cache
         Out[12]: True
         """
         self._main_ns_cache[os.path.abspath(fname)] = ns.copy()
 
     def clear_main_mod_cache(self):
         """Clear the cache of main modules.
 
         Mainly for use by utilities like %reset.
 
         Examples
         --------
 
         In [15]: import IPython
 
         In [16]: _ip.cache_main_mod(IPython.__dict__,IPython.__file__)
 
         In [17]: len(_ip._main_ns_cache) > 0
         Out[17]: True
 
         In [18]: _ip.clear_main_mod_cache()
 
         In [19]: len(_ip._main_ns_cache) == 0
         Out[19]: True
         """
         self._main_ns_cache.clear()
 
     #-------------------------------------------------------------------------
     # Things related to debugging
     #-------------------------------------------------------------------------
 
     def init_pdb(self):
         # Set calling of pdb on exceptions
         # self.call_pdb is a property
         self.call_pdb = self.pdb
 
     def _get_call_pdb(self):
         return self._call_pdb
 
     def _set_call_pdb(self,val):
 
         if val not in (0,1,False,True):
             raise ValueError,'new call_pdb value must be boolean'
 
         # store value in instance
         self._call_pdb = val
 
         # notify the actual exception handlers
         self.InteractiveTB.call_pdb = val
 
     call_pdb = property(_get_call_pdb,_set_call_pdb,None,
                         'Control auto-activation of pdb at exceptions')
 
     def debugger(self,force=False):
         """Call the pydb/pdb debugger.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
           flag and does not actually invoke the debugger if the flag is false.
           The 'force' option forces the debugger to activate even if the flag
           is false.
         """
 
         if not (force or self.call_pdb):
             return
 
         if not hasattr(sys,'last_traceback'):
             error('No traceback has been produced, nothing to debug.')
             return
 
         # use pydb if available
         if debugger.has_pydb:
             from pydb import pm
         else:
             # fallback to our internal debugger
             pm = lambda : self.InteractiveTB.debugger(force=True)
         self.history_saving_wrapper(pm)()
 
     #-------------------------------------------------------------------------
     # Things related to IPython's various namespaces
     #-------------------------------------------------------------------------
 
     def init_create_namespaces(self, user_ns=None, user_global_ns=None):
         # Create the namespace where the user will operate.  user_ns is
         # normally the only one used, and it is passed to the exec calls as
         # the locals argument.  But we do carry a user_global_ns namespace
         # given as the exec 'globals' argument,  This is useful in embedding
         # situations where the ipython shell opens in a context where the
         # distinction between locals and globals is meaningful.  For
         # non-embedded contexts, it is just the same object as the user_ns dict.
 
         # FIXME. For some strange reason, __builtins__ is showing up at user
         # level as a dict instead of a module. This is a manual fix, but I
         # should really track down where the problem is coming from. Alex
         # Schmolck reported this problem first.
 
         # A useful post by Alex Martelli on this topic:
         # Re: inconsistent value from __builtins__
         # Von: Alex Martelli <[e-mail]>
         # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
         # Gruppen: comp.lang.python
 
         # Michael Hohn <[e-mail]> wrote:
         # > >>> print type(builtin_check.get_global_binding('__builtins__'))
         # > <type 'dict'>
         # > >>> print type(__builtins__)
         # > <type 'module'>
         # > Is this difference in return value intentional?
 
         # Well, it's documented that '__builtins__' can be either a dictionary
         # or a module, and it's been that way for a long time. Whether it's
         # intentional (or sensible), I don't know. In any case, the idea is
         # that if you need to access the built-in namespace directly, you
         # should start with "import __builtin__" (note, no 's') which will
         # definitely give you a module. Yeah, it's somewhat confusing:-(.
 
         # These routines return properly built dicts as needed by the rest of
         # the code, and can also be used by extension writers to generate
         # properly initialized namespaces.
         user_ns, user_global_ns = self.make_user_namespaces(user_ns,
                                                             user_global_ns)
 
         # Assign namespaces
         # This is the namespace where all normal user variables live
         self.user_ns = user_ns
         self.user_global_ns = user_global_ns
 
         # An auxiliary namespace that checks what parts of the user_ns were
         # loaded at startup, so we can list later only variables defined in
         # actual interactive use.  Since it is always a subset of user_ns, it
         # doesn't need to be separately tracked in the ns_table.
         self.user_ns_hidden = {}
 
         # A namespace to keep track of internal data structures to prevent
         # them from cluttering user-visible stuff.  Will be updated later
         self.internal_ns = {}
 
         # Now that FakeModule produces a real module, we've run into a nasty
         # problem: after script execution (via %run), the module where the user
         # code ran is deleted.  Now that this object is a true module (needed
         # so docetst and other tools work correctly), the Python module
         # teardown mechanism runs over it, and sets to None every variable
         # present in that module.  Top-level references to objects from the
         # script survive, because the user_ns is updated with them.  However,
         # calling functions defined in the script that use other things from
         # the script will fail, because the function's closure had references
         # to the original objects, which are now all None.  So we must protect
         # these modules from deletion by keeping a cache.
         # 
         # To avoid keeping stale modules around (we only need the one from the
         # last run), we use a dict keyed with the full path to the script, so
         # only the last version of the module is held in the cache.  Note,
         # however, that we must cache the module *namespace contents* (their
         # __dict__).  Because if we try to cache the actual modules, old ones
         # (uncached) could be destroyed while still holding references (such as
         # those held by GUI objects that tend to be long-lived)>
         # 
         # The %reset command will flush this cache.  See the cache_main_mod()
         # and clear_main_mod_cache() methods for details on use.
 
         # This is the cache used for 'main' namespaces
         self._main_ns_cache = {}
         # And this is the single instance of FakeModule whose __dict__ we keep
         # copying and clearing for reuse on each %run
         self._user_main_module = FakeModule()
 
         # A table holding all the namespaces IPython deals with, so that
         # introspection facilities can search easily.
         self.ns_table = {'user':user_ns,
                          'user_global':user_global_ns,
                          'internal':self.internal_ns,
                          'builtin':__builtin__.__dict__
                          }
 
         # Similarly, track all namespaces where references can be held and that
         # we can safely clear (so it can NOT include builtin).  This one can be
         # a simple list.  Note that the main execution namespaces, user_ns and
         # user_global_ns, can NOT be listed here, as clearing them blindly
         # causes errors in object __del__ methods.  Instead, the reset() method
         # clears them manually and carefully.
         self.ns_refs_table = [ self.user_ns_hidden,
                                self.internal_ns, self._main_ns_cache ]
 
     def make_user_namespaces(self, user_ns=None, user_global_ns=None):
         """Return a valid local and global user interactive namespaces.
 
         This builds a dict with the minimal information needed to operate as a
         valid IPython user namespace, which you can pass to the various
         embedding classes in ipython. The default implementation returns the
         same dict for both the locals and the globals to allow functions to
         refer to variables in the namespace. Customized implementations can
         return different dicts. The locals dictionary can actually be anything
         following the basic mapping protocol of a dict, but the globals dict
         must be a true dict, not even a subclass. It is recommended that any
         custom object for the locals namespace synchronize with the globals
         dict somehow.
 
         Raises TypeError if the provided globals namespace is not a true dict.
 
         Parameters
         ----------
         user_ns : dict-like, optional
             The current user namespace. The items in this namespace should
             be included in the output. If None, an appropriate blank
             namespace should be created.
         user_global_ns : dict, optional
             The current user global namespace. The items in this namespace
             should be included in the output. If None, an appropriate
             blank namespace should be created.
 
         Returns
         -------
             A pair of dictionary-like object to be used as the local namespace
             of the interpreter and a dict to be used as the global namespace.
         """
 
 
         # We must ensure that __builtin__ (without the final 's') is always
         # available and pointing to the __builtin__ *module*.  For more details:
         # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
 
         if user_ns is None:
             # Set __name__ to __main__ to better match the behavior of the
             # normal interpreter.
             user_ns = {'__name__'     :'__main__',
                        '__builtin__' : __builtin__,
                        '__builtins__' : __builtin__,
                       }
         else:
             user_ns.setdefault('__name__','__main__')
             user_ns.setdefault('__builtin__',__builtin__)
             user_ns.setdefault('__builtins__',__builtin__)
 
         if user_global_ns is None:
             user_global_ns = user_ns
         if type(user_global_ns) is not dict:
             raise TypeError("user_global_ns must be a true dict; got %r"
                 % type(user_global_ns))
 
         return user_ns, user_global_ns
 
     def init_sys_modules(self):
         # We need to insert into sys.modules something that looks like a
         # module but which accesses the IPython namespace, for shelve and
         # pickle to work interactively. Normally they rely on getting
         # everything out of __main__, but for embedding purposes each IPython
         # instance has its own private namespace, so we can't go shoving
         # everything into __main__.
 
         # note, however, that we should only do this for non-embedded
         # ipythons, which really mimic the __main__.__dict__ with their own
         # namespace.  Embedded instances, on the other hand, should not do
         # this because they need to manage the user local/global namespaces
         # only, but they live within a 'normal' __main__ (meaning, they
         # shouldn't overtake the execution environment of the script they're
         # embedded in).
 
         # This is overridden in the InteractiveShellEmbed subclass to a no-op.
 
         try:
             main_name = self.user_ns['__name__']
         except KeyError:
             raise KeyError('user_ns dictionary MUST have a "__name__" key')
         else:
             sys.modules[main_name] = FakeModule(self.user_ns)
 
     def init_user_ns(self):
         """Initialize all user-visible namespaces to their minimum defaults.
 
         Certain history lists are also initialized here, as they effectively
         act as user namespaces.
 
         Notes
         -----
         All data structures here are only filled in, they are NOT reset by this
         method.  If they were not empty before, data will simply be added to
         therm.
         """
         # This function works in two parts: first we put a few things in
         # user_ns, and we sync that contents into user_ns_hidden so that these
         # initial variables aren't shown by %who.  After the sync, we add the
         # rest of what we *do* want the user to see with %who even on a new
         # session (probably nothing, so theye really only see their own stuff)
 
         # The user dict must *always* have a __builtin__ reference to the
         # Python standard __builtin__ namespace,  which must be imported.
         # This is so that certain operations in prompt evaluation can be
         # reliably executed with builtins.  Note that we can NOT use
         # __builtins__ (note the 's'),  because that can either be a dict or a
         # module, and can even mutate at runtime, depending on the context
         # (Python makes no guarantees on it).  In contrast, __builtin__ is
         # always a module object, though it must be explicitly imported.
         
         # For more details:
         # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
         ns = dict(__builtin__ = __builtin__)
         
         # Put 'help' in the user namespace
         try:
             from site import _Helper
             ns['help'] = _Helper()
         except ImportError:
             warn('help() not available - check site.py')
 
         # make global variables for user access to the histories
         ns['_ih'] = self.input_hist
         ns['_oh'] = self.output_hist
         ns['_dh'] = self.dir_hist
 
         ns['_sh'] = shadowns
 
         # user aliases to input and output histories.  These shouldn't show up
         # in %who, as they can have very large reprs.
         ns['In']  = self.input_hist
         ns['Out'] = self.output_hist
 
         # Store myself as the public api!!!
         ns['get_ipython'] = self.get_ipython
 
         # Sync what we've added so far to user_ns_hidden so these aren't seen
         # by %who
         self.user_ns_hidden.update(ns)
 
         # Anything put into ns now would show up in %who.  Think twice before
         # putting anything here, as we really want %who to show the user their
         # stuff, not our variables.
         
         # Finally, update the real user's namespace
         self.user_ns.update(ns)
 
 
     def reset(self):
         """Clear all internal namespaces.
 
         Note that this is much more aggressive than %reset, since it clears
         fully all namespaces, as well as all input/output lists.
         """
         self.alias_manager.clear_aliases()
 
         # Clear input and output histories
         self.input_hist[:] = []
         self.input_hist_raw[:] = []
         self.output_hist.clear()
 
         # Clear namespaces holding user references
         for ns in self.ns_refs_table:
             ns.clear()
             
         # The main execution namespaces must be cleared very carefully,
         # skipping the deletion of the builtin-related keys, because doing so
         # would cause errors in many object's __del__ methods.
         for ns in [self.user_ns, self.user_global_ns]:
             drop_keys = set(ns.keys())
             drop_keys.discard('__builtin__')
             drop_keys.discard('__builtins__')
             for k in drop_keys:
                 del ns[k]
 
         # Restore the user namespaces to minimal usability
         self.init_user_ns()
         
         # Restore the default and user aliases
         self.alias_manager.init_aliases()
 
     def reset_selective(self, regex=None):
         """Clear selective variables from internal namespaces based on a
         specified regular expression.
 
         Parameters
         ----------
         regex : string or compiled pattern, optional
             A regular expression pattern that will be used in searching
             variable names in the users namespaces.
         """
         if regex is not None:
             try:
                 m = re.compile(regex)
             except TypeError:
                 raise TypeError('regex must be a string or compiled pattern')
             # Search for keys in each namespace that match the given regex
             # If a match is found, delete the key/value pair.
             for ns in self.ns_refs_table:
                 for var in ns:
                     if m.search(var):
                         del ns[var]        
         
     def push(self, variables, interactive=True):
         """Inject a group of variables into the IPython user namespace.
 
         Parameters
         ----------
         variables : dict, str or list/tuple of str
             The variables to inject into the user's namespace.  If a dict, a
             simple update is done.  If a str, the string is assumed to have
             variable names separated by spaces.  A list/tuple of str can also
             be used to give the variable names.  If just the variable names are
             give (list/tuple/str) then the variable values looked up in the
             callers frame.
         interactive : bool
             If True (default), the variables will be listed with the ``who``
             magic.
         """
         vdict = None
 
         # We need a dict of name/value pairs to do namespace updates.
         if isinstance(variables, dict):
             vdict = variables
         elif isinstance(variables, (basestring, list, tuple)):
             if isinstance(variables, basestring):
                 vlist = variables.split()
             else:
                 vlist = variables
             vdict = {}
             cf = sys._getframe(1)
             for name in vlist:
                 try:
                     vdict[name] = eval(name, cf.f_globals, cf.f_locals)
                 except:
                     print ('Could not get variable %s from %s' %
                            (name,cf.f_code.co_name))
         else:
             raise ValueError('variables must be a dict/str/list/tuple')
             
         # Propagate variables to user namespace
         self.user_ns.update(vdict)
 
         # And configure interactive visibility
         config_ns = self.user_ns_hidden
         if interactive:
             for name, val in vdict.iteritems():
                 config_ns.pop(name, None)
         else:
             for name,val in vdict.iteritems():
                 config_ns[name] = val
 
     #-------------------------------------------------------------------------
     # Things related to object introspection
     #-------------------------------------------------------------------------
 
     def _ofind(self, oname, namespaces=None):
         """Find an object in the available namespaces.
 
         self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
 
         Has special code to detect magic functions.
         """
         #oname = oname.strip()
         #print '1- oname: <%r>' % oname  # dbg
         try:
             oname = oname.strip().encode('ascii')
             #print '2- oname: <%r>' % oname  # dbg
         except UnicodeEncodeError:
             print 'Python identifiers can only contain ascii characters.'
             return dict(found=False)
 
         alias_ns = None
         if namespaces is None:
             # Namespaces to search in:
             # Put them in a list. The order is important so that we
             # find things in the same order that Python finds them.
             namespaces = [ ('Interactive', self.user_ns),
                            ('IPython internal', self.internal_ns),
                            ('Python builtin', __builtin__.__dict__),
                            ('Alias', self.alias_manager.alias_table),
                            ]
             alias_ns = self.alias_manager.alias_table
 
         # initialize results to 'null'
         found = False; obj = None;  ospace = None;  ds = None;
         ismagic = False; isalias = False; parent = None
 
         # We need to special-case 'print', which as of python2.6 registers as a
         # function but should only be treated as one if print_function was
         # loaded with a future import.  In this case, just bail.
         if (oname == 'print' and not (self.compile.compiler.flags &
                                       __future__.CO_FUTURE_PRINT_FUNCTION)):
             return {'found':found, 'obj':obj, 'namespace':ospace,
                     'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
 
         # Look for the given name by splitting it in parts.  If the head is
         # found, then we look for all the remaining parts as members, and only
         # declare success if we can find them all.
         oname_parts = oname.split('.')
         oname_head, oname_rest = oname_parts[0],oname_parts[1:]
         for nsname,ns in namespaces:
             try:
                 obj = ns[oname_head]
             except KeyError:
                 continue
             else:
                 #print 'oname_rest:', oname_rest  # dbg
                 for part in oname_rest:
                     try:
                         parent = obj
                         obj = getattr(obj,part)
                     except:
                         # Blanket except b/c some badly implemented objects
                         # allow __getattr__ to raise exceptions other than
                         # AttributeError, which then crashes IPython.
                         break
                 else:
                     # If we finish the for loop (no break), we got all members
                     found = True
                     ospace = nsname
                     if ns == alias_ns:
                         isalias = True
                     break  # namespace loop
 
         # Try to see if it's magic
         if not found:
             if oname.startswith(ESC_MAGIC):
                 oname = oname[1:]
             obj = getattr(self,'magic_'+oname,None)
             if obj is not None:
                 found = True
                 ospace = 'IPython internal'
                 ismagic = True
 
         # Last try: special-case some literals like '', [], {}, etc:
         if not found and oname_head in ["''",'""','[]','{}','()']:
             obj = eval(oname_head)
             found = True
             ospace = 'Interactive'
 
         return {'found':found, 'obj':obj, 'namespace':ospace,
                 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
 
     def _ofind_property(self, oname, info):
         """Second part of object finding, to look for property details."""
         if info.found:
             # Get the docstring of the class property if it exists.
             path = oname.split('.')
             root = '.'.join(path[:-1])
             if info.parent is not None:
                 try:
                     target = getattr(info.parent, '__class__') 
                     # The object belongs to a class instance. 
                     try: 
                         target = getattr(target, path[-1])
                         # The class defines the object. 
                         if isinstance(target, property):
                             oname = root + '.__class__.' + path[-1]
                             info = Struct(self._ofind(oname))
                     except AttributeError: pass
                 except AttributeError: pass
 
         # We return either the new info or the unmodified input if the object
         # hadn't been found
         return info
 
     def _object_find(self, oname, namespaces=None):
         """Find an object and return a struct with info about it."""
         inf = Struct(self._ofind(oname, namespaces))
         return Struct(self._ofind_property(oname, inf))
         
     def _inspect(self, meth, oname, namespaces=None, **kw):
         """Generic interface to the inspector system.
 
         This function is meant to be called by pdef, pdoc & friends."""
         info = self._object_find(oname)
         if info.found:
             pmethod = getattr(self.inspector, meth)
             formatter = format_screen if info.ismagic else None
             if meth == 'pdoc':
                 pmethod(info.obj, oname, formatter)
             elif meth == 'pinfo':
                 pmethod(info.obj, oname, formatter, info, **kw)
             else:
                 pmethod(info.obj, oname)
         else:
             print 'Object `%s` not found.' % oname
             return 'not found'  # so callers can take other action
 
     def object_inspect(self, oname):
         info = self._object_find(oname)
         if info.found:
             return self.inspector.info(info.obj, oname, info=info)
         else:
             return oinspect.object_info(name=oname, found=False)
 
     #-------------------------------------------------------------------------
     # Things related to history management
     #-------------------------------------------------------------------------
 
     def init_history(self):
         # List of input with multi-line handling.
         self.input_hist = InputList()
         # This one will hold the 'raw' input history, without any
         # pre-processing.  This will allow users to retrieve the input just as
         # it was exactly typed in by the user, with %hist -r.
         self.input_hist_raw = InputList()
 
         # list of visited directories
         try:
             self.dir_hist = [os.getcwd()]
         except OSError:
             self.dir_hist = []
 
         # dict of output history
         self.output_hist = {}
 
         # Now the history file
         if self.profile:
             histfname = 'history-%s' % self.profile
         else:
             histfname = 'history'
         self.histfile = os.path.join(self.ipython_dir, histfname)
 
         # Fill the history zero entry, user counter starts at 1
         self.input_hist.append('\n')
         self.input_hist_raw.append('\n')
 
     def init_shadow_hist(self):
         try:
             self.db = pickleshare.PickleShareDB(self.ipython_dir + "/db")
         except UnicodeDecodeError:
             print "Your ipython_dir can't be decoded to unicode!"
             print "Please set HOME environment variable to something that"
             print r"only has ASCII characters, e.g. c:\home"
             print "Now it is", self.ipython_dir
             sys.exit()
         self.shadowhist = ipcorehist.ShadowHist(self.db)
 
     def savehist(self):
         """Save input history to a file (via readline library)."""
 
         try:
             self.readline.write_history_file(self.histfile)
         except:
             print 'Unable to save IPython command history to file: ' + \
                   `self.histfile`
 
     def reloadhist(self):
         """Reload the input history from disk file."""
 
         try:
             self.readline.clear_history()
             self.readline.read_history_file(self.shell.histfile)
         except AttributeError:
             pass
 
     def history_saving_wrapper(self, func):
         """ Wrap func for readline history saving
 
         Convert func into callable that saves & restores
         history around the call """
 
         if self.has_readline:
             from IPython.utils import rlineimpl as readline
         else:
             return func
 
         def wrapper():
             self.savehist()
             try:
                 func()
             finally:
                 readline.read_history_file(self.histfile)
         return wrapper
 
     def get_history(self, index=None, raw=False, output=True):
         """Get the history list.
 
         Get the input and output history.
 
         Parameters
         ----------
         index : n or (n1, n2) or None
             If n, then the last entries. If a tuple, then all in
             range(n1, n2). If None, then all entries. Raises IndexError if
             the format of index is incorrect.
         raw : bool
             If True, return the raw input.
         output : bool
             If True, then return the output as well.
 
         Returns
         -------
         If output is True, then return a dict of tuples, keyed by the prompt
         numbers and with values of (input, output). If output is False, then
         a dict, keyed by the prompt number with the values of input. Raises
         IndexError if no history is found.
         """
         if raw:
             input_hist = self.input_hist_raw
         else:
             input_hist = self.input_hist
         if output:
             output_hist = self.user_ns['Out']
         n = len(input_hist)
         if index is None:
             start=0; stop=n
         elif isinstance(index, int):
             start=n-index; stop=n
         elif isinstance(index, tuple) and len(index) == 2:
             start=index[0]; stop=index[1]
         else:
             raise IndexError('Not a valid index for the input history: %r'
                              % index)
         hist = {}
         for i in range(start, stop):
             if output:
                 hist[i] = (input_hist[i], output_hist.get(i))
             else:
                 hist[i] = input_hist[i]
         if len(hist)==0:
             raise IndexError('No history for range of indices: %r' % index)
         return hist
 
     #-------------------------------------------------------------------------
     # Things related to exception handling and tracebacks (not debugging)
     #-------------------------------------------------------------------------
 
     def init_traceback_handlers(self, custom_exceptions):
         # Syntax error handler.
         self.SyntaxTB = ultratb.SyntaxTB(color_scheme='NoColor')
         
         # The interactive one is initialized with an offset, meaning we always
         # want to remove the topmost item in the traceback, which is our own
         # internal code. Valid modes: ['Plain','Context','Verbose']
         self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
                                                      color_scheme='NoColor',
                                                      tb_offset = 1)
 
         # The instance will store a pointer to the system-wide exception hook,
         # so that runtime code (such as magics) can access it.  This is because
         # during the read-eval loop, it may get temporarily overwritten.
         self.sys_excepthook = sys.excepthook
 
         # and add any custom exception handlers the user may have specified
         self.set_custom_exc(*custom_exceptions)
 
         # Set the exception mode
         self.InteractiveTB.set_mode(mode=self.xmode)
 
     def set_custom_exc(self, exc_tuple, handler):
         """set_custom_exc(exc_tuple,handler)
 
         Set a custom exception handler, which will be called if any of the
         exceptions in exc_tuple occur in the mainloop (specifically, in the
         runcode() method.
 
         Inputs:
 
           - exc_tuple: a *tuple* of valid exceptions to call the defined
           handler for.  It is very important that you use a tuple, and NOT A
           LIST here, because of the way Python's except statement works.  If
           you only want to trap a single exception, use a singleton tuple:
 
             exc_tuple == (MyCustomException,)
 
           - handler: this must be defined as a function with the following
           basic interface::
 
             def my_handler(self, etype, value, tb, tb_offset=None)
                 ...
                 # The return value must be
                 return structured_traceback
 
           This will be made into an instance method (via new.instancemethod)
           of IPython itself, and it will be called if any of the exceptions
           listed in the exc_tuple are caught.  If the handler is None, an
           internal basic one is used, which just prints basic info.
 
         WARNING: by putting in your own exception handler into IPython's main
         execution loop, you run a very good chance of nasty crashes.  This
         facility should only be used if you really know what you are doing."""
 
         assert type(exc_tuple)==type(()) , \
                "The custom exceptions must be given AS A TUPLE."
 
         def dummy_handler(self,etype,value,tb):
             print '*** Simple custom exception handler ***'
             print 'Exception type :',etype
             print 'Exception value:',value
             print 'Traceback      :',tb
             print 'Source code    :','\n'.join(self.buffer)
 
         if handler is None: handler = dummy_handler
 
         self.CustomTB = types.MethodType(handler, self)
         self.custom_exceptions = exc_tuple
 
     def excepthook(self, etype, value, tb):
       """One more defense for GUI apps that call sys.excepthook.
 
       GUI frameworks like wxPython trap exceptions and call
       sys.excepthook themselves.  I guess this is a feature that
       enables them to keep running after exceptions that would
       otherwise kill their mainloop. This is a bother for IPython
       which excepts to catch all of the program exceptions with a try:
       except: statement.
 
       Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
       any app directly invokes sys.excepthook, it will look to the user like
       IPython crashed.  In order to work around this, we can disable the
       CrashHandler and replace it with this excepthook instead, which prints a
       regular traceback using our InteractiveTB.  In this fashion, apps which
       call sys.excepthook will generate a regular-looking exception from
       IPython, and the CrashHandler will only be triggered by real IPython
       crashes.
 
       This hook should be used sparingly, only in places which are not likely
       to be true IPython errors.
       """
       self.showtraceback((etype,value,tb),tb_offset=0)
 
     def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None,
                       exception_only=False):
         """Display the exception that just occurred.
 
         If nothing is known about the exception, this is the method which
         should be used throughout the code for presenting user tracebacks,
         rather than directly invoking the InteractiveTB object.
 
         A specific showsyntaxerror() also exists, but this method can take
         care of calling it if needed, so unless you are explicitly catching a
         SyntaxError exception, don't try to analyze the stack manually and
         simply call this method."""
         
         try:
             if exc_tuple is None:
                 etype, value, tb = sys.exc_info()
             else:
                 etype, value, tb = exc_tuple
 
             if etype is None:
                 if hasattr(sys, 'last_type'):
                     etype, value, tb = sys.last_type, sys.last_value, \
                                        sys.last_traceback
                 else:
                     self.write_err('No traceback available to show.\n')
                     return
     
             if etype is SyntaxError:
                 # Though this won't be called by syntax errors in the input
                 # line, there may be SyntaxError cases whith imported code.
                 self.showsyntaxerror(filename)
             elif etype is UsageError:
                 print "UsageError:", value
             else:
                 # WARNING: these variables are somewhat deprecated and not
                 # necessarily safe to use in a threaded environment, but tools
                 # like pdb depend on their existence, so let's set them.  If we
                 # find problems in the field, we'll need to revisit their use.
                 sys.last_type = etype
                 sys.last_value = value
                 sys.last_traceback = tb
     
                 if etype in self.custom_exceptions:
                     # FIXME: Old custom traceback objects may just return a
                     # string, in that case we just put it into a list
                     stb = self.CustomTB(etype, value, tb, tb_offset)
                     if isinstance(ctb, basestring):
                         stb = [stb]
                 else:
                     if exception_only:
                         stb = ['An exception has occurred, use %tb to see '
                                'the full traceback.\n']
                         stb.extend(self.InteractiveTB.get_exception_only(etype,
                                                                          value))
                     else:
                         stb = self.InteractiveTB.structured_traceback(etype,
                                                 value, tb, tb_offset=tb_offset)
                         # FIXME: the pdb calling should be done by us, not by
                         # the code computing the traceback.
                         if self.InteractiveTB.call_pdb:
                             # pdb mucks up readline, fix it back
                             self.set_readline_completer()
 
                 # Actually show the traceback
                 self._showtraceback(etype, value, stb)
                 
         except KeyboardInterrupt:
             self.write_err("\nKeyboardInterrupt\n")
 
     def _showtraceback(self, etype, evalue, stb):
         """Actually show a traceback.
 
         Subclasses may override this method to put the traceback on a different
         place, like a side channel.
         """
         print >> io.Term.cout, self.InteractiveTB.stb2text(stb)
 
     def showsyntaxerror(self, filename=None):
         """Display the syntax error that just occurred.
 
         This doesn't display a stack trace because there isn't one.
 
         If a filename is given, it is stuffed in the exception instead
         of what was there before (because Python's parser always uses
         "<string>" when reading from a string).
         """
         etype, value, last_traceback = sys.exc_info()
 
         # See note about these variables in showtraceback() above
         sys.last_type = etype
         sys.last_value = value
         sys.last_traceback = last_traceback
         
         if filename and etype is SyntaxError:
             # Work hard to stuff the correct filename in the exception
             try:
                 msg, (dummy_filename, lineno, offset, line) = value
             except:
                 # Not the format we expect; leave it alone
                 pass
             else:
                 # Stuff in the right filename
                 try:
                     # Assume SyntaxError is a class exception
                     value = SyntaxError(msg, (filename, lineno, offset, line))
                 except:
                     # If that failed, assume SyntaxError is a string
                     value = msg, (filename, lineno, offset, line)
         stb = self.SyntaxTB.structured_traceback(etype, value, [])
         self._showtraceback(etype, value, stb)
 
     #-------------------------------------------------------------------------
     # Things related to readline
     #-------------------------------------------------------------------------
 
     def init_readline(self):
         """Command history completion/saving/reloading."""
 
         if self.readline_use:
             import IPython.utils.rlineimpl as readline
 
         self.rl_next_input = None
         self.rl_do_indent = False
 
         if not self.readline_use or not readline.have_readline:
             self.has_readline = False
             self.readline = None
             # Set a number of methods that depend on readline to be no-op
             self.savehist = no_op
             self.reloadhist = no_op
             self.set_readline_completer = no_op
             self.set_custom_completer = no_op
             self.set_completer_frame = no_op
             warn('Readline services not available or not loaded.')
         else:
             self.has_readline = True
             self.readline = readline
             sys.modules['readline'] = readline
             
             # Platform-specific configuration
             if os.name == 'nt':
                 # FIXME - check with Frederick to see if we can harmonize
                 # naming conventions with pyreadline to avoid this
                 # platform-dependent check
                 self.readline_startup_hook = readline.set_pre_input_hook
             else:
                 self.readline_startup_hook = readline.set_startup_hook
 
             # Load user's initrc file (readline config)
             # Or if libedit is used, load editrc.
             inputrc_name = os.environ.get('INPUTRC')
             if inputrc_name is None:
                 home_dir = get_home_dir()
                 if home_dir is not None:
                     inputrc_name = '.inputrc'
                     if readline.uses_libedit:
                         inputrc_name = '.editrc'
                     inputrc_name = os.path.join(home_dir, inputrc_name)
             if os.path.isfile(inputrc_name):
                 try:
                     readline.read_init_file(inputrc_name)
                 except:
                     warn('Problems reading readline initialization file <%s>'
                          % inputrc_name)
             
             # Configure readline according to user's prefs
             # This is only done if GNU readline is being used.  If libedit
             # is being used (as on Leopard) the readline config is
             # not run as the syntax for libedit is different.
             if not readline.uses_libedit:
                 for rlcommand in self.readline_parse_and_bind:
                     #print "loading rl:",rlcommand  # dbg
                     readline.parse_and_bind(rlcommand)
 
             # Remove some chars from the delimiters list.  If we encounter
             # unicode chars, discard them.
             delims = readline.get_completer_delims().encode("ascii", "ignore")
             delims = delims.translate(string._idmap,
                                       self.readline_remove_delims)
             delims = delims.replace(ESC_MAGIC, '')
             readline.set_completer_delims(delims)
             # otherwise we end up with a monster history after a while:
             readline.set_history_length(1000)
             try:
                 #print '*** Reading readline history'  # dbg
                 readline.read_history_file(self.histfile)
             except IOError:
                 pass  # It doesn't exist yet.
 
             # If we have readline, we want our history saved upon ipython
             # exiting. 
             atexit.register(self.savehist)
 
         # Configure auto-indent for all platforms
         self.set_autoindent(self.autoindent)
 
     def set_next_input(self, s):
         """ Sets the 'default' input string for the next command line.
         
         Requires readline.
         
         Example:
         
         [D:\ipython]|1> _ip.set_next_input("Hello Word")
         [D:\ipython]|2> Hello Word_  # cursor is here        
         """
 
         self.rl_next_input = s
 
     # Maybe move this to the terminal subclass?
     def pre_readline(self):
         """readline hook to be used at the start of each line.
 
         Currently it handles auto-indent only."""
 
         if self.rl_do_indent:
             self.readline.insert_text(self._indent_current_str())
         if self.rl_next_input is not None:
             self.readline.insert_text(self.rl_next_input)
             self.rl_next_input = None
 
     def _indent_current_str(self):
         """return the current level of indentation as a string"""
         return self.indent_current_nsp * ' '
 
     #-------------------------------------------------------------------------
     # Things related to text completion
     #-------------------------------------------------------------------------
 
     def init_completer(self):
         """Initialize the completion machinery.
 
         This creates completion machinery that can be used by client code,
         either interactively in-process (typically triggered by the readline
         library), programatically (such as in test suites) or out-of-prcess
         (typically over the network by remote frontends).
         """
         from IPython.core.completer import IPCompleter
         from IPython.core.completerlib import (module_completer,
                                                magic_run_completer, cd_completer)
         
         self.Completer = IPCompleter(self,
                                      self.user_ns,
                                      self.user_global_ns,
                                      self.readline_omit__names,
                                      self.alias_manager.alias_table,
                                      self.has_readline)
         
         # Add custom completers to the basic ones built into IPCompleter
         sdisp = self.strdispatchers.get('complete_command', StrDispatch())
         self.strdispatchers['complete_command'] = sdisp
         self.Completer.custom_completers = sdisp
 
         self.set_hook('complete_command', module_completer, str_key = 'import')
         self.set_hook('complete_command', module_completer, str_key = 'from')
         self.set_hook('complete_command', magic_run_completer, str_key = '%run')
         self.set_hook('complete_command', cd_completer, str_key = '%cd')
 
         # Only configure readline if we truly are using readline.  IPython can
         # do tab-completion over the network, in GUIs, etc, where readline
         # itself may be absent
         if self.has_readline:
             self.set_readline_completer()
 
     def complete(self, text, line=None, cursor_pos=None):
         """Return the completed text and a list of completions.
 
         Parameters
         ----------
 
            text : string
              A string of text to be completed on.  It can be given as empty and
              instead a line/position pair are given.  In this case, the
              completer itself will split the line like readline does.
 
            line : string, optional
              The complete line that text is part of.
 
            cursor_pos : int, optional
              The position of the cursor on the input line.
 
         Returns
         -------
           text : string
             The actual text that was completed.
 
           matches : list
             A sorted list with all possible completions.
 
         The optional arguments allow the completion to take more context into
         account, and are part of the low-level completion API.
         
         This is a wrapper around the completion mechanism, similar to what
         readline does at the command line when the TAB key is hit.  By
         exposing it as a method, it can be used by other non-readline
         environments (such as GUIs) for text completion.
 
         Simple usage example:
 
         In [1]: x = 'hello'
 
         In [2]: _ip.complete('x.l')
         Out[2]: ('x.l', ['x.ljust', 'x.lower', 'x.lstrip'])
         """
 
         # Inject names into __builtin__ so we can complete on the added names.
         with self.builtin_trap:
             return self.Completer.complete(text, line, cursor_pos)
 
     def set_custom_completer(self, completer, pos=0):
         """Adds a new custom completer function.
 
         The position argument (defaults to 0) is the index in the completers
         list where you want the completer to be inserted."""
 
         newcomp = types.MethodType(completer, self.Completer)
         self.Completer.matchers.insert(pos,newcomp)
 
     def set_readline_completer(self):
         """Reset readline's completer to be our own."""
         self.readline.set_completer(self.Completer.rlcomplete)
 
     def set_completer_frame(self, frame=None):
         """Set the frame of the completer."""
         if frame:
             self.Completer.namespace = frame.f_locals
             self.Completer.global_namespace = frame.f_globals
         else:
             self.Completer.namespace = self.user_ns
             self.Completer.global_namespace = self.user_global_ns
 
     #-------------------------------------------------------------------------
     # Things related to magics
     #-------------------------------------------------------------------------
 
     def init_magics(self):
         # FIXME: Move the color initialization to the DisplayHook, which
         # should be split into a prompt manager and displayhook. We probably
         # even need a centralize colors management object.
         self.magic_colors(self.colors)
         # History was moved to a separate module
         from . import history
         history.init_ipython(self)
 
     def magic(self,arg_s):
         """Call a magic function by name.
 
         Input: a string containing the name of the magic function to call and
         any additional arguments to be passed to the magic.
 
         magic('name -opt foo bar') is equivalent to typing at the ipython
         prompt:
 
         In[1]: %name -opt foo bar
 
         To call a magic without arguments, simply use magic('name').
 
         This provides a proper Python function to call IPython's magics in any
         valid Python code you can type at the interpreter, including loops and
         compound statements.
         """
         args = arg_s.split(' ',1)
         magic_name = args[0]
         magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
 
         try:
             magic_args = args[1]
         except IndexError:
             magic_args = ''
         fn = getattr(self,'magic_'+magic_name,None)
         if fn is None:
             error("Magic function `%s` not found." % magic_name)
         else:
             magic_args = self.var_expand(magic_args,1)
             with nested(self.builtin_trap,):
                 result = fn(magic_args)
                 return result
 
     def define_magic(self, magicname, func):
         """Expose own function as magic function for ipython 
     
         def foo_impl(self,parameter_s=''):
             'My very own magic!. (Use docstrings, IPython reads them).'
             print 'Magic function. Passed parameter is between < >:'
             print '<%s>' % parameter_s
             print 'The self object is:',self
     newcomp = types.MethodType(completer, self.Completer)
         self.define_magic('foo',foo_impl)
         """
         
         im = types.MethodType(func, self)
         old = getattr(self, "magic_" + magicname, None)
         setattr(self, "magic_" + magicname, im)
         return old
 
     #-------------------------------------------------------------------------
     # Things related to macros
     #-------------------------------------------------------------------------
 
     def define_macro(self, name, themacro):
         """Define a new macro
 
         Parameters
         ----------
         name : str
             The name of the macro.
         themacro : str or Macro
             The action to do upon invoking the macro.  If a string, a new 
             Macro object is created by passing the string to it.
         """
         
         from IPython.core import macro
 
         if isinstance(themacro, basestring):
             themacro = macro.Macro(themacro)
         if not isinstance(themacro, macro.Macro):
             raise ValueError('A macro must be a string or a Macro instance.')
         self.user_ns[name] = themacro
 
     #-------------------------------------------------------------------------
     # Things related to the running of system commands
     #-------------------------------------------------------------------------
 
     def system(self, cmd):
         """Call the given cmd in a subprocess.
 
         Parameters
         ----------
         cmd : str
           Command to execute (can not end in '&', as bacground processes are
           not supported.
         """
         # We do not support backgrounding processes because we either use
         # pexpect or pipes to read from.  Users can always just call
         # os.system() if they really want a background process.
         if cmd.endswith('&'):
             raise OSError("Background processes not supported.")
 
         return system(self.var_expand(cmd, depth=2))
 
     def getoutput(self, cmd, split=True):
         """Get output (possibly including stderr) from a subprocess.
 
         Parameters
         ----------
         cmd : str
           Command to execute (can not end in '&', as background processes are
           not supported.
         split : bool, optional
         
           If True, split the output into an IPython SList.  Otherwise, an
           IPython LSString is returned.  These are objects similar to normal
           lists and strings, with a few convenience attributes for easier
           manipulation of line-based output.  You can use '?' on them for
           details.
           """
         if cmd.endswith('&'):
             raise OSError("Background processes not supported.")
         out = getoutput(self.var_expand(cmd, depth=2))
         if split:
             out = SList(out.splitlines())
         else:
             out = LSString(out)
         return out
 
     #-------------------------------------------------------------------------
     # Things related to aliases
     #-------------------------------------------------------------------------
 
     def init_alias(self):
         self.alias_manager = AliasManager(shell=self, config=self.config)
         self.ns_table['alias'] = self.alias_manager.alias_table,
 
     #-------------------------------------------------------------------------
     # Things related to extensions and plugins
     #-------------------------------------------------------------------------
 
     def init_extension_manager(self):
         self.extension_manager = ExtensionManager(shell=self, config=self.config)
 
     def init_plugin_manager(self):
         self.plugin_manager = PluginManager(config=self.config)
 
     #-------------------------------------------------------------------------
     # Things related to payloads
     #-------------------------------------------------------------------------
 
     def init_payload(self):
         self.payload_manager = PayloadManager(config=self.config)
 
     #-------------------------------------------------------------------------
     # Things related to the prefilter
     #-------------------------------------------------------------------------
 
     def init_prefilter(self):
         self.prefilter_manager = PrefilterManager(shell=self, config=self.config)
         # Ultimately this will be refactored in the new interpreter code, but
         # for now, we should expose the main prefilter method (there's legacy
         # code out there that may rely on this).
         self.prefilter = self.prefilter_manager.prefilter_lines
 
 
     def auto_rewrite_input(self, cmd):
         """Print to the screen the rewritten form of the user's command.
 
         This shows visual feedback by rewriting input lines that cause
         automatic calling to kick in, like::
 
           /f x
 
         into::
 
           ------> f(x)
           
         after the user's input prompt.  This helps the user understand that the
         input line was transformed automatically by IPython.
         """
         rw = self.displayhook.prompt1.auto_rewrite() + cmd
 
         try:
             # plain ascii works better w/ pyreadline, on some machines, so
             # we use it and only print uncolored rewrite if we have unicode
             rw = str(rw)
             print >> IPython.utils.io.Term.cout, rw
         except UnicodeEncodeError:
             print "------> " + cmd
             
     #-------------------------------------------------------------------------
     # Things related to extracting values/expressions from kernel and user_ns
     #-------------------------------------------------------------------------
 
     def _simple_error(self):
         etype, value = sys.exc_info()[:2]
         return u'[ERROR] {e.__name__}: {v}'.format(e=etype, v=value)
 
     def user_variables(self, names):
         """Get a list of variable names from the user's namespace.
 
         Parameters
         ----------
         names : list of strings
           A list of names of variables to be read from the user namespace.
 
         Returns
         -------
         A dict, keyed by the input names and with the repr() of each value.
         """
         out = {}
         user_ns = self.user_ns
         for varname in names:
             try:
                 value = repr(user_ns[varname])
             except:
                 value = self._simple_error()
             out[varname] = value
         return out
         
     def user_expressions(self, expressions):
         """Evaluate a dict of expressions in the user's namespace.
 
         Parameters
         ----------
         expressions : dict
           A dict with string keys and string values.  The expression values
           should be valid Python expressions, each of which will be evaluated
           in the user namespace.
         
         Returns
         -------
         A dict, keyed like the input expressions dict, with the repr() of each
         value.
         """
         out = {}
         user_ns = self.user_ns
         global_ns = self.user_global_ns
         for key, expr in expressions.iteritems():
             try:
                 value = repr(eval(expr, global_ns, user_ns))
             except:
                 value = self._simple_error()
             out[key] = value
         return out
 
     #-------------------------------------------------------------------------
     # Things related to the running of code
     #-------------------------------------------------------------------------
 
     def ex(self, cmd):
         """Execute a normal python statement in user namespace."""
         with nested(self.builtin_trap,):
             exec cmd in self.user_global_ns, self.user_ns
 
     def ev(self, expr):
         """Evaluate python expression expr in user namespace.
 
         Returns the result of evaluation
         """
         with nested(self.builtin_trap,):
             return eval(expr, self.user_global_ns, self.user_ns)
 
     def safe_execfile(self, fname, *where, **kw):
         """A safe version of the builtin execfile().
 
         This version will never throw an exception, but instead print
         helpful error messages to the screen.  This only works on pure
         Python files with the .py extension.
 
         Parameters
         ----------
         fname : string
             The name of the file to be executed.
         where : tuple
             One or two namespaces, passed to execfile() as (globals,locals).
             If only one is given, it is passed as both.
         exit_ignore : bool (False)
             If True, then silence SystemExit for non-zero status (it is always
             silenced for zero status, as it is so common).
         """
         kw.setdefault('exit_ignore', False)
 
         fname = os.path.abspath(os.path.expanduser(fname))
 
         # Make sure we have a .py file
         if not fname.endswith('.py'):
             warn('File must end with .py to be run using execfile: <%s>' % fname)
 
         # Make sure we can open the file
         try:
             with open(fname) as thefile:
                 pass
         except:
             warn('Could not open file <%s> for safe execution.' % fname)
             return
 
         # Find things also in current directory.  This is needed to mimic the
         # behavior of running a script from the system command line, where
         # Python inserts the script's directory into sys.path
         dname = os.path.dirname(fname)
 
         with prepended_to_syspath(dname):
             try:
                 execfile(fname,*where)
             except SystemExit, status:
                 # If the call was made with 0 or None exit status (sys.exit(0)
                 # or sys.exit() ), don't bother showing a traceback, as both of
                 # these are considered normal by the OS:
                 # > python -c'import sys;sys.exit(0)'; echo $?
                 # 0
                 # > python -c'import sys;sys.exit()'; echo $?
                 # 0
                 # For other exit status, we show the exception unless
                 # explicitly silenced, but only in short form.
                 if status.code not in (0, None) and not kw['exit_ignore']:
                     self.showtraceback(exception_only=True)
             except:
                 self.showtraceback()
 
     def safe_execfile_ipy(self, fname):
         """Like safe_execfile, but for .ipy files with IPython syntax.
 
         Parameters
         ----------
         fname : str
             The name of the file to execute.  The filename must have a
             .ipy extension.
         """
         fname = os.path.abspath(os.path.expanduser(fname))
 
         # Make sure we have a .py file
         if not fname.endswith('.ipy'):
             warn('File must end with .py to be run using execfile: <%s>' % fname)
 
         # Make sure we can open the file
         try:
             with open(fname) as thefile:
                 pass
         except:
             warn('Could not open file <%s> for safe execution.' % fname)
             return
 
         # Find things also in current directory.  This is needed to mimic the
         # behavior of running a script from the system command line, where
         # Python inserts the script's directory into sys.path
         dname = os.path.dirname(fname)
 
         with prepended_to_syspath(dname):
             try:
                 with open(fname) as thefile:
                     script = thefile.read()
                     # self.runlines currently captures all exceptions
                     # raise in user code.  It would be nice if there were
                     # versions of runlines, execfile that did raise, so
                     # we could catch the errors.
                     self.runlines(script, clean=True)
             except:
                 self.showtraceback()
                 warn('Unknown failure executing file: <%s>' % fname)
 
     def run_cell(self, cell):
         """Run the contents of an entire multiline 'cell' of code.
 
         The cell is split into separate blocks which can be executed
         individually.  Then, based on how many blocks there are, they are
         executed as follows:
 
         - A single block: 'single' mode.
 
         If there's more than one block, it depends:
 
         - if the last one is no more than two lines long, run all but the last
         in 'exec' mode and the very last one in 'single' mode.  This makes it
         easy to type simple expressions at the end to see computed values.  -
         otherwise (last one is also multiline), run all in 'exec' mode
 
         When code is executed in 'single' mode, :func:`sys.displayhook` fires,
         results are displayed and output prompts are computed.  In 'exec' mode,
         no results are displayed unless :func:`print` is called explicitly;
         this mode is more akin to running a script.
 
         Parameters
         ----------
         cell : str
           A single or multiline string.
         """
         #################################################################
         # FIXME
         # =====
         # This execution logic should stop calling runlines altogether, and
         # instead we should do what runlines does, in a controlled manner, here
         # (runlines mutates lots of state as it goes calling sub-methods that
         # also mutate state).  Basically we should:
         # - apply dynamic transforms for single-line input (the ones that
         # split_blocks won't apply since they need context).
         # - increment the global execution counter (we need to pull that out
         # from outputcache's control; outputcache should instead read it from
         # the main object).
         # - do any logging of input
         # - update histories (raw/translated)
         # - then, call plain runsource (for single blocks, so displayhook is
         # triggered) or runcode (for multiline blocks in exec mode).
         #
         # Once this is done, we'll be able to stop using runlines and we'll
         # also have a much cleaner separation of logging, input history and
         # output cache management.
         #################################################################
         
         # We need to break up the input into executable blocks that can be run
         # in 'single' mode, to provide comfortable user behavior.
         blocks = self.input_splitter.split_blocks(cell)
         
         if not blocks:
             return
         
         # Single-block input should behave like an interactive prompt
         if len(blocks) == 1:
             self.runlines(blocks[0])
             return
 
         # In multi-block input, if the last block is a simple (one-two lines)
         # expression, run it in single mode so it produces output.  Otherwise
         # just feed the whole thing to runcode.
         # This seems like a reasonable usability design.
         last = blocks[-1]
         
         # Note: below, whenever we call runcode, we must sync history
         # ourselves, because runcode is NOT meant to manage history at all.
         if len(last.splitlines()) < 2:
             # Get the main body to run as a cell
             body = ''.join(blocks[:-1])
             self.input_hist.append(body)
             self.input_hist_raw.append(body)
             retcode = self.runcode(body, post_execute=False)
             if retcode==0:
                 # And the last expression via runlines so it produces output
                 self.runlines(last)
         else:
             # Run the whole cell as one entity
             self.input_hist.append(cell)
             self.input_hist_raw.append(cell)
             self.runcode(cell)
 
     def runlines(self, lines, clean=False):
         """Run a string of one or more lines of source.
 
         This method is capable of running a string containing multiple source
         lines, as if they had been entered at the IPython prompt.  Since it
         exposes IPython's processing machinery, the given strings can contain
         magic calls (%magic), special shell access (!cmd), etc.
         """
         
         if isinstance(lines, (list, tuple)):
             lines = '\n'.join(lines)
 
         if clean:
             lines = self._cleanup_ipy_script(lines)
 
         # We must start with a clean buffer, in case this is run from an
         # interactive IPython session (via a magic, for example).
         self.resetbuffer()
         lines = lines.splitlines()
         more = 0
         with nested(self.builtin_trap, self.display_trap):
             for line in lines:
                 # skip blank lines so we don't mess up the prompt counter, but
                 # do NOT skip even a blank line if we are in a code block (more
                 # is true)
             
                 if line or more:
                     # push to raw history, so hist line numbers stay in sync
                     self.input_hist_raw.append(line + '\n')
                     prefiltered = self.prefilter_manager.prefilter_lines(line,
                                                                          more)
                     more = self.push_line(prefiltered)
                     # IPython's runsource returns None if there was an error
                     # compiling the code.  This allows us to stop processing
                     # right away, so the user gets the error message at the
                     # right place.
                     if more is None:
                         break
                 else:
                     self.input_hist_raw.append("\n")
             # final newline in case the input didn't have it, so that the code
             # actually does get executed
             if more:
                 self.push_line('\n')
 
     def runsource(self, source, filename='<input>', symbol='single'):
         """Compile and run some source in the interpreter.
 
         Arguments are as for compile_command().
 
         One several things can happen:
 
         1) The input is incorrect; compile_command() raised an
         exception (SyntaxError or OverflowError).  A syntax traceback
         will be printed by calling the showsyntaxerror() method.
 
         2) The input is incomplete, and more input is required;
         compile_command() returned None.  Nothing happens.
 
         3) The input is complete; compile_command() returned a code
         object.  The code is executed by calling self.runcode() (which
         also handles run-time exceptions, except for SystemExit).
 
         The return value is:
 
           - True in case 2
 
           - False in the other cases, unless an exception is raised, where
           None is returned instead.  This can be used by external callers to
           know whether to continue feeding input or not.
 
         The return value can be used to decide whether to use sys.ps1 or
         sys.ps2 to prompt the next line."""
 
         # We need to ensure that the source is unicode from here on.
         if type(source)==str:
             source = source.decode(self.stdin_encoding)
         
         # if the source code has leading blanks, add 'if 1:\n' to it
         # this allows execution of indented pasted code. It is tempting
         # to add '\n' at the end of source to run commands like ' a=1'
         # directly, but this fails for more complicated scenarios
 
         if source[:1] in [' ', '\t']:
             source = u'if 1:\n%s' % source
 
         try:
             code = self.compile(source,filename,symbol)
         except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError):
             # Case 1
             self.showsyntaxerror(filename)
             return None
 
         if code is None:
             # Case 2
             return True
 
         # Case 3
         # We store the code object so that threaded shells and
         # custom exception handlers can access all this info if needed.
         # The source corresponding to this can be obtained from the
         # buffer attribute as '\n'.join(self.buffer).
         self.code_to_run = code
         # now actually execute the code object
         if self.runcode(code) == 0:
             return False
         else:
             return None
 
     def runcode(self, code_obj, post_execute=True):
         """Execute a code object.
 
         When an exception occurs, self.showtraceback() is called to display a
         traceback.
 
         Return value: a flag indicating whether the code to be run completed
         successfully:
 
           - 0: successful execution.
           - 1: an error occurred.
         """
 
         # Set our own excepthook in case the user code tries to call it
         # directly, so that the IPython crash handler doesn't get triggered
         old_excepthook,sys.excepthook = sys.excepthook, self.excepthook
 
         # we save the original sys.excepthook in the instance, in case config
         # code (such as magics) needs access to it.
         self.sys_excepthook = old_excepthook
         outflag = 1  # happens in more places, so it's easier as default
         try:
             try:
                 self.hooks.pre_runcode_hook()
                 #rprint('Running code') # dbg
                 exec code_obj in self.user_global_ns, self.user_ns
             finally:
                 # Reset our crash handler in place
                 sys.excepthook = old_excepthook
         except SystemExit:
             self.resetbuffer()
             self.showtraceback(exception_only=True)
             warn("To exit: use any of 'exit', 'quit', %Exit or Ctrl-D.", level=1)
         except self.custom_exceptions:
             etype,value,tb = sys.exc_info()
             self.CustomTB(etype,value,tb)
         except:
             self.showtraceback()
         else:
             outflag = 0
             if softspace(sys.stdout, 0):
                 print
 
         # Execute any registered post-execution functions.  Here, any errors
         # are reported only minimally and just on the terminal, because the
         # main exception channel may be occupied with a user traceback.
         # FIXME: we need to think this mechanism a little more carefully.
         if post_execute:
             for func in self._post_execute:
                 try:
                     func()
                 except:
                     head = '[ ERROR ] Evaluating post_execute function: %s' % \
                            func
                     print >> io.Term.cout, head
                     print >> io.Term.cout, self._simple_error()
                     print >> io.Term.cout, 'Removing from post_execute'
                     self._post_execute.remove(func)
 
         # Flush out code object which has been run (and source)
         self.code_to_run = None
         return outflag
         
     def push_line(self, line):
         """Push a line to the interpreter.
 
         The line should not have a trailing newline; it may have
         internal newlines.  The line is appended to a buffer and the
         interpreter's runsource() method is called with the
         concatenated contents of the buffer as source.  If this
         indicates that the command was executed or invalid, the buffer
         is reset; otherwise, the command is incomplete, and the buffer
         is left as it was after the line was appended.  The return
         value is 1 if more input is required, 0 if the line was dealt
         with in some way (this is the same as runsource()).
         """
 
         # autoindent management should be done here, and not in the
         # interactive loop, since that one is only seen by keyboard input.  We
         # need this done correctly even for code run via runlines (which uses
         # push).
 
         #print 'push line: <%s>' % line  # dbg
         for subline in line.splitlines():
             self._autoindent_update(subline)
         self.buffer.append(line)
         more = self.runsource('\n'.join(self.buffer), self.filename)
         if not more:
             self.resetbuffer()
         return more
 
     def resetbuffer(self):
         """Reset the input buffer."""
         self.buffer[:] = []
 
     def _is_secondary_block_start(self, s):
         if not s.endswith(':'):
             return False
         if (s.startswith('elif') or 
             s.startswith('else') or 
             s.startswith('except') or
             s.startswith('finally')):
             return True
 
     def _cleanup_ipy_script(self, script):
         """Make a script safe for self.runlines()
 
         Currently, IPython is lines based, with blocks being detected by
         empty lines.  This is a problem for block based scripts that may
         not have empty lines after blocks.  This script adds those empty
         lines to make scripts safe for running in the current line based
         IPython.
         """
         res = []
         lines = script.splitlines()
         level = 0
 
         for l in lines:
             lstripped = l.lstrip()
             stripped = l.strip()                
             if not stripped:
                 continue
             newlevel = len(l) - len(lstripped)                    
             if level > 0 and newlevel == 0 and \
                    not self._is_secondary_block_start(stripped): 
                 # add empty line
                 res.append('')
             res.append(l)
             level = newlevel
 
         return '\n'.join(res) + '\n'
 
     def _autoindent_update(self,line):
         """Keep track of the indent level."""
 
         #debugx('line')
         #debugx('self.indent_current_nsp')
         if self.autoindent:
             if line:
                 inisp = num_ini_spaces(line)
                 if inisp < self.indent_current_nsp:
                     self.indent_current_nsp = inisp
 
                 if line[-1] == ':':
                     self.indent_current_nsp += 4
                 elif dedent_re.match(line):
                     self.indent_current_nsp -= 4
             else:
                 self.indent_current_nsp = 0
 
     #-------------------------------------------------------------------------
     # Things related to GUI support and pylab
     #-------------------------------------------------------------------------
 
     def enable_pylab(self, gui=None):
         raise NotImplementedError('Implement enable_pylab in a subclass')
 
     #-------------------------------------------------------------------------
     # Utilities
     #-------------------------------------------------------------------------
 
     def var_expand(self,cmd,depth=0):
         """Expand python variables in a string.
 
         The depth argument indicates how many frames above the caller should
         be walked to look for the local namespace where to expand variables.
 
         The global namespace for expansion is always the user's interactive
         namespace.
         """
 
         return str(ItplNS(cmd,
                           self.user_ns,  # globals
                           # Skip our own frame in searching for locals:
                           sys._getframe(depth+1).f_locals # locals
                           ))
 
     def mktempfile(self,data=None):
         """Make a new tempfile and return its filename.
 
         This makes a call to tempfile.mktemp, but it registers the created
         filename internally so ipython cleans it up at exit time.
 
         Optional inputs:
 
           - data(None): if data is given, it gets written out to the temp file
           immediately, and the file is closed again."""
 
         filename = tempfile.mktemp('.py','ipython_edit_')
         self.tempfiles.append(filename)
         
         if data:
             tmp_file = open(filename,'w')
             tmp_file.write(data)
             tmp_file.close()
         return filename
 
     # TODO:  This should be removed when Term is refactored.
     def write(self,data):
         """Write a string to the default output"""
         io.Term.cout.write(data)
 
     # TODO:  This should be removed when Term is refactored.
     def write_err(self,data):
         """Write a string to the default error output"""
         io.Term.cerr.write(data)
 
     def ask_yes_no(self,prompt,default=True):
         if self.quiet:
             return True
         return ask_yes_no(prompt,default)
     
     def show_usage(self):
         """Show a usage message"""
         page.page(IPython.core.usage.interactive_usage)
 
     #-------------------------------------------------------------------------
     # Things related to IPython exiting
     #-------------------------------------------------------------------------
     def atexit_operations(self):
         """This will be executed at the time of exit.
 
         Cleanup operations and saving of persistent data that is done
         unconditionally by IPython should be performed here.
 
         For things that may depend on startup flags or platform specifics (such
         as having readline or not), register a separate atexit function in the
         code that has the appropriate information, rather than trying to
         clutter 
         """
         # Cleanup all tempfiles left around
         for tfile in self.tempfiles:
             try:
                 os.unlink(tfile)
             except OSError:
                 pass
 
         # Clear all user namespaces to release all references cleanly.
         self.reset()
 
         # Run user hooks
         self.hooks.shutdown_hook()
 
     def cleanup(self):
         self.restore_sys_module_state()
 
 
 class InteractiveShellABC(object):
     """An abstract base class for InteractiveShell."""
     __metaclass__ = abc.ABCMeta
 
 InteractiveShellABC.register(InteractiveShell)

IPython/core/tests/test_inputsplitter.py

@@ -1,658 +1,658
 # -*- coding: utf-8 -*-
 """Tests for the inputsplitter module.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 # stdlib
 import unittest
 import sys
 
 # Third party
 import nose.tools as nt
 
 # Our own
 from IPython.core import inputsplitter as isp
 
 #-----------------------------------------------------------------------------
 # Semi-complete examples (also used as tests)
 #-----------------------------------------------------------------------------
 
 # Note: at the bottom, there's a slightly more complete version of this that
 # can be useful during development of code here.
 
 def mini_interactive_loop(raw_input):
     """Minimal example of the logic of an interactive interpreter loop.
 
     This serves as an example, and it is used by the test system with a fake
     raw_input that simulates interactive input."""
 
     from IPython.core.inputsplitter import InputSplitter
     
     isp = InputSplitter()
     # In practice, this input loop would be wrapped in an outside loop to read
     # input indefinitely, until some exit/quit command was issued.  Here we
     # only illustrate the basic inner loop.
     while isp.push_accepts_more():
         indent = ' '*isp.indent_spaces
         prompt = '>>> ' + indent
         line = indent + raw_input(prompt)
         isp.push(line)
 
     # Here we just return input so we can use it in a test suite, but a real
     # interpreter would instead send it for execution somewhere.
     src = isp.source_reset()
     #print 'Input source was:\n', src  # dbg
     return src
 
 #-----------------------------------------------------------------------------
 # Test utilities, just for local use
 #-----------------------------------------------------------------------------
 
 def assemble(block):
     """Assemble a block into multi-line sub-blocks."""
     return ['\n'.join(sub_block)+'\n' for sub_block in block]
 
 
 def pseudo_input(lines):
     """Return a function that acts like raw_input but feeds the input list."""
     ilines = iter(lines)
     def raw_in(prompt):
         try:
             return next(ilines)
         except StopIteration:
             return ''
     return raw_in
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 def test_spaces():
     tests = [('', 0),
              (' ', 1),
              ('\n', 0),
              (' \n', 1),
              ('x', 0),
              (' x', 1),
              ('  x',2),
              ('    x',4),
              # Note: tabs are counted as a single whitespace!
              ('\tx', 1),
              ('\t x', 2),
              ]
     
     for s, nsp in tests:
         nt.assert_equal(isp.num_ini_spaces(s), nsp)
 
 
 def test_remove_comments():
     tests = [('text', 'text'),
              ('text # comment', 'text '),
              ('text # comment\n', 'text \n'),
              ('text # comment \n', 'text \n'),
              ('line # c \nline\n','line \nline\n'),
              ('line # c \nline#c2  \nline\nline #c\n\n',
               'line \nline\nline\nline \n\n'),
              ]
 
     for inp, out in tests:
         nt.assert_equal(isp.remove_comments(inp), out)
 
 
 def test_get_input_encoding():
     encoding = isp.get_input_encoding()
     nt.assert_true(isinstance(encoding, basestring))
     # simple-minded check that at least encoding a simple string works with the
     # encoding we got.
     nt.assert_equal('test'.encode(encoding), 'test')
 
 
 class NoInputEncodingTestCase(unittest.TestCase):
     def setUp(self):
         self.old_stdin = sys.stdin
         class X: pass
         fake_stdin = X()
         sys.stdin = fake_stdin
         
     def test(self):
         # Verify that if sys.stdin has no 'encoding' attribute we do the right
         # thing
         enc = isp.get_input_encoding()
         self.assertEqual(enc, 'ascii')
         
     def tearDown(self):
         sys.stdin = self.old_stdin
 
 
 class InputSplitterTestCase(unittest.TestCase):
     def setUp(self):
         self.isp = isp.InputSplitter()
 
     def test_reset(self):
         isp = self.isp
         isp.push('x=1')
         isp.reset()
         self.assertEqual(isp._buffer, [])
         self.assertEqual(isp.indent_spaces, 0)
         self.assertEqual(isp.source, '')
         self.assertEqual(isp.code, None)
         self.assertEqual(isp._is_complete, False)
 
     def test_source(self):
         self.isp._store('1')
         self.isp._store('2')
         self.assertEqual(self.isp.source, '1\n2\n')
         self.assertTrue(len(self.isp._buffer)>0)
         self.assertEqual(self.isp.source_reset(), '1\n2\n')
         self.assertEqual(self.isp._buffer, [])
         self.assertEqual(self.isp.source, '')
         
     def test_indent(self):
         isp = self.isp # shorthand
         isp.push('x=1')
         self.assertEqual(isp.indent_spaces, 0)
         isp.push('if 1:\n    x=1')
         self.assertEqual(isp.indent_spaces, 4)
         isp.push('y=2\n')
         self.assertEqual(isp.indent_spaces, 0)
         isp.push('if 1:')
         self.assertEqual(isp.indent_spaces, 4)
         isp.push('    x=1')
         self.assertEqual(isp.indent_spaces, 4)
         # Blank lines shouldn't change the indent level
         isp.push(' '*2)
         self.assertEqual(isp.indent_spaces, 4)
 
     def test_indent2(self):
         isp = self.isp
         # When a multiline statement contains parens or multiline strings, we
         # shouldn't get confused.
         isp.push("if 1:")
         isp.push("    x = (1+\n    2)")
         self.assertEqual(isp.indent_spaces, 4)
 
     def test_dedent(self):
         isp = self.isp # shorthand
         isp.push('if 1:')
         self.assertEqual(isp.indent_spaces, 4)
         isp.push('    pass')
         self.assertEqual(isp.indent_spaces, 0)
         
     def test_push(self):
         isp = self.isp
         self.assertTrue(isp.push('x=1'))
 
     def test_push2(self):
         isp = self.isp
         self.assertFalse(isp.push('if 1:'))
         for line in ['  x=1', '# a comment', '  y=2']:
             self.assertTrue(isp.push(line))
             
     def test_push3(self):
         """Test input with leading whitespace"""
         isp = self.isp
         isp.push('  x=1')
         isp.push('  y=2')
         self.assertEqual(isp.source, 'if 1:\n  x=1\n  y=2\n')
 
     def test_replace_mode(self):
         isp = self.isp
         isp.input_mode = 'cell'
         isp.push('x=1')
         self.assertEqual(isp.source, 'x=1\n')
         isp.push('x=2')
         self.assertEqual(isp.source, 'x=2\n')
 
     def test_push_accepts_more(self):
         isp = self.isp
         isp.push('x=1')
         self.assertFalse(isp.push_accepts_more())
 
     def test_push_accepts_more2(self):
         isp = self.isp
         isp.push('if 1:')
         self.assertTrue(isp.push_accepts_more())
         isp.push('  x=1')
         self.assertTrue(isp.push_accepts_more())
         isp.push('')
         self.assertFalse(isp.push_accepts_more())
         
     def test_push_accepts_more3(self):
         isp = self.isp
         isp.push("x = (2+\n3)")
         self.assertFalse(isp.push_accepts_more())
 
     def test_push_accepts_more4(self):
         isp = self.isp
         # When a multiline statement contains parens or multiline strings, we
         # shouldn't get confused.
         # FIXME: we should be able to better handle de-dents in statements like
         # multiline strings and multiline expressions (continued with \ or
         # parens).  Right now we aren't handling the indentation tracking quite
         # correctly with this, though in practice it may not be too much of a
         # problem.  We'll need to see.
         isp.push("if 1:")
         isp.push("    x = (2+")
         isp.push("    3)")
         self.assertTrue(isp.push_accepts_more())
         isp.push("    y = 3")
         self.assertTrue(isp.push_accepts_more())
         isp.push('')
         self.assertFalse(isp.push_accepts_more())
 
     def test_continuation(self):
         isp = self.isp
         isp.push("import os, \\")
         self.assertTrue(isp.push_accepts_more())
         isp.push("sys")
         self.assertFalse(isp.push_accepts_more())
 
     def test_syntax_error(self):
         isp = self.isp
         # Syntax errors immediately produce a 'ready' block, so the invalid
         # Python can be sent to the kernel for evaluation with possible ipython
         # special-syntax conversion.
         isp.push('run foo')
         self.assertFalse(isp.push_accepts_more())
 
     def check_split(self, block_lines, compile=True):
         blocks = assemble(block_lines)
         lines = ''.join(blocks)
         oblock = self.isp.split_blocks(lines)
         self.assertEqual(oblock, blocks)
         if compile:
             for block in blocks:
                 self.isp._compile(block)
 
     def test_split(self):
         # All blocks of input we want to test in a list.  The format for each
         # block is a list of lists, with each inner lists consisting of all the
         # lines (as single-lines) that should make up a sub-block.
 
         # Note: do NOT put here sub-blocks that don't compile, as the
         # check_split() routine makes a final verification pass to check that
         # each sub_block, as returned by split_blocks(), does compile
         # correctly.
         all_blocks = [ [['x=1']],
 
                        [['x=1'],
                         ['y=2']],
 
                        [['x=1',
                          '# a comment'],
                         ['y=11']],
 
                        [['if 1:',
                          '  x=1'],
                         ['y=3']],
 
                        [['def f(x):',
                          '  return x'],
                         ['x=1']],
 
                        [['def f(x):',
                          '  x+=1',
                          '  ',
                          '  return x'],
                         ['x=1']],
 
                        [['def f(x):',
                          '  if x>0:',
                          '    y=1',
                          '  # a comment',
                          '  else:',
                          '    y=4',
                          ' ',
                          '  return y'],
                         ['x=1'],
                         ['if 1:',
                          '  y=11'] ],
                        
                        [['for i in range(10):'
                          '  x=i**2']],
 
                        [['for i in range(10):'
                          '  x=i**2'],
                         ['z = 1']],
                        ]
         for block_lines in all_blocks:
             self.check_split(block_lines)
         
     def test_split_syntax_errors(self):
         # Block splitting with invalid syntax
         all_blocks = [ [['a syntax error']],
             
                        [['x=1',
                          'another syntax error']],
 
                        [['for i in range(10):'
                          '  yet another error']],
                        
                        ]
         for block_lines in all_blocks:
             self.check_split(block_lines, compile=False)
 
 
 class InteractiveLoopTestCase(unittest.TestCase):
     """Tests for an interactive loop like a python shell.
     """
     def check_ns(self, lines, ns):
         """Validate that the given input lines produce the resulting namespace.
 
         Note: the input lines are given exactly as they would be typed in an
         auto-indenting environment, as mini_interactive_loop above already does
         auto-indenting and prepends spaces to the input.
         """
         src = mini_interactive_loop(pseudo_input(lines))
         test_ns = {}
         exec src in test_ns
         # We can't check that the provided ns is identical to the test_ns,
         # because Python fills test_ns with extra keys (copyright, etc).  But
         # we can check that the given dict is *contained* in test_ns
-        for k,v in ns.items():
+        for k,v in ns.iteritems():
             self.assertEqual(test_ns[k], v)
         
     def test_simple(self):
         self.check_ns(['x=1'], dict(x=1))
 
     def test_simple2(self):
         self.check_ns(['if 1:', 'x=2'], dict(x=2))
 
     def test_xy(self):
         self.check_ns(['x=1; y=2'], dict(x=1, y=2))
 
     def test_abc(self):
         self.check_ns(['if 1:','a=1','b=2','c=3'], dict(a=1, b=2, c=3))
     
     def test_multi(self):
         self.check_ns(['x =(1+','1+','2)'], dict(x=4))
     
 
 def test_LineInfo():
     """Simple test for LineInfo construction and str()"""
     linfo = isp.LineInfo('  %cd /home')
     nt.assert_equals(str(linfo), 'LineInfo [  |%|cd|/home]')
 
 
 def test_split_user_input():
     """Unicode test - split_user_input already has good doctests"""
     line = u"Pérez Fernando"
     parts = isp.split_user_input(line)
     parts_expected = (u'', u'', u'', line)
     nt.assert_equal(parts, parts_expected)
 
 
 # Transformer tests
 def transform_checker(tests, func):
     """Utility to loop over test inputs"""
     for inp, tr in tests:
         nt.assert_equals(func(inp), tr)
 
 # Data for all the syntax tests in the form of lists of pairs of
 # raw/transformed input.  We store it here as a global dict so that we can use
 # it both within single-function tests and also to validate the behavior of the
 # larger objects
 
 syntax = \
   dict(assign_system =
        [('a =! ls', 'a = get_ipython().getoutput("ls")'),
         ('b = !ls', 'b = get_ipython().getoutput("ls")'),
         ('x=1', 'x=1'), # normal input is unmodified
         ('    ','    '),  # blank lines are kept intact
         ],
 
        assign_magic =
        [('a =% who', 'a = get_ipython().magic("who")'),
         ('b = %who', 'b = get_ipython().magic("who")'),
         ('x=1', 'x=1'), # normal input is unmodified
         ('    ','    '),  # blank lines are kept intact
         ],
 
        classic_prompt =
        [('>>> x=1', 'x=1'),
         ('x=1', 'x=1'), # normal input is unmodified
         ('    ', '    '),  # blank lines are kept intact
         ('... ', ''), # continuation prompts
         ],
 
        ipy_prompt =
        [('In [1]: x=1', 'x=1'),
         ('x=1', 'x=1'), # normal input is unmodified
         ('    ','    '),  # blank lines are kept intact
         ('   ....: ', ''), # continuation prompts
         ],
 
        # Tests for the escape transformer to leave normal code alone
        escaped_noesc =
        [ ('    ', '    '),
          ('x=1', 'x=1'),
          ],
 
        # System calls
        escaped_shell =
        [ ('!ls', 'get_ipython().system("ls")'),
          # Double-escape shell, this means to capture the output of the
          # subprocess and return it
          ('!!ls', 'get_ipython().getoutput("ls")'),
          ],
 
        # Help/object info
        escaped_help =
        [ ('?', 'get_ipython().show_usage()'),
          ('?x1', 'get_ipython().magic("pinfo x1")'),
          ('??x2', 'get_ipython().magic("pinfo2 x2")'),
          ('x3?', 'get_ipython().magic("pinfo x3")'),
          ('x4??', 'get_ipython().magic("pinfo2 x4")'),
          ('%hist?', 'get_ipython().magic("pinfo %hist")'),
          ('f*?', 'get_ipython().magic("psearch f*")'),
          ('ax.*aspe*?', 'get_ipython().magic("psearch ax.*aspe*")'),
          ],
 
        # Explicit magic calls
        escaped_magic =
        [ ('%cd', 'get_ipython().magic("cd")'),
          ('%cd /home', 'get_ipython().magic("cd /home")'),
          ('    %magic', '    get_ipython().magic("magic")'),
          ],
        
        # Quoting with separate arguments
        escaped_quote =
        [ (',f', 'f("")'),
          (',f x', 'f("x")'),
          ('  ,f y', '  f("y")'),
          (',f a b', 'f("a", "b")'),
          ],
        
        # Quoting with single argument
        escaped_quote2 = 
        [ (';f', 'f("")'),
          (';f x', 'f("x")'),
          ('  ;f y', '  f("y")'),
          (';f a b', 'f("a b")'),
          ],
        
        # Simply apply parens
        escaped_paren = 
        [ ('/f', 'f()'),
          ('/f x', 'f(x)'),
          ('  /f y', '  f(y)'),
          ('/f a b', 'f(a, b)'),
          ],
 
        )
 
 # multiline syntax examples.  Each of these should be a list of lists, with
 # each entry itself having pairs of raw/transformed input.  The union (with
 # '\n'.join() of the transformed inputs is what the splitter should produce
 # when fed the raw lines one at a time via push.
 syntax_ml = \
   dict(classic_prompt =
        [ [('>>> for i in range(10):','for i in range(10):'),
           ('...     print i','    print i'),
           ('... ', ''),
           ],
         ],
 
        ipy_prompt =
        [ [('In [24]: for i in range(10):','for i in range(10):'),
           ('   ....:     print i','    print i'),
           ('   ....: ', ''),
           ],
          ],
        )
 
 
 def test_assign_system():
     transform_checker(syntax['assign_system'], isp.transform_assign_system)
 
     
 def test_assign_magic():
     transform_checker(syntax['assign_magic'], isp.transform_assign_magic)
 
 
 def test_classic_prompt():
     transform_checker(syntax['classic_prompt'], isp.transform_classic_prompt)
     for example in syntax_ml['classic_prompt']:
         transform_checker(example, isp.transform_classic_prompt)
     
 
 def test_ipy_prompt():
     transform_checker(syntax['ipy_prompt'], isp.transform_ipy_prompt)
     for example in syntax_ml['ipy_prompt']:
         transform_checker(example, isp.transform_ipy_prompt)
     
 
 def test_escaped_noesc():
     transform_checker(syntax['escaped_noesc'], isp.transform_escaped)
 
 
 def test_escaped_shell():
     transform_checker(syntax['escaped_shell'], isp.transform_escaped)
 
 
 def test_escaped_help():
     transform_checker(syntax['escaped_help'], isp.transform_escaped)
 
 
 def test_escaped_magic():
     transform_checker(syntax['escaped_magic'], isp.transform_escaped)
 
 
 def test_escaped_quote():
     transform_checker(syntax['escaped_quote'], isp.transform_escaped)
 
 
 def test_escaped_quote2():
     transform_checker(syntax['escaped_quote2'], isp.transform_escaped)
 
 
 def test_escaped_paren():
     transform_checker(syntax['escaped_paren'], isp.transform_escaped)
 
 
 class IPythonInputTestCase(InputSplitterTestCase):
     """By just creating a new class whose .isp is a different instance, we
     re-run the same test battery on the new input splitter.
 
     In addition, this runs the tests over the syntax and syntax_ml dicts that
     were tested by individual functions, as part of the OO interface.
     """
 
     def setUp(self):
         self.isp = isp.IPythonInputSplitter(input_mode='line')
 
     def test_syntax(self):
         """Call all single-line syntax tests from the main object"""
         isp = self.isp
         for example in syntax.itervalues():
             for raw, out_t in example:
                 if raw.startswith(' '):
                     continue
                 
                 isp.push(raw)
                 out = isp.source_reset().rstrip()
                 self.assertEqual(out, out_t)
 
     def test_syntax_multiline(self):
         isp = self.isp
         for example in syntax_ml.itervalues():
             out_t_parts = []
             for line_pairs in example:
                 for raw, out_t_part in line_pairs:
                     isp.push(raw)
                     out_t_parts.append(out_t_part)
 
                 out = isp.source_reset().rstrip()
                 out_t = '\n'.join(out_t_parts).rstrip()
                 self.assertEqual(out, out_t)
                 
 
 class BlockIPythonInputTestCase(IPythonInputTestCase):
 
     # Deactivate tests that don't make sense for the block mode
     test_push3 = test_split = lambda s: None
     
     def setUp(self):
         self.isp = isp.IPythonInputSplitter(input_mode='cell')
 
     def test_syntax_multiline(self):
         isp = self.isp
         for example in syntax_ml.itervalues():
             raw_parts = []
             out_t_parts = []
             for line_pairs in example:
                 for raw, out_t_part in line_pairs:
                     raw_parts.append(raw)
                     out_t_parts.append(out_t_part)
 
                 raw = '\n'.join(raw_parts)
                 out_t = '\n'.join(out_t_parts)
 
                 isp.push(raw)
                 out = isp.source_reset()
                 # Match ignoring trailing whitespace
                 self.assertEqual(out.rstrip(), out_t.rstrip())
     
 
 #-----------------------------------------------------------------------------
 # Main - use as a script, mostly for developer experiments
 #-----------------------------------------------------------------------------
 
 if __name__ == '__main__':
     # A simple demo for interactive experimentation.  This code will not get
     # picked up by any test suite.
     from IPython.core.inputsplitter import InputSplitter, IPythonInputSplitter
 
     # configure here the syntax to use, prompt and whether to autoindent
     #isp, start_prompt = InputSplitter(), '>>> '
     isp, start_prompt = IPythonInputSplitter(), 'In> '
 
     autoindent = True
     #autoindent = False
     
     try:
         while True:
             prompt = start_prompt
             while isp.push_accepts_more():
                 indent = ' '*isp.indent_spaces
                 if autoindent:
                     line = indent + raw_input(prompt+indent)
                 else:
                     line = raw_input(prompt)
                 isp.push(line)
                 prompt = '... '
 
             # Here we just return input so we can use it in a test suite, but a
             # real interpreter would instead send it for execution somewhere.
             #src = isp.source; raise EOFError # dbg
             src = isp.source_reset()
             print 'Input source was:\n', src
     except EOFError:
         print 'Bye'

IPython/core/tests/test_magic.py

@@ -1,357 +1,357
 """Tests for various magic functions.
 
 Needs to be run by nose (to make ipython session available).
 """
 from __future__ import absolute_import
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 import tempfile
 import types
 from cStringIO import StringIO
 
 import nose.tools as nt
 
 from IPython.utils.path import get_long_path_name
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 
 #-----------------------------------------------------------------------------
 # Test functions begin
 #-----------------------------------------------------------------------------
 def test_rehashx():
     # clear up everything
     _ip = get_ipython()
     _ip.alias_manager.alias_table.clear()
     del _ip.db['syscmdlist']
     
     _ip.magic('rehashx')
     # Practically ALL ipython development systems will have more than 10 aliases
 
     yield (nt.assert_true, len(_ip.alias_manager.alias_table) > 10)
-    for key, val in _ip.alias_manager.alias_table.items():
+    for key, val in _ip.alias_manager.alias_table.iteritems():
         # we must strip dots from alias names
         nt.assert_true('.' not in key)
 
     # rehashx must fill up syscmdlist
     scoms = _ip.db['syscmdlist']
     yield (nt.assert_true, len(scoms) > 10)
 
 
 def test_magic_parse_options():
     """Test that we don't mangle paths when parsing magic options."""
     ip = get_ipython()
     path = 'c:\\x'
     opts = ip.parse_options('-f %s' % path,'f:')[0]
     # argv splitting is os-dependent
     if os.name == 'posix':
         expected = 'c:x'
     else:
         expected = path
     nt.assert_equals(opts['f'], expected)
 
     
 def doctest_hist_f():
     """Test %hist -f with temporary filename.
 
     In [9]: import tempfile
 
     In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
 
     In [11]: %hist -n -f $tfile 3
 
     In [13]: import os; os.unlink(tfile)
     """
 
 
 def doctest_hist_r():
     """Test %hist -r
 
     XXX - This test is not recording the output correctly.  For some reason, in
     testing mode the raw history isn't getting populated.  No idea why.
     Disabling the output checking for now, though at least we do run it.
 
     In [1]: 'hist' in _ip.lsmagic()
     Out[1]: True
 
     In [2]: x=1
 
     In [3]: %hist -r 2
     x=1 # random
     %hist -r 2
     """
 
 def doctest_hist_op():
     """Test %hist -op
 
     In [1]: class b:
        ...:         pass
        ...: 
 
     In [2]: class s(b):
        ...:         def __str__(self):
        ...:             return 's'
        ...: 
 
     In [3]: 
 
     In [4]: class r(b):
        ...:         def __repr__(self):
        ...:             return 'r'
        ...: 
 
     In [5]: class sr(s,r): pass
        ...: 
 
     In [6]: 
 
     In [7]: bb=b()
 
     In [8]: ss=s()
 
     In [9]: rr=r()
 
     In [10]: ssrr=sr()
 
     In [11]: bb
     Out[11]: <...b instance at ...>
 
     In [12]: ss
     Out[12]: <...s instance at ...>
 
     In [13]: 
 
     In [14]: %hist -op
     >>> class b:
     ...     pass
     ... 
     >>> class s(b):
     ...     def __str__(self):
     ...         return 's'
     ... 
     >>> 
     >>> class r(b):
     ...     def __repr__(self):
     ...         return 'r'
     ... 
     >>> class sr(s,r): pass
     >>> 
     >>> bb=b()
     >>> ss=s()
     >>> rr=r()
     >>> ssrr=sr()
     >>> bb
     <...b instance at ...>
     >>> ss
     <...s instance at ...>
     >>> 
     """
 
 def test_shist():
     # Simple tests of ShadowHist class - test generator.
     import os, shutil, tempfile
 
     from IPython.utils import pickleshare
     from IPython.core.history import ShadowHist
     
     tfile = tempfile.mktemp('','tmp-ipython-')
     
     db = pickleshare.PickleShareDB(tfile)
     s = ShadowHist(db)
     s.add('hello')
     s.add('world')
     s.add('hello')
     s.add('hello')
     s.add('karhu')
 
     yield nt.assert_equals,s.all(),[(1, 'hello'), (2, 'world'), (3, 'karhu')]
     
     yield nt.assert_equal,s.get(2),'world'
     
     shutil.rmtree(tfile)
 
     
 # XXX failing for now, until we get clearcmd out of quarantine.  But we should
 # fix this and revert the skip to happen only if numpy is not around.
 #@dec.skipif_not_numpy
 @dec.skip_known_failure
 def test_numpy_clear_array_undec():
     from IPython.extensions import clearcmd
 
     _ip.ex('import numpy as np')
     _ip.ex('a = np.empty(2)')
     yield (nt.assert_true, 'a' in _ip.user_ns)
     _ip.magic('clear array')
     yield (nt.assert_false, 'a' in _ip.user_ns)
     
 
 # Multiple tests for clipboard pasting
 @dec.parametric
 def test_paste():
     _ip = get_ipython()
     def paste(txt, flags='-q'):
         """Paste input text, by default in quiet mode"""
         hooks.clipboard_get = lambda : txt
         _ip.magic('paste '+flags)
 
     # Inject fake clipboard hook but save original so we can restore it later
     hooks = _ip.hooks
     user_ns = _ip.user_ns
     original_clip = hooks.clipboard_get
 
     try:
         # This try/except with an emtpy except clause is here only because
         # try/yield/finally is invalid syntax in Python 2.4.  This will be
         # removed when we drop 2.4-compatibility, and the emtpy except below
         # will be changed to a finally.
 
         # Run tests with fake clipboard function
         user_ns.pop('x', None)
         paste('x=1')
         yield nt.assert_equal(user_ns['x'], 1)
 
         user_ns.pop('x', None)
         paste('>>> x=2')
         yield nt.assert_equal(user_ns['x'], 2)
 
         paste("""
         >>> x = [1,2,3]
         >>> y = []
         >>> for i in x:
         ...     y.append(i**2)
         ...
         """)
         yield nt.assert_equal(user_ns['x'], [1,2,3])
         yield nt.assert_equal(user_ns['y'], [1,4,9])
 
         # Now, test that paste -r works
         user_ns.pop('x', None)
         yield nt.assert_false('x' in user_ns)
         _ip.magic('paste -r')
         yield nt.assert_equal(user_ns['x'], [1,2,3])
 
         # Also test paste echoing, by temporarily faking the writer
         w = StringIO()
         writer = _ip.write
         _ip.write = w.write
         code = """
         a = 100
         b = 200"""
         try:
             paste(code,'')
             out = w.getvalue()
         finally:
             _ip.write = writer
         yield nt.assert_equal(user_ns['a'], 100)
         yield nt.assert_equal(user_ns['b'], 200)
         yield nt.assert_equal(out, code+"\n## -- End pasted text --\n")
         
     finally:
         # This should be in a finally clause, instead of the bare except above.
         # Restore original hook
         hooks.clipboard_get = original_clip
 
 
 def test_time():
     _ip.magic('time None')
 
 
 def doctest_time():
     """
     In [10]: %time None
     CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
     Wall time: 0.00 s
     """
 
 
 def test_doctest_mode():
     "Toggle doctest_mode twice, it should be a no-op and run without error"
     _ip.magic('doctest_mode')
     _ip.magic('doctest_mode')
 
 
 def test_parse_options():
     """Tests for basic options parsing in magics."""
     # These are only the most minimal of tests, more should be added later.  At
     # the very least we check that basic text/unicode calls work OK.
     nt.assert_equal(_ip.parse_options('foo', '')[1], 'foo')
     nt.assert_equal(_ip.parse_options(u'foo', '')[1], u'foo')
 
     
 def test_dirops():
     """Test various directory handling operations."""
     curpath = lambda :os.path.splitdrive(os.getcwd())[1].replace('\\','/')
 
     startdir = os.getcwd()
     ipdir = _ip.ipython_dir
     try:
         _ip.magic('cd "%s"' % ipdir)
         nt.assert_equal(curpath(), ipdir)
         _ip.magic('cd -')
         nt.assert_equal(curpath(), startdir)
         _ip.magic('pushd "%s"' % ipdir)
         nt.assert_equal(curpath(), ipdir)
         _ip.magic('popd')
         nt.assert_equal(curpath(), startdir)
     finally:
         os.chdir(startdir)
 
 
 def check_cpaste(code, should_fail=False):
     """Execute code via 'cpaste' and ensure it was executed, unless
     should_fail is set.
     """
     _ip.user_ns['code_ran'] = False
 
     src = StringIO()
     src.write('\n')
     src.write(code)
     src.write('\n--\n')
     src.seek(0)
 
     stdin_save = sys.stdin
     sys.stdin = src
     
     try:
         _ip.magic('cpaste')
     except:
         if not should_fail:
             raise AssertionError("Failure not expected : '%s'" %
                                  code)
     else:
         assert _ip.user_ns['code_ran']
         if should_fail:
             raise AssertionError("Failure expected : '%s'" % code)
     finally:
         sys.stdin = stdin_save
 
 
 def test_cpaste():
     """Test cpaste magic"""
 
     def run():
         """Marker function: sets a flag when executed.
         """
         _ip.user_ns['code_ran'] = True
         return 'run' # return string so '+ run()' doesn't result in success
 
     tests = {'pass': ["> > > run()",
                       ">>> > run()",
                       "+++ run()",
                       "++ run()",
                       "  >>> run()"],
 
              'fail': ["+ + run()",
                       " ++ run()"]}
 
     _ip.user_ns['run'] = run
 
     for code in tests['pass']:
         check_cpaste(code)
 
     for code in tests['fail']:
         check_cpaste(code, should_fail=True)

IPython/external/configobj.py

@@ -1,2501 +1,2472
+# -*- coding: utf-8 -*-
 # configobj.py
 # A config file reader/writer that supports nested sections in config files.
 # Copyright (C) 2005-2008 Michael Foord, Nicola Larosa
 # E-mail: fuzzyman AT voidspace DOT org DOT uk
 #         nico AT tekNico DOT net
 
 # ConfigObj 4
 # http://www.voidspace.org.uk/python/configobj.html
 
 # Released subject to the BSD License
 # Please see http://www.voidspace.org.uk/python/license.shtml
 
 # Scripts maintained at http://www.voidspace.org.uk/python/index.shtml
 # For information about bugfixes, updates and support, please join the
 # ConfigObj mailing list:
 # http://lists.sourceforge.net/lists/listinfo/configobj-develop
 # Comments, suggestions and bug reports welcome.
 
 from __future__ import generators
 
 import sys
 INTP_VER = sys.version_info[:2]
 if INTP_VER < (2, 2):
     raise RuntimeError("Python v.2.2 or later needed")
 
 import os, re
 compiler = None
 try:
     import compiler
 except ImportError:
     # for IronPython
     pass
 from types import StringTypes
 from warnings import warn
-try:
     from codecs import BOM_UTF8, BOM_UTF16, BOM_UTF16_BE, BOM_UTF16_LE
-except ImportError:
-    # Python 2.2 does not have these
-    # UTF-8
-    BOM_UTF8 = '\xef\xbb\xbf'
-    # UTF-16, little endian
-    BOM_UTF16_LE = '\xff\xfe'
-    # UTF-16, big endian
-    BOM_UTF16_BE = '\xfe\xff'
-    if sys.byteorder == 'little':
-        # UTF-16, native endianness
-        BOM_UTF16 = BOM_UTF16_LE
-    else:
-        # UTF-16, native endianness
-        BOM_UTF16 = BOM_UTF16_BE
 
 # A dictionary mapping BOM to
 # the encoding to decode with, and what to set the
 # encoding attribute to.
 BOMS = {
     BOM_UTF8: ('utf_8', None),
     BOM_UTF16_BE: ('utf16_be', 'utf_16'),
     BOM_UTF16_LE: ('utf16_le', 'utf_16'),
     BOM_UTF16: ('utf_16', 'utf_16'),
     }
 # All legal variants of the BOM codecs.
 # TODO: the list of aliases is not meant to be exhaustive, is there a
 #   better way ?
 BOM_LIST = {
     'utf_16': 'utf_16',
     'u16': 'utf_16',
     'utf16': 'utf_16',
     'utf-16': 'utf_16',
     'utf16_be': 'utf16_be',
     'utf_16_be': 'utf16_be',
     'utf-16be': 'utf16_be',
     'utf16_le': 'utf16_le',
     'utf_16_le': 'utf16_le',
     'utf-16le': 'utf16_le',
     'utf_8': 'utf_8',
     'u8': 'utf_8',
     'utf': 'utf_8',
     'utf8': 'utf_8',
     'utf-8': 'utf_8',
     }
 
 # Map of encodings to the BOM to write.
 BOM_SET = {
     'utf_8': BOM_UTF8,
     'utf_16': BOM_UTF16,
     'utf16_be': BOM_UTF16_BE,
     'utf16_le': BOM_UTF16_LE,
     None: BOM_UTF8
     }
 
 
 def match_utf8(encoding):
     return BOM_LIST.get(encoding.lower()) == 'utf_8'
 
 
 # Quote strings used for writing values
 squot = "'%s'"
 dquot = '"%s"'
 noquot = "%s"
 wspace_plus = ' \r\t\n\v\t\'"'
 tsquot = '"""%s"""'
 tdquot = "'''%s'''"
 
-try:
-    enumerate
-except NameError:
-    def enumerate(obj):
-        """enumerate for Python 2.2."""
-        i = -1
-        for item in obj:
-            i += 1
-            yield i, item
-
-try:
-    True, False
-except NameError:
-    True, False = 1, 0
-
 
 __version__ = '4.5.2'
 
 __revision__ = '$Id: configobj.py 156 2006-01-31 14:57:08Z fuzzyman $'
 
 __docformat__ = "restructuredtext en"
 
 __all__ = (
     '__version__',
     'DEFAULT_INDENT_TYPE',
     'DEFAULT_INTERPOLATION',
     'ConfigObjError',
     'NestingError',
     'ParseError',
     'DuplicateError',
     'ConfigspecError',
     'ConfigObj',
     'SimpleVal',
     'InterpolationError',
     'InterpolationLoopError',
     'MissingInterpolationOption',
     'RepeatSectionError',
     'ReloadError',
     'UnreprError',
     'UnknownType',
     '__docformat__',
     'flatten_errors',
 )
 
 DEFAULT_INTERPOLATION = 'configparser'
 DEFAULT_INDENT_TYPE = '    '
 MAX_INTERPOL_DEPTH = 10
 
 OPTION_DEFAULTS = {
     'interpolation': True,
     'raise_errors': False,
     'list_values': True,
     'create_empty': False,
     'file_error': False,
     'configspec': None,
     'stringify': True,
     # option may be set to one of ('', ' ', '\t')
     'indent_type': None,
     'encoding': None,
     'default_encoding': None,
     'unrepr': False,
     'write_empty_values': False,
 }
 
 
 
 def getObj(s):
     s = "a=" + s
     if compiler is None:
         raise ImportError('compiler module not available')
     p = compiler.parse(s)
     return p.getChildren()[1].getChildren()[0].getChildren()[1]
 
 
 class UnknownType(Exception):
     pass
 
 
 class Builder(object):
     
     def build(self, o):
         m = getattr(self, 'build_' + o.__class__.__name__, None)
         if m is None:
             raise UnknownType(o.__class__.__name__)
         return m(o)
     
     def build_List(self, o):
         return map(self.build, o.getChildren())
     
     def build_Const(self, o):
         return o.value
     
     def build_Dict(self, o):
         d = {}
         i = iter(map(self.build, o.getChildren()))
         for el in i:
             d[el] = i.next()
         return d
     
     def build_Tuple(self, o):
         return tuple(self.build_List(o))
     
     def build_Name(self, o):
         if o.name == 'None':
             return None
         if o.name == 'True':
             return True
         if o.name == 'False':
             return False
         
         # An undefined Name
         raise UnknownType('Undefined Name')
     
     def build_Add(self, o):
         real, imag = map(self.build_Const, o.getChildren())
         try:
             real = float(real)
         except TypeError:
             raise UnknownType('Add')
         if not isinstance(imag, complex) or imag.real != 0.0:
             raise UnknownType('Add')
         return real+imag
     
     def build_Getattr(self, o):
         parent = self.build(o.expr)
         return getattr(parent, o.attrname)
     
     def build_UnarySub(self, o):
         return -self.build_Const(o.getChildren()[0])
     
     def build_UnaryAdd(self, o):
         return self.build_Const(o.getChildren()[0])
 
 
 _builder = Builder()
 
 
 def unrepr(s):
     if not s:
         return s
     return _builder.build(getObj(s))
 
 
 
 class ConfigObjError(SyntaxError):
     """
     This is the base class for all errors that ConfigObj raises.
     It is a subclass of SyntaxError.
     """
     def __init__(self, message='', line_number=None, line=''):
         self.line = line
         self.line_number = line_number
         self.message = message
         SyntaxError.__init__(self, message)
 
 
 class NestingError(ConfigObjError):
     """
     This error indicates a level of nesting that doesn't match.
     """
 
 
 class ParseError(ConfigObjError):
     """
     This error indicates that a line is badly written.
     It is neither a valid ``key = value`` line,
     nor a valid section marker line.
     """
 
 
 class ReloadError(IOError):
     """
     A 'reload' operation failed.
     This exception is a subclass of ``IOError``.
     """
     def __init__(self):
         IOError.__init__(self, 'reload failed, filename is not set.')
 
 
 class DuplicateError(ConfigObjError):
     """
     The keyword or section specified already exists.
     """
 
 
 class ConfigspecError(ConfigObjError):
     """
     An error occured whilst parsing a configspec.
     """
 
 
 class InterpolationError(ConfigObjError):
     """Base class for the two interpolation errors."""
 
 
 class InterpolationLoopError(InterpolationError):
     """Maximum interpolation depth exceeded in string interpolation."""
 
     def __init__(self, option):
         InterpolationError.__init__(
             self,
             'interpolation loop detected in value "%s".' % option)
 
 
 class RepeatSectionError(ConfigObjError):
     """
     This error indicates additional sections in a section with a
     ``__many__`` (repeated) section.
     """
 
 
 class MissingInterpolationOption(InterpolationError):
     """A value specified for interpolation was missing."""
 
     def __init__(self, option):
         InterpolationError.__init__(
             self,
             'missing option "%s" in interpolation.' % option)
 
 
 class UnreprError(ConfigObjError):
     """An error parsing in unrepr mode."""
 
 
 
 class InterpolationEngine(object):
     """
     A helper class to help perform string interpolation.
 
     This class is an abstract base class; its descendants perform
     the actual work.
     """
 
     # compiled regexp to use in self.interpolate()
     _KEYCRE = re.compile(r"%\(([^)]*)\)s")
 
     def __init__(self, section):
         # the Section instance that "owns" this engine
         self.section = section
 
 
     def interpolate(self, key, value):
         def recursive_interpolate(key, value, section, backtrail):
             """The function that does the actual work.
 
             ``value``: the string we're trying to interpolate.
             ``section``: the section in which that string was found
             ``backtrail``: a dict to keep track of where we've been,
             to detect and prevent infinite recursion loops
 
             This is similar to a depth-first-search algorithm.
             """
             # Have we been here already?
             if backtrail.has_key((key, section.name)):
                 # Yes - infinite loop detected
                 raise InterpolationLoopError(key)
             # Place a marker on our backtrail so we won't come back here again
             backtrail[(key, section.name)] = 1
 
             # Now start the actual work
             match = self._KEYCRE.search(value)
             while match:
                 # The actual parsing of the match is implementation-dependent,
                 # so delegate to our helper function
                 k, v, s = self._parse_match(match)
                 if k is None:
                     # That's the signal that no further interpolation is needed
                     replacement = v
                 else:
                     # Further interpolation may be needed to obtain final value
                     replacement = recursive_interpolate(k, v, s, backtrail)
                 # Replace the matched string with its final value
                 start, end = match.span()
                 value = ''.join((value[:start], replacement, value[end:]))
                 new_search_start = start + len(replacement)
                 # Pick up the next interpolation key, if any, for next time
                 # through the while loop
                 match = self._KEYCRE.search(value, new_search_start)
 
             # Now safe to come back here again; remove marker from backtrail
             del backtrail[(key, section.name)]
 
             return value
 
         # Back in interpolate(), all we have to do is kick off the recursive
         # function with appropriate starting values
         value = recursive_interpolate(key, value, self.section, {})
         return value
 
 
     def _fetch(self, key):
         """Helper function to fetch values from owning section.
 
         Returns a 2-tuple: the value, and the section where it was found.
         """
         # switch off interpolation before we try and fetch anything !
         save_interp = self.section.main.interpolation
         self.section.main.interpolation = False
 
         # Start at section that "owns" this InterpolationEngine
         current_section = self.section
         while True:
             # try the current section first
             val = current_section.get(key)
             if val is not None:
                 break
             # try "DEFAULT" next
             val = current_section.get('DEFAULT', {}).get(key)
             if val is not None:
                 break
             # move up to parent and try again
             # top-level's parent is itself
             if current_section.parent is current_section:
                 # reached top level, time to give up
                 break
             current_section = current_section.parent
 
         # restore interpolation to previous value before returning
         self.section.main.interpolation = save_interp
         if val is None:
             raise MissingInterpolationOption(key)
         return val, current_section
 
 
     def _parse_match(self, match):
         """Implementation-dependent helper function.
 
         Will be passed a match object corresponding to the interpolation
         key we just found (e.g., "%(foo)s" or "$foo"). Should look up that
         key in the appropriate config file section (using the ``_fetch()``
         helper function) and return a 3-tuple: (key, value, section)
 
         ``key`` is the name of the key we're looking for
         ``value`` is the value found for that key
         ``section`` is a reference to the section where it was found
 
         ``key`` and ``section`` should be None if no further
         interpolation should be performed on the resulting value
         (e.g., if we interpolated "$$" and returned "$").
         """
         raise NotImplementedError()
     
 
 
 class ConfigParserInterpolation(InterpolationEngine):
     """Behaves like ConfigParser."""
     _KEYCRE = re.compile(r"%\(([^)]*)\)s")
 
     def _parse_match(self, match):
         key = match.group(1)
         value, section = self._fetch(key)
         return key, value, section
 
 
 
 class TemplateInterpolation(InterpolationEngine):
     """Behaves like string.Template."""
     _delimiter = '$'
     _KEYCRE = re.compile(r"""
         \$(?:
           (?P<escaped>\$)              |   # Two $ signs
           (?P<named>[_a-z][_a-z0-9]*)  |   # $name format
           {(?P<braced>[^}]*)}              # ${name} format
         )
         """, re.IGNORECASE | re.VERBOSE)
 
     def _parse_match(self, match):
         # Valid name (in or out of braces): fetch value from section
         key = match.group('named') or match.group('braced')
         if key is not None:
             value, section = self._fetch(key)
             return key, value, section
         # Escaped delimiter (e.g., $$): return single delimiter
         if match.group('escaped') is not None:
             # Return None for key and section to indicate it's time to stop
             return None, self._delimiter, None
         # Anything else: ignore completely, just return it unchanged
         return None, match.group(), None
 
 
 interpolation_engines = {
     'configparser': ConfigParserInterpolation,
     'template': TemplateInterpolation,
 }
 
 
 
 class Section(dict):
     """
     A dictionary-like object that represents a section in a config file.
     
     It does string interpolation if the 'interpolation' attribute
     of the 'main' object is set to True.
     
     Interpolation is tried first from this object, then from the 'DEFAULT'
     section of this object, next from the parent and its 'DEFAULT' section,
     and so on until the main object is reached.
     
     A Section will behave like an ordered dictionary - following the
     order of the ``scalars`` and ``sections`` attributes.
     You can use this to change the order of members.
     
     Iteration follows the order: scalars, then sections.
     """
 
     def __init__(self, parent, depth, main, indict=None, name=None):
         """
         * parent is the section above
         * depth is the depth level of this section
         * main is the main ConfigObj
         * indict is a dictionary to initialise the section with
         """
         if indict is None:
             indict = {}
         dict.__init__(self)
         # used for nesting level *and* interpolation
         self.parent = parent
         # used for the interpolation attribute
         self.main = main
         # level of nesting depth of this Section
         self.depth = depth
         # purely for information
         self.name = name
         #
         self._initialise()
         # we do this explicitly so that __setitem__ is used properly
         # (rather than just passing to ``dict.__init__``)
         for entry, value in indict.iteritems():
             self[entry] = value
             
             
     def _initialise(self):
         # the sequence of scalar values in this Section
         self.scalars = []
         # the sequence of sections in this Section
         self.sections = []
         # for comments :-)
         self.comments = {}
         self.inline_comments = {}
         # for the configspec
         self.configspec = {}
         self._order = []
         self._configspec_comments = {}
         self._configspec_inline_comments = {}
         self._cs_section_comments = {}
         self._cs_section_inline_comments = {}
         # for defaults
         self.defaults = []
         self.default_values = {}
 
 
     def _interpolate(self, key, value):
         try:
             # do we already have an interpolation engine?
             engine = self._interpolation_engine
         except AttributeError:
             # not yet: first time running _interpolate(), so pick the engine
             name = self.main.interpolation
             if name == True:  # note that "if name:" would be incorrect here
                 # backwards-compatibility: interpolation=True means use default
                 name = DEFAULT_INTERPOLATION
             name = name.lower()  # so that "Template", "template", etc. all work
             class_ = interpolation_engines.get(name, None)
             if class_ is None:
                 # invalid value for self.main.interpolation
                 self.main.interpolation = False
                 return value
             else:
                 # save reference to engine so we don't have to do this again
                 engine = self._interpolation_engine = class_(self)
         # let the engine do the actual work
         return engine.interpolate(key, value)
 
 
     def __getitem__(self, key):
         """Fetch the item and do string interpolation."""
         val = dict.__getitem__(self, key)
         if self.main.interpolation and isinstance(val, StringTypes):
             return self._interpolate(key, val)
         return val
 
 
     def __setitem__(self, key, value, unrepr=False):
         """
         Correctly set a value.
         
         Making dictionary values Section instances.
         (We have to special case 'Section' instances - which are also dicts)
         
         Keys must be strings.
         Values need only be strings (or lists of strings) if
         ``main.stringify`` is set.
         
         `unrepr`` must be set when setting a value to a dictionary, without
         creating a new sub-section.
         """
         if not isinstance(key, StringTypes):
             raise ValueError('The key "%s" is not a string.' % key)
         
         # add the comment
         if not self.comments.has_key(key):
             self.comments[key] = []
             self.inline_comments[key] = ''
         # remove the entry from defaults
         if key in self.defaults:
             self.defaults.remove(key)
         #
         if isinstance(value, Section):
             if not self.has_key(key):
                 self.sections.append(key)
             dict.__setitem__(self, key, value)
         elif isinstance(value, dict) and not unrepr:
             # First create the new depth level,
             # then create the section
             if not self.has_key(key):
                 self.sections.append(key)
             new_depth = self.depth + 1
             dict.__setitem__(
                 self,
                 key,
                 Section(
                     self,
                     new_depth,
                     self.main,
                     indict=value,
                     name=key))
         else:
             if not self.has_key(key):
                 self.scalars.append(key)
             if not self.main.stringify:
                 if isinstance(value, StringTypes):
                     pass
                 elif isinstance(value, (list, tuple)):
                     for entry in value:
                         if not isinstance(entry, StringTypes):
                             raise TypeError('Value is not a string "%s".' % entry)
                 else:
                     raise TypeError('Value is not a string "%s".' % value)
             dict.__setitem__(self, key, value)
 
 
     def __delitem__(self, key):
         """Remove items from the sequence when deleting."""
         dict. __delitem__(self, key)
         if key in self.scalars:
             self.scalars.remove(key)
         else:
             self.sections.remove(key)
         del self.comments[key]
         del self.inline_comments[key]
 
 
     def get(self, key, default=None):
         """A version of ``get`` that doesn't bypass string interpolation."""
         try:
             return self[key]
         except KeyError:
             return default
 
 
     def update(self, indict):
         """
         A version of update that uses our ``__setitem__``.
         """
         for entry in indict:
             self[entry] = indict[entry]
 
 
     def pop(self, key, *args):
         """
         'D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
         If key is not found, d is returned if given, otherwise KeyError is raised'
         """
         val = dict.pop(self, key, *args)
         if key in self.scalars:
             del self.comments[key]
             del self.inline_comments[key]
             self.scalars.remove(key)
         elif key in self.sections:
             del self.comments[key]
             del self.inline_comments[key]
             self.sections.remove(key)
         if self.main.interpolation and isinstance(val, StringTypes):
             return self._interpolate(key, val)
         return val
 
 
     def popitem(self):
         """Pops the first (key,val)"""
         sequence = (self.scalars + self.sections)
         if not sequence:
             raise KeyError(": 'popitem(): dictionary is empty'")
         key = sequence[0]
         val =  self[key]
         del self[key]
         return key, val
 
 
     def clear(self):
         """
         A version of clear that also affects scalars/sections
         Also clears comments and configspec.
         
         Leaves other attributes alone :
             depth/main/parent are not affected
         """
         dict.clear(self)
         self.scalars = []
         self.sections = []
         self.comments = {}
         self.inline_comments = {}
         self.configspec = {}
 
 
     def setdefault(self, key, default=None):
         """A version of setdefault that sets sequence if appropriate."""
         try:
             return self[key]
         except KeyError:
             self[key] = default
             return self[key]
 
 
     def items(self):
         """D.items() -> list of D's (key, value) pairs, as 2-tuples"""
         return zip((self.scalars + self.sections), self.values())
 
 
     def keys(self):
         """D.keys() -> list of D's keys"""
         return (self.scalars + self.sections)
 
 
     def values(self):
         """D.values() -> list of D's values"""
         return [self[key] for key in (self.scalars + self.sections)]
 
 
     def iteritems(self):
         """D.iteritems() -> an iterator over the (key, value) items of D"""
         return iter(self.items())
 
 
     def iterkeys(self):
         """D.iterkeys() -> an iterator over the keys of D"""
         return iter((self.scalars + self.sections))
 
     __iter__ = iterkeys
 
 
     def itervalues(self):
         """D.itervalues() -> an iterator over the values of D"""
         return iter(self.values())
 
 
     def __repr__(self):
         """x.__repr__() <==> repr(x)"""
         return '{%s}' % ', '.join([('%s: %s' % (repr(key), repr(self[key])))
             for key in (self.scalars + self.sections)])
 
     __str__ = __repr__
     __str__.__doc__ = "x.__str__() <==> str(x)"
 
 
     # Extra methods - not in a normal dictionary
 
     def dict(self):
         """
         Return a deepcopy of self as a dictionary.
         
         All members that are ``Section`` instances are recursively turned to
         ordinary dictionaries - by calling their ``dict`` method.
         
         >>> n = a.dict()
         >>> n == a
         1
         >>> n is a
         0
         """
         newdict = {}
         for entry in self:
             this_entry = self[entry]
             if isinstance(this_entry, Section):
                 this_entry = this_entry.dict()
             elif isinstance(this_entry, list):
                 # create a copy rather than a reference
                 this_entry = list(this_entry)
             elif isinstance(this_entry, tuple):
                 # create a copy rather than a reference
                 this_entry = tuple(this_entry)
             newdict[entry] = this_entry
         return newdict
 
 
     def merge(self, indict):
         """
         A recursive update - useful for merging config files.
         
         >>> a = '''[section1]
         ...     option1 = True
         ...     [[subsection]]
         ...     more_options = False
         ...     # end of file'''.splitlines()
         >>> b = '''# File is user.ini
         ...     [section1]
         ...     option1 = False
         ...     # end of file'''.splitlines()
         >>> c1 = ConfigObj(b)
         >>> c2 = ConfigObj(a)
         >>> c2.merge(c1)
         >>> c2
         {'section1': {'option1': 'False', 'subsection': {'more_options': 'False'}}}
         """
-        for key, val in indict.items():
+        for key, val in indict.iteritems():
             if (key in self and isinstance(self[key], dict) and
                                 isinstance(val, dict)):
                 self[key].merge(val)
             else:   
                 self[key] = val
 
 
     def rename(self, oldkey, newkey):
         """
         Change a keyname to another, without changing position in sequence.
         
         Implemented so that transformations can be made on keys,
         as well as on values. (used by encode and decode)
         
         Also renames comments.
         """
         if oldkey in self.scalars:
             the_list = self.scalars
         elif oldkey in self.sections:
             the_list = self.sections
         else:
             raise KeyError('Key "%s" not found.' % oldkey)
         pos = the_list.index(oldkey)
         #
         val = self[oldkey]
         dict.__delitem__(self, oldkey)
         dict.__setitem__(self, newkey, val)
         the_list.remove(oldkey)
         the_list.insert(pos, newkey)
         comm = self.comments[oldkey]
         inline_comment = self.inline_comments[oldkey]
         del self.comments[oldkey]
         del self.inline_comments[oldkey]
         self.comments[newkey] = comm
         self.inline_comments[newkey] = inline_comment
 
 
     def walk(self, function, raise_errors=True,
             call_on_sections=False, **keywargs):
         """
         Walk every member and call a function on the keyword and value.
         
         Return a dictionary of the return values
         
         If the function raises an exception, raise the errror
         unless ``raise_errors=False``, in which case set the return value to
         ``False``.
         
         Any unrecognised keyword arguments you pass to walk, will be pased on
         to the function you pass in.
         
         Note: if ``call_on_sections`` is ``True`` then - on encountering a
         subsection, *first* the function is called for the *whole* subsection,
         and then recurses into it's members. This means your function must be
         able to handle strings, dictionaries and lists. This allows you
         to change the key of subsections as well as for ordinary members. The
         return value when called on the whole subsection has to be discarded.
         
         See  the encode and decode methods for examples, including functions.
         
         .. caution::
         
             You can use ``walk`` to transform the names of members of a section
             but you mustn't add or delete members.
         
         >>> config = '''[XXXXsection]
         ... XXXXkey = XXXXvalue'''.splitlines()
         >>> cfg = ConfigObj(config)
         >>> cfg
         {'XXXXsection': {'XXXXkey': 'XXXXvalue'}}
         >>> def transform(section, key):
         ...     val = section[key]
         ...     newkey = key.replace('XXXX', 'CLIENT1')
         ...     section.rename(key, newkey)
         ...     if isinstance(val, (tuple, list, dict)):
         ...         pass
         ...     else:
         ...         val = val.replace('XXXX', 'CLIENT1')
         ...         section[newkey] = val
         >>> cfg.walk(transform, call_on_sections=True)
         {'CLIENT1section': {'CLIENT1key': None}}
         >>> cfg
         {'CLIENT1section': {'CLIENT1key': 'CLIENT1value'}}
         """
         out = {}
         # scalars first
         for i in range(len(self.scalars)):
             entry = self.scalars[i]
             try:
                 val = function(self, entry, **keywargs)
                 # bound again in case name has changed
                 entry = self.scalars[i]
                 out[entry] = val
             except Exception:
                 if raise_errors:
                     raise
                 else:
                     entry = self.scalars[i]
                     out[entry] = False
         # then sections
         for i in range(len(self.sections)):
             entry = self.sections[i]
             if call_on_sections:
                 try:
                     function(self, entry, **keywargs)
                 except Exception:
                     if raise_errors:
                         raise
                     else:
                         entry = self.sections[i]
                         out[entry] = False
                 # bound again in case name has changed
                 entry = self.sections[i]
             # previous result is discarded
             out[entry] = self[entry].walk(
                 function,
                 raise_errors=raise_errors,
                 call_on_sections=call_on_sections,
                 **keywargs)
         return out
 
 
     def decode(self, encoding):
         """
         Decode all strings and values to unicode, using the specified encoding.
         
         Works with subsections and list values.
         
         Uses the ``walk`` method.
         
         Testing ``encode`` and ``decode``.
         >>> m = ConfigObj(a)
         >>> m.decode('ascii')
         >>> def testuni(val):
         ...     for entry in val:
         ...         if not isinstance(entry, unicode):
         ...             print >> sys.stderr, type(entry)
         ...             raise AssertionError, 'decode failed.'
         ...         if isinstance(val[entry], dict):
         ...             testuni(val[entry])
         ...         elif not isinstance(val[entry], unicode):
         ...             raise AssertionError, 'decode failed.'
         >>> testuni(m)
         >>> m.encode('ascii')
         >>> a == m
         1
         """
         warn('use of ``decode`` is deprecated.', DeprecationWarning)
         def decode(section, key, encoding=encoding, warn=True):
             """ """
             val = section[key]
             if isinstance(val, (list, tuple)):
                 newval = []
                 for entry in val:
                     newval.append(entry.decode(encoding))
             elif isinstance(val, dict):
                 newval = val
             else:
                 newval = val.decode(encoding)
             newkey = key.decode(encoding)
             section.rename(key, newkey)
             section[newkey] = newval
         # using ``call_on_sections`` allows us to modify section names
         self.walk(decode, call_on_sections=True)
 
 
     def encode(self, encoding):
         """
         Encode all strings and values from unicode,
         using the specified encoding.
         
         Works with subsections and list values.
         Uses the ``walk`` method.
         """
         warn('use of ``encode`` is deprecated.', DeprecationWarning)
         def encode(section, key, encoding=encoding):
             """ """
             val = section[key]
             if isinstance(val, (list, tuple)):
                 newval = []
                 for entry in val:
                     newval.append(entry.encode(encoding))
             elif isinstance(val, dict):
                 newval = val
             else:
                 newval = val.encode(encoding)
             newkey = key.encode(encoding)
             section.rename(key, newkey)
             section[newkey] = newval
         self.walk(encode, call_on_sections=True)
 
 
     def istrue(self, key):
         """A deprecated version of ``as_bool``."""
         warn('use of ``istrue`` is deprecated. Use ``as_bool`` method '
                 'instead.', DeprecationWarning)
         return self.as_bool(key)
 
 
     def as_bool(self, key):
         """
         Accepts a key as input. The corresponding value must be a string or
         the objects (``True`` or 1) or (``False`` or 0). We allow 0 and 1 to
         retain compatibility with Python 2.2.
         
         If the string is one of  ``True``, ``On``, ``Yes``, or ``1`` it returns 
         ``True``.
         
         If the string is one of  ``False``, ``Off``, ``No``, or ``0`` it returns 
         ``False``.
         
         ``as_bool`` is not case sensitive.
         
         Any other input will raise a ``ValueError``.
         
         >>> a = ConfigObj()
         >>> a['a'] = 'fish'
         >>> a.as_bool('a')
         Traceback (most recent call last):
         ValueError: Value "fish" is neither True nor False
         >>> a['b'] = 'True'
         >>> a.as_bool('b')
         1
         >>> a['b'] = 'off'
         >>> a.as_bool('b')
         0
         """
         val = self[key]
         if val == True:
             return True
         elif val == False:
             return False
         else:
             try:
                 if not isinstance(val, StringTypes):
                     # TODO: Why do we raise a KeyError here?
                     raise KeyError()
                 else:
                     return self.main._bools[val.lower()]
             except KeyError:
                 raise ValueError('Value "%s" is neither True nor False' % val)
 
 
     def as_int(self, key):
         """
         A convenience method which coerces the specified value to an integer.
         
         If the value is an invalid literal for ``int``, a ``ValueError`` will
         be raised.
         
         >>> a = ConfigObj()
         >>> a['a'] = 'fish'
         >>> a.as_int('a')
         Traceback (most recent call last):
         ValueError: invalid literal for int(): fish
         >>> a['b'] = '1'
         >>> a.as_int('b')
         1
         >>> a['b'] = '3.2'
         >>> a.as_int('b')
         Traceback (most recent call last):
         ValueError: invalid literal for int(): 3.2
         """
         return int(self[key])
 
 
     def as_float(self, key):
         """
         A convenience method which coerces the specified value to a float.
         
         If the value is an invalid literal for ``float``, a ``ValueError`` will
         be raised.
         
         >>> a = ConfigObj()
         >>> a['a'] = 'fish'
         >>> a.as_float('a')
         Traceback (most recent call last):
         ValueError: invalid literal for float(): fish
         >>> a['b'] = '1'
         >>> a.as_float('b')
         1.0
         >>> a['b'] = '3.2'
         >>> a.as_float('b')
         3.2000000000000002
         """
         return float(self[key])
 
 
     def restore_default(self, key):
         """
         Restore (and return) default value for the specified key.
         
         This method will only work for a ConfigObj that was created
         with a configspec and has been validated.
         
         If there is no default value for this key, ``KeyError`` is raised.
         """
         default = self.default_values[key]
         dict.__setitem__(self, key, default)
         if key not in self.defaults:
             self.defaults.append(key)
         return default
 
     
     def restore_defaults(self):
         """
         Recursively restore default values to all members
         that have them.
         
         This method will only work for a ConfigObj that was created
         with a configspec and has been validated.
         
         It doesn't delete or modify entries without default values.
         """
         for key in self.default_values:
             self.restore_default(key)
             
         for section in self.sections:
             self[section].restore_defaults()
 
 
 class ConfigObj(Section):
     """An object to read, create, and write config files."""
 
     _keyword = re.compile(r'''^ # line start
         (\s*)                   # indentation
         (                       # keyword
             (?:".*?")|          # double quotes
             (?:'.*?')|          # single quotes
             (?:[^'"=].*?)       # no quotes
         )
         \s*=\s*                 # divider
         (.*)                    # value (including list values and comments)
         $   # line end
         ''',
         re.VERBOSE)
 
     _sectionmarker = re.compile(r'''^
         (\s*)                     # 1: indentation
         ((?:\[\s*)+)              # 2: section marker open
         (                         # 3: section name open
             (?:"\s*\S.*?\s*")|    # at least one non-space with double quotes
             (?:'\s*\S.*?\s*')|    # at least one non-space with single quotes
             (?:[^'"\s].*?)        # at least one non-space unquoted
         )                         # section name close
         ((?:\s*\])+)              # 4: section marker close
         \s*(\#.*)?                # 5: optional comment
         $''',
         re.VERBOSE)
 
     # this regexp pulls list values out as a single string
     # or single values and comments
     # FIXME: this regex adds a '' to the end of comma terminated lists
     #   workaround in ``_handle_value``
     _valueexp = re.compile(r'''^
         (?:
             (?:
                 (
                     (?:
                         (?:
                             (?:".*?")|              # double quotes
                             (?:'.*?')|              # single quotes
                             (?:[^'",\#][^,\#]*?)    # unquoted
                         )
                         \s*,\s*                     # comma
                     )*      # match all list items ending in a comma (if any)
                 )
                 (
                     (?:".*?")|                      # double quotes
                     (?:'.*?')|                      # single quotes
                     (?:[^'",\#\s][^,]*?)|           # unquoted
                     (?:(?<!,))                      # Empty value
                 )?          # last item in a list - or string value
             )|
             (,)             # alternatively a single comma - empty list
         )
         \s*(\#.*)?          # optional comment
         $''',
         re.VERBOSE)
 
     # use findall to get the members of a list value
     _listvalueexp = re.compile(r'''
         (
             (?:".*?")|          # double quotes
             (?:'.*?')|          # single quotes
             (?:[^'",\#].*?)       # unquoted
         )
         \s*,\s*                 # comma
         ''',
         re.VERBOSE)
 
     # this regexp is used for the value
     # when lists are switched off
     _nolistvalue = re.compile(r'''^
         (
             (?:".*?")|          # double quotes
             (?:'.*?')|          # single quotes
             (?:[^'"\#].*?)|     # unquoted
             (?:)                # Empty value
         )
         \s*(\#.*)?              # optional comment
         $''',
         re.VERBOSE)
 
     # regexes for finding triple quoted values on one line
     _single_line_single = re.compile(r"^'''(.*?)'''\s*(#.*)?$")
     _single_line_double = re.compile(r'^"""(.*?)"""\s*(#.*)?$')
     _multi_line_single = re.compile(r"^(.*?)'''\s*(#.*)?$")
     _multi_line_double = re.compile(r'^(.*?)"""\s*(#.*)?$')
 
     _triple_quote = {
         "'''": (_single_line_single, _multi_line_single),
         '"""': (_single_line_double, _multi_line_double),
     }
 
     # Used by the ``istrue`` Section method
     _bools = {
         'yes': True, 'no': False,
         'on': True, 'off': False,
         '1': True, '0': False,
         'true': True, 'false': False,
         }
 
 
     def __init__(self, infile=None, options=None, **kwargs):
         """
         Parse a config file or create a config file object.
         
         ``ConfigObj(infile=None, options=None, **kwargs)``
         """
         # init the superclass
         Section.__init__(self, self, 0, self)
         
         if infile is None:
             infile = []
         if options is None:
             options = {}
         else:
             options = dict(options)
             
         # keyword arguments take precedence over an options dictionary
         options.update(kwargs)
         
         defaults = OPTION_DEFAULTS.copy()
         # TODO: check the values too.
         for entry in options:
             if entry not in defaults:
                 raise TypeError('Unrecognised option "%s".' % entry)
         
         # Add any explicit options to the defaults
         defaults.update(options)
         self._initialise(defaults)
         configspec = defaults['configspec']
         self._original_configspec = configspec
         self._load(infile, configspec)
         
         
     def _load(self, infile, configspec):
         if isinstance(infile, StringTypes):
             self.filename = infile
             if os.path.isfile(infile):
                 h = open(infile, 'rb')
                 infile = h.read() or []
                 h.close()
             elif self.file_error:
                 # raise an error if the file doesn't exist
                 raise IOError('Config file not found: "%s".' % self.filename)
             else:
                 # file doesn't already exist
                 if self.create_empty:
                     # this is a good test that the filename specified
                     # isn't impossible - like on a non-existent device
                     h = open(infile, 'w')
                     h.write('')
                     h.close()
                 infile = []
                 
         elif isinstance(infile, (list, tuple)):
             infile = list(infile)
             
         elif isinstance(infile, dict):
             # initialise self
             # the Section class handles creating subsections
             if isinstance(infile, ConfigObj):
                 # get a copy of our ConfigObj
                 infile = infile.dict()
                 
             for entry in infile:
                 self[entry] = infile[entry]
             del self._errors
             
             if configspec is not None:
                 self._handle_configspec(configspec)
             else:
                 self.configspec = None
             return
         
         elif hasattr(infile, 'read'):
             # This supports file like objects
             infile = infile.read() or []
             # needs splitting into lines - but needs doing *after* decoding
             # in case it's not an 8 bit encoding
         else:
             raise TypeError('infile must be a filename, file like object, or list of lines.')
         
         if infile:
             # don't do it for the empty ConfigObj
             infile = self._handle_bom(infile)
             # infile is now *always* a list
             #
             # Set the newlines attribute (first line ending it finds)
             # and strip trailing '\n' or '\r' from lines
             for line in infile:
                 if (not line) or (line[-1] not in ('\r', '\n', '\r\n')):
                     continue
                 for end in ('\r\n', '\n', '\r'):
                     if line.endswith(end):
                         self.newlines = end
                         break
                 break
 
             infile = [line.rstrip('\r\n') for line in infile]
             
         self._parse(infile)
         # if we had any errors, now is the time to raise them
         if self._errors:
             info = "at line %s." % self._errors[0].line_number
             if len(self._errors) > 1:
                 msg = "Parsing failed with several errors.\nFirst error %s" % info
                 error = ConfigObjError(msg)
             else:
                 error = self._errors[0]
             # set the errors attribute; it's a list of tuples:
             # (error_type, message, line_number)
             error.errors = self._errors
             # set the config attribute
             error.config = self
             raise error
         # delete private attributes
         del self._errors
         
         if configspec is None:
             self.configspec = None
         else:
             self._handle_configspec(configspec)
     
     
     def _initialise(self, options=None):
         if options is None:
             options = OPTION_DEFAULTS
             
         # initialise a few variables
         self.filename = None
         self._errors = []
         self.raise_errors = options['raise_errors']
         self.interpolation = options['interpolation']
         self.list_values = options['list_values']
         self.create_empty = options['create_empty']
         self.file_error = options['file_error']
         self.stringify = options['stringify']
         self.indent_type = options['indent_type']
         self.encoding = options['encoding']
         self.default_encoding = options['default_encoding']
         self.BOM = False
         self.newlines = None
         self.write_empty_values = options['write_empty_values']
         self.unrepr = options['unrepr']
         
         self.initial_comment = []
         self.final_comment = []
         self.configspec = {}
         
         # Clear section attributes as well
         Section._initialise(self)
         
         
     def __repr__(self):
         return ('ConfigObj({%s})' % 
                 ', '.join([('%s: %s' % (repr(key), repr(self[key]))) 
                 for key in (self.scalars + self.sections)]))
     
     
     def _handle_bom(self, infile):
         """
         Handle any BOM, and decode if necessary.
         
         If an encoding is specified, that *must* be used - but the BOM should
         still be removed (and the BOM attribute set).
         
         (If the encoding is wrongly specified, then a BOM for an alternative
         encoding won't be discovered or removed.)
         
         If an encoding is not specified, UTF8 or UTF16 BOM will be detected and
         removed. The BOM attribute will be set. UTF16 will be decoded to
         unicode.
         
         NOTE: This method must not be called with an empty ``infile``.
         
         Specifying the *wrong* encoding is likely to cause a
         ``UnicodeDecodeError``.
         
         ``infile`` must always be returned as a list of lines, but may be
         passed in as a single string.
         """
         if ((self.encoding is not None) and
             (self.encoding.lower() not in BOM_LIST)):
             # No need to check for a BOM
             # the encoding specified doesn't have one
             # just decode
             return self._decode(infile, self.encoding)
         
         if isinstance(infile, (list, tuple)):
             line = infile[0]
         else:
             line = infile
         if self.encoding is not None:
             # encoding explicitly supplied
             # And it could have an associated BOM
             # TODO: if encoding is just UTF16 - we ought to check for both
             # TODO: big endian and little endian versions.
             enc = BOM_LIST[self.encoding.lower()]
             if enc == 'utf_16':
                 # For UTF16 we try big endian and little endian
-                for BOM, (encoding, final_encoding) in BOMS.items():
+                for BOM, (encoding, final_encoding) in BOMS.iteritems():
                     if not final_encoding:
                         # skip UTF8
                         continue
                     if infile.startswith(BOM):
                         ### BOM discovered
                         ##self.BOM = True
                         # Don't need to remove BOM
                         return self._decode(infile, encoding)
                     
                 # If we get this far, will *probably* raise a DecodeError
                 # As it doesn't appear to start with a BOM
                 return self._decode(infile, self.encoding)
             
             # Must be UTF8
             BOM = BOM_SET[enc]
             if not line.startswith(BOM):
                 return self._decode(infile, self.encoding)
             
             newline = line[len(BOM):]
             
             # BOM removed
             if isinstance(infile, (list, tuple)):
                 infile[0] = newline
             else:
                 infile = newline
             self.BOM = True
             return self._decode(infile, self.encoding)
         
         # No encoding specified - so we need to check for UTF8/UTF16
-        for BOM, (encoding, final_encoding) in BOMS.items():
+        for BOM, (encoding, final_encoding) in BOMS.iteritems():
             if not line.startswith(BOM):
                 continue
             else:
                 # BOM discovered
                 self.encoding = final_encoding
                 if not final_encoding:
                     self.BOM = True
                     # UTF8
                     # remove BOM
                     newline = line[len(BOM):]
                     if isinstance(infile, (list, tuple)):
                         infile[0] = newline
                     else:
                         infile = newline
                     # UTF8 - don't decode
                     if isinstance(infile, StringTypes):
                         return infile.splitlines(True)
                     else:
                         return infile
                 # UTF16 - have to decode
                 return self._decode(infile, encoding)
             
         # No BOM discovered and no encoding specified, just return
         if isinstance(infile, StringTypes):
             # infile read from a file will be a single string
             return infile.splitlines(True)
         return infile
 
 
     def _a_to_u(self, aString):
         """Decode ASCII strings to unicode if a self.encoding is specified."""
         if self.encoding:
             return aString.decode('ascii')
         else:
             return aString
 
 
     def _decode(self, infile, encoding):
         """
         Decode infile to unicode. Using the specified encoding.
         
         if is a string, it also needs converting to a list.
         """
         if isinstance(infile, StringTypes):
             # can't be unicode
             # NOTE: Could raise a ``UnicodeDecodeError``
             return infile.decode(encoding).splitlines(True)
         for i, line in enumerate(infile):
             if not isinstance(line, unicode):
                 # NOTE: The isinstance test here handles mixed lists of unicode/string
                 # NOTE: But the decode will break on any non-string values
                 # NOTE: Or could raise a ``UnicodeDecodeError``
                 infile[i] = line.decode(encoding)
         return infile
 
 
     def _decode_element(self, line):
         """Decode element to unicode if necessary."""
         if not self.encoding:
             return line
         if isinstance(line, str) and self.default_encoding:
             return line.decode(self.default_encoding)
         return line
 
 
     def _str(self, value):
         """
         Used by ``stringify`` within validate, to turn non-string values
         into strings.
         """
         if not isinstance(value, StringTypes):
             return str(value)
         else:
             return value
 
 
     def _parse(self, infile):
         """Actually parse the config file."""
         temp_list_values = self.list_values
         if self.unrepr:
             self.list_values = False
             
         comment_list = []
         done_start = False
         this_section = self
         maxline = len(infile) - 1
         cur_index = -1
         reset_comment = False
         
         while cur_index < maxline:
             if reset_comment:
                 comment_list = []
             cur_index += 1
             line = infile[cur_index]
             sline = line.strip()
             # do we have anything on the line ?
             if not sline or sline.startswith('#'):
                 reset_comment = False
                 comment_list.append(line)
                 continue
             
             if not done_start:
                 # preserve initial comment
                 self.initial_comment = comment_list
                 comment_list = []
                 done_start = True
                 
             reset_comment = True
             # first we check if it's a section marker
             mat = self._sectionmarker.match(line)
             if mat is not None:
                 # is a section line
                 (indent, sect_open, sect_name, sect_close, comment) = mat.groups()
                 if indent and (self.indent_type is None):
                     self.indent_type = indent
                 cur_depth = sect_open.count('[')
                 if cur_depth != sect_close.count(']'):
                     self._handle_error("Cannot compute the section depth at line %s.",
                                        NestingError, infile, cur_index)
                     continue
                 
                 if cur_depth < this_section.depth:
                     # the new section is dropping back to a previous level
                     try:
                         parent = self._match_depth(this_section,
                                                    cur_depth).parent
                     except SyntaxError:
                         self._handle_error("Cannot compute nesting level at line %s.",
                                            NestingError, infile, cur_index)
                         continue
                 elif cur_depth == this_section.depth:
                     # the new section is a sibling of the current section
                     parent = this_section.parent
                 elif cur_depth == this_section.depth + 1:
                     # the new section is a child the current section
                     parent = this_section
                 else:
                     self._handle_error("Section too nested at line %s.",
                                        NestingError, infile, cur_index)
                     
                 sect_name = self._unquote(sect_name)
                 if parent.has_key(sect_name):
                     self._handle_error('Duplicate section name at line %s.',
                                        DuplicateError, infile, cur_index)
                     continue
                 
                 # create the new section
                 this_section = Section(
                     parent,
                     cur_depth,
                     self,
                     name=sect_name)
                 parent[sect_name] = this_section
                 parent.inline_comments[sect_name] = comment
                 parent.comments[sect_name] = comment_list
                 continue
             #
             # it's not a section marker,
             # so it should be a valid ``key = value`` line
             mat = self._keyword.match(line)
             if mat is None:
                 # it neither matched as a keyword
                 # or a section marker
                 self._handle_error(
                     'Invalid line at line "%s".',
                     ParseError, infile, cur_index)
             else:
                 # is a keyword value
                 # value will include any inline comment
                 (indent, key, value) = mat.groups()
                 if indent and (self.indent_type is None):
                     self.indent_type = indent
                 # check for a multiline value
                 if value[:3] in ['"""', "'''"]:
                     try:
                         (value, comment, cur_index) = self._multiline(
                             value, infile, cur_index, maxline)
                     except SyntaxError:
                         self._handle_error(
                             'Parse error in value at line %s.',
                             ParseError, infile, cur_index)
                         continue
                     else:
                         if self.unrepr:
                             comment = ''
                             try:
                                 value = unrepr(value)
                             except Exception, e:
                                 if type(e) == UnknownType:
                                     msg = 'Unknown name or type in value at line %s.'
                                 else:
                                     msg = 'Parse error in value at line %s.'
                                 self._handle_error(msg, UnreprError, infile,
                                     cur_index)
                                 continue
                 else:
                     if self.unrepr:
                         comment = ''
                         try:
                             value = unrepr(value)
                         except Exception, e:
                             if isinstance(e, UnknownType):
                                 msg = 'Unknown name or type in value at line %s.'
                             else:
                                 msg = 'Parse error in value at line %s.'
                             self._handle_error(msg, UnreprError, infile,
                                 cur_index)
                             continue
                     else:
                         # extract comment and lists
                         try:
                             (value, comment) = self._handle_value(value)
                         except SyntaxError:
                             self._handle_error(
                                 'Parse error in value at line %s.',
                                 ParseError, infile, cur_index)
                             continue
                 #
                 key = self._unquote(key)
                 if this_section.has_key(key):
                     self._handle_error(
                         'Duplicate keyword name at line %s.',
                         DuplicateError, infile, cur_index)
                     continue
                 # add the key.
                 # we set unrepr because if we have got this far we will never
                 # be creating a new section
                 this_section.__setitem__(key, value, unrepr=True)
                 this_section.inline_comments[key] = comment
                 this_section.comments[key] = comment_list
                 continue
         #
         if self.indent_type is None:
             # no indentation used, set the type accordingly
             self.indent_type = ''
 
         # preserve the final comment
         if not self and not self.initial_comment:
             self.initial_comment = comment_list
         elif not reset_comment:
             self.final_comment = comment_list
         self.list_values = temp_list_values
 
 
     def _match_depth(self, sect, depth):
         """
         Given a section and a depth level, walk back through the sections
         parents to see if the depth level matches a previous section.
         
         Return a reference to the right section,
         or raise a SyntaxError.
         """
         while depth < sect.depth:
             if sect is sect.parent:
                 # we've reached the top level already
                 raise SyntaxError()
             sect = sect.parent
         if sect.depth == depth:
             return sect
         # shouldn't get here
         raise SyntaxError()
 
 
     def _handle_error(self, text, ErrorClass, infile, cur_index):
         """
         Handle an error according to the error settings.
         
         Either raise the error or store it.
         The error will have occured at ``cur_index``
         """
         line = infile[cur_index]
         cur_index += 1
         message = text % cur_index
         error = ErrorClass(message, cur_index, line)
         if self.raise_errors:
             # raise the error - parsing stops here
             raise error
         # store the error
         # reraise when parsing has finished
         self._errors.append(error)
 
 
     def _unquote(self, value):
         """Return an unquoted version of a value"""
         if (value[0] == value[-1]) and (value[0] in ('"', "'")):
             value = value[1:-1]
         return value
 
 
     def _quote(self, value, multiline=True):
         """
         Return a safely quoted version of a value.
         
         Raise a ConfigObjError if the value cannot be safely quoted.
         If multiline is ``True`` (default) then use triple quotes
         if necessary.
         
         Don't quote values that don't need it.
         Recursively quote members of a list and return a comma joined list.
         Multiline is ``False`` for lists.
         Obey list syntax for empty and single member lists.
         
         If ``list_values=False`` then the value is only quoted if it contains
         a ``\n`` (is multiline) or '#'.
         
         If ``write_empty_values`` is set, and the value is an empty string, it
         won't be quoted.
         """
         if multiline and self.write_empty_values and value == '':
             # Only if multiline is set, so that it is used for values not
             # keys, and not values that are part of a list
             return ''
         
         if multiline and isinstance(value, (list, tuple)):
             if not value:
                 return ','
             elif len(value) == 1:
                 return self._quote(value[0], multiline=False) + ','
             return ', '.join([self._quote(val, multiline=False)
                 for val in value])
         if not isinstance(value, StringTypes):
             if self.stringify:
                 value = str(value)
             else:
                 raise TypeError('Value "%s" is not a string.' % value)
 
         if not value:
             return '""'
         
         no_lists_no_quotes = not self.list_values and '\n' not in value and '#' not in value
         need_triple = multiline and ((("'" in value) and ('"' in value)) or ('\n' in value ))
         hash_triple_quote = multiline and not need_triple and ("'" in value) and ('"' in value) and ('#' in value)
         check_for_single = (no_lists_no_quotes or not need_triple) and not hash_triple_quote
         
         if check_for_single:
             if not self.list_values:
                 # we don't quote if ``list_values=False``
                 quot = noquot
             # for normal values either single or double quotes will do
             elif '\n' in value:
                 # will only happen if multiline is off - e.g. '\n' in key
                 raise ConfigObjError('Value "%s" cannot be safely quoted.' % value)
             elif ((value[0] not in wspace_plus) and
                     (value[-1] not in wspace_plus) and
                     (',' not in value)):
                 quot = noquot
             else:
                 quot = self._get_single_quote(value)
         else:
             # if value has '\n' or "'" *and* '"', it will need triple quotes
             quot = self._get_triple_quote(value)
         
         if quot == noquot and '#' in value and self.list_values:
             quot = self._get_single_quote(value)
                 
         return quot % value
     
     
     def _get_single_quote(self, value):
         if ("'" in value) and ('"' in value):
             raise ConfigObjError('Value "%s" cannot be safely quoted.' % value)
         elif '"' in value:
             quot = squot
         else:
             quot = dquot
         return quot
     
     
     def _get_triple_quote(self, value):
         if (value.find('"""') != -1) and (value.find("'''") != -1):
             raise ConfigObjError('Value "%s" cannot be safely quoted.' % value)
         if value.find('"""') == -1:
             quot = tdquot
         else:
             quot = tsquot 
         return quot
 
 
     def _handle_value(self, value):
         """
         Given a value string, unquote, remove comment,
         handle lists. (including empty and single member lists)
         """
         # do we look for lists in values ?
         if not self.list_values:
             mat = self._nolistvalue.match(value)
             if mat is None:
                 raise SyntaxError()
             # NOTE: we don't unquote here
             return mat.groups()
         #
         mat = self._valueexp.match(value)
         if mat is None:
             # the value is badly constructed, probably badly quoted,
             # or an invalid list
             raise SyntaxError()
         (list_values, single, empty_list, comment) = mat.groups()
         if (list_values == '') and (single is None):
             # change this if you want to accept empty values
             raise SyntaxError()
         # NOTE: note there is no error handling from here if the regex
         # is wrong: then incorrect values will slip through
         if empty_list is not None:
             # the single comma - meaning an empty list
             return ([], comment)
         if single is not None:
             # handle empty values
             if list_values and not single:
                 # FIXME: the '' is a workaround because our regex now matches
                 #   '' at the end of a list if it has a trailing comma
                 single = None
             else:
                 single = single or '""'
                 single = self._unquote(single)
         if list_values == '':
             # not a list value
             return (single, comment)
         the_list = self._listvalueexp.findall(list_values)
         the_list = [self._unquote(val) for val in the_list]
         if single is not None:
             the_list += [single]
         return (the_list, comment)
 
 
     def _multiline(self, value, infile, cur_index, maxline):
         """Extract the value, where we are in a multiline situation."""
         quot = value[:3]
         newvalue = value[3:]
         single_line = self._triple_quote[quot][0]
         multi_line = self._triple_quote[quot][1]
         mat = single_line.match(value)
         if mat is not None:
             retval = list(mat.groups())
             retval.append(cur_index)
             return retval
         elif newvalue.find(quot) != -1:
             # somehow the triple quote is missing
             raise SyntaxError()
         #
         while cur_index < maxline:
             cur_index += 1
             newvalue += '\n'
             line = infile[cur_index]
             if line.find(quot) == -1:
                 newvalue += line
             else:
                 # end of multiline, process it
                 break
         else:
             # we've got to the end of the config, oops...
             raise SyntaxError()
         mat = multi_line.match(line)
         if mat is None:
             # a badly formed line
             raise SyntaxError()
         (value, comment) = mat.groups()
         return (newvalue + value, comment, cur_index)
 
 
     def _handle_configspec(self, configspec):
         """Parse the configspec."""
         # FIXME: Should we check that the configspec was created with the 
         #        correct settings ? (i.e. ``list_values=False``)
         if not isinstance(configspec, ConfigObj):
             try:
                 configspec = ConfigObj(configspec,
                                        raise_errors=True,
                                        file_error=True,
                                        list_values=False)
             except ConfigObjError, e:
                 # FIXME: Should these errors have a reference
                 #        to the already parsed ConfigObj ?
                 raise ConfigspecError('Parsing configspec failed: %s' % e)
             except IOError, e:
                 raise IOError('Reading configspec failed: %s' % e)
         
         self._set_configspec_value(configspec, self)
 
 
     def _set_configspec_value(self, configspec, section):
         """Used to recursively set configspec values."""
         if '__many__' in configspec.sections:
             section.configspec['__many__'] = configspec['__many__']
             if len(configspec.sections) > 1:
                 # FIXME: can we supply any useful information here ?
                 raise RepeatSectionError()
             
         if hasattr(configspec, 'initial_comment'):
             section._configspec_initial_comment = configspec.initial_comment
             section._configspec_final_comment = configspec.final_comment
             section._configspec_encoding = configspec.encoding
             section._configspec_BOM = configspec.BOM
             section._configspec_newlines = configspec.newlines
             section._configspec_indent_type = configspec.indent_type
             
         for entry in configspec.scalars:
             section._configspec_comments[entry] = configspec.comments[entry]
             section._configspec_inline_comments[entry] = configspec.inline_comments[entry]
             section.configspec[entry] = configspec[entry]
             section._order.append(entry)
             
         for entry in configspec.sections:
             if entry == '__many__':
                 continue
             
             section._cs_section_comments[entry] = configspec.comments[entry]
             section._cs_section_inline_comments[entry] = configspec.inline_comments[entry]
             if not section.has_key(entry):
                 section[entry] = {}
             self._set_configspec_value(configspec[entry], section[entry])
 
 
     def _handle_repeat(self, section, configspec):
         """Dynamically assign configspec for repeated section."""
         try:
             section_keys = configspec.sections
             scalar_keys = configspec.scalars
         except AttributeError:
             section_keys = [entry for entry in configspec 
                                 if isinstance(configspec[entry], dict)]
             scalar_keys = [entry for entry in configspec 
                                 if not isinstance(configspec[entry], dict)]
             
         if '__many__' in section_keys and len(section_keys) > 1:
             # FIXME: can we supply any useful information here ?
             raise RepeatSectionError()
         
         scalars = {}
         sections = {}
         for entry in scalar_keys:
             val = configspec[entry]
             scalars[entry] = val
         for entry in section_keys:
             val = configspec[entry]
             if entry == '__many__':
                 scalars[entry] = val
                 continue
             sections[entry] = val
             
         section.configspec = scalars
         for entry in sections:
             if not section.has_key(entry):
                 section[entry] = {}
             self._handle_repeat(section[entry], sections[entry])
 
 
     def _write_line(self, indent_string, entry, this_entry, comment):
         """Write an individual line, for the write method"""
         # NOTE: the calls to self._quote here handles non-StringType values.
         if not self.unrepr:
             val = self._decode_element(self._quote(this_entry))
         else:
             val = repr(this_entry)
         return '%s%s%s%s%s' % (indent_string,
                                self._decode_element(self._quote(entry, multiline=False)),
                                self._a_to_u(' = '),
                                val,
                                self._decode_element(comment))
 
 
     def _write_marker(self, indent_string, depth, entry, comment):
         """Write a section marker line"""
         return '%s%s%s%s%s' % (indent_string,
                                self._a_to_u('[' * depth),
                                self._quote(self._decode_element(entry), multiline=False),
                                self._a_to_u(']' * depth),
                                self._decode_element(comment))
 
 
     def _handle_comment(self, comment):
         """Deal with a comment."""
         if not comment:
             return ''
         start = self.indent_type
         if not comment.startswith('#'):
             start += self._a_to_u(' # ')
         return (start + comment)
 
 
     # Public methods
 
     def write(self, outfile=None, section=None):
         """
         Write the current ConfigObj as a file
         
         tekNico: FIXME: use StringIO instead of real files
         
         >>> filename = a.filename
         >>> a.filename = 'test.ini'
         >>> a.write()
         >>> a.filename = filename
         >>> a == ConfigObj('test.ini', raise_errors=True)
         1
         """
         if self.indent_type is None:
             # this can be true if initialised from a dictionary
             self.indent_type = DEFAULT_INDENT_TYPE
             
         out = []
         cs = self._a_to_u('#')
         csp = self._a_to_u('# ')
         if section is None:
             int_val = self.interpolation
             self.interpolation = False
             section = self
             for line in self.initial_comment:
                 line = self._decode_element(line)
                 stripped_line = line.strip()
                 if stripped_line and not stripped_line.startswith(cs):
                     line = csp + line
                 out.append(line)
                 
         indent_string = self.indent_type * section.depth
         for entry in (section.scalars + section.sections):
             if entry in section.defaults:
                 # don't write out default values
                 continue
             for comment_line in section.comments[entry]:
                 comment_line = self._decode_element(comment_line.lstrip())
                 if comment_line and not comment_line.startswith(cs):
                     comment_line = csp + comment_line
                 out.append(indent_string + comment_line)
             this_entry = section[entry]
             comment = self._handle_comment(section.inline_comments[entry])
             
             if isinstance(this_entry, dict):
                 # a section
                 out.append(self._write_marker(
                     indent_string,
                     this_entry.depth,
                     entry,
                     comment))
                 out.extend(self.write(section=this_entry))
             else:
                 out.append(self._write_line(
                     indent_string,
                     entry,
                     this_entry,
                     comment))
                 
         if section is self:
             for line in self.final_comment:
                 line = self._decode_element(line)
                 stripped_line = line.strip()
                 if stripped_line and not stripped_line.startswith(cs):
                     line = csp + line
                 out.append(line)
             self.interpolation = int_val
             
         if section is not self:
             return out
         
         if (self.filename is None) and (outfile is None):
             # output a list of lines
             # might need to encode
             # NOTE: This will *screw* UTF16, each line will start with the BOM
             if self.encoding:
                 out = [l.encode(self.encoding) for l in out]
             if (self.BOM and ((self.encoding is None) or
                 (BOM_LIST.get(self.encoding.lower()) == 'utf_8'))):
                 # Add the UTF8 BOM
                 if not out:
                     out.append('')
                 out[0] = BOM_UTF8 + out[0]
             return out
         
         # Turn the list to a string, joined with correct newlines
         newline = self.newlines or os.linesep
         output = self._a_to_u(newline).join(out)
         if self.encoding:
             output = output.encode(self.encoding)
         if self.BOM and ((self.encoding is None) or match_utf8(self.encoding)):
             # Add the UTF8 BOM
             output = BOM_UTF8 + output
             
         if not output.endswith(newline):
             output += newline
         if outfile is not None:
             outfile.write(output)
         else:
             h = open(self.filename, 'wb')
             h.write(output)
             h.close()
 
 
     def validate(self, validator, preserve_errors=False, copy=False,
                  section=None):
         """
         Test the ConfigObj against a configspec.
         
         It uses the ``validator`` object from *validate.py*.
         
         To run ``validate`` on the current ConfigObj, call: ::
         
             test = config.validate(validator)
         
         (Normally having previously passed in the configspec when the ConfigObj
         was created - you can dynamically assign a dictionary of checks to the
         ``configspec`` attribute of a section though).
         
         It returns ``True`` if everything passes, or a dictionary of
         pass/fails (True/False). If every member of a subsection passes, it
         will just have the value ``True``. (It also returns ``False`` if all
         members fail).
         
         In addition, it converts the values from strings to their native
         types if their checks pass (and ``stringify`` is set).
         
         If ``preserve_errors`` is ``True`` (``False`` is default) then instead
         of a marking a fail with a ``False``, it will preserve the actual
         exception object. This can contain info about the reason for failure.
         For example the ``VdtValueTooSmallError`` indicates that the value
         supplied was too small. If a value (or section) is missing it will
         still be marked as ``False``.
         
         You must have the validate module to use ``preserve_errors=True``.
         
         You can then use the ``flatten_errors`` function to turn your nested
         results dictionary into a flattened list of failures - useful for
         displaying meaningful error messages.
         """
         if section is None:
             if self.configspec is None:
                 raise ValueError('No configspec supplied.')
             if preserve_errors:
                 # We do this once to remove a top level dependency on the validate module
                 # Which makes importing configobj faster
                 from validate import VdtMissingValue
                 self._vdtMissingValue = VdtMissingValue
             section = self
         #
         spec_section = section.configspec
         if copy and hasattr(section, '_configspec_initial_comment'):
             section.initial_comment = section._configspec_initial_comment
             section.final_comment = section._configspec_final_comment
             section.encoding = section._configspec_encoding
             section.BOM = section._configspec_BOM
             section.newlines = section._configspec_newlines
             section.indent_type = section._configspec_indent_type
             
         if '__many__' in section.configspec:
             many = spec_section['__many__']
             # dynamically assign the configspecs
             # for the sections below
             for entry in section.sections:
                 self._handle_repeat(section[entry], many)
         #
         out = {}
         ret_true = True
         ret_false = True
         order = [k for k in section._order if k in spec_section]
         order += [k for k in spec_section if k not in order]
         for entry in order:
             if entry == '__many__':
                 continue
             if (not entry in section.scalars) or (entry in section.defaults):
                 # missing entries
                 # or entries from defaults
                 missing = True
                 val = None
                 if copy and not entry in section.scalars:
                     # copy comments
                     section.comments[entry] = (
                         section._configspec_comments.get(entry, []))
                     section.inline_comments[entry] = (
                         section._configspec_inline_comments.get(entry, ''))
                 #
             else:
                 missing = False
                 val = section[entry]
             try:
                 check = validator.check(spec_section[entry],
                                         val,
                                         missing=missing
                                         )
             except validator.baseErrorClass, e:
                 if not preserve_errors or isinstance(e, self._vdtMissingValue):
                     out[entry] = False
                 else:
                     # preserve the error
                     out[entry] = e
                     ret_false = False
                 ret_true = False
             else:
                 try: 
                     section.default_values.pop(entry, None)
                 except AttributeError: 
                     # For Python 2.2 compatibility
                     try:
                         del section.default_values[entry]
                     except KeyError:
                         pass
                     
                 if hasattr(validator, 'get_default_value'):
                     try: 
                         section.default_values[entry] = validator.get_default_value(spec_section[entry])
                     except KeyError:
                         # No default
                         pass
                     
                 ret_false = False
                 out[entry] = True
                 if self.stringify or missing:
                     # if we are doing type conversion
                     # or the value is a supplied default
                     if not self.stringify:
                         if isinstance(check, (list, tuple)):
                             # preserve lists
                             check = [self._str(item) for item in check]
                         elif missing and check is None:
                             # convert the None from a default to a ''
                             check = ''
                         else:
                             check = self._str(check)
                     if (check != val) or missing:
                         section[entry] = check
                 if not copy and missing and entry not in section.defaults:
                     section.defaults.append(entry)
         # Missing sections will have been created as empty ones when the
         # configspec was read.
         for entry in section.sections:
             # FIXME: this means DEFAULT is not copied in copy mode
             if section is self and entry == 'DEFAULT':
                 continue
             if copy:
                 section.comments[entry] = section._cs_section_comments[entry]
                 section.inline_comments[entry] = (
                     section._cs_section_inline_comments[entry])
             check = self.validate(validator, preserve_errors=preserve_errors,
                 copy=copy, section=section[entry])
             out[entry] = check
             if check == False:
                 ret_true = False
             elif check == True:
                 ret_false = False
             else:
                 ret_true = False
                 ret_false = False
         #
         if ret_true:
             return True
         elif ret_false:
             return False
         return out
 
 
     def reset(self):
         """Clear ConfigObj instance and restore to 'freshly created' state."""
         self.clear()
         self._initialise()
         # FIXME: Should be done by '_initialise', but ConfigObj constructor (and reload)
         #        requires an empty dictionary
         self.configspec = None
         # Just to be sure ;-)
         self._original_configspec = None
         
         
     def reload(self):
         """
         Reload a ConfigObj from file.
         
         This method raises a ``ReloadError`` if the ConfigObj doesn't have
         a filename attribute pointing to a file.
         """
         if not isinstance(self.filename, StringTypes):
             raise ReloadError()
 
         filename = self.filename
         current_options = {}
         for entry in OPTION_DEFAULTS:
             if entry == 'configspec':
                 continue
             current_options[entry] = getattr(self, entry)
             
         configspec = self._original_configspec
         current_options['configspec'] = configspec
             
         self.clear()
         self._initialise(current_options)
         self._load(filename, configspec)
         
 
 
 class SimpleVal(object):
     """
     A simple validator.
     Can be used to check that all members expected are present.
     
     To use it, provide a configspec with all your members in (the value given
     will be ignored). Pass an instance of ``SimpleVal`` to the ``validate``
     method of your ``ConfigObj``. ``validate`` will return ``True`` if all
     members are present, or a dictionary with True/False meaning
     present/missing. (Whole missing sections will be replaced with ``False``)
     """
     
     def __init__(self):
         self.baseErrorClass = ConfigObjError
     
     def check(self, check, member, missing=False):
         """A dummy check method, always returns the value unchanged."""
         if missing:
             raise self.baseErrorClass()
         return member
 
 
 # Check / processing functions for options
 def flatten_errors(cfg, res, levels=None, results=None):
     """
     An example function that will turn a nested dictionary of results
     (as returned by ``ConfigObj.validate``) into a flat list.
     
     ``cfg`` is the ConfigObj instance being checked, ``res`` is the results
     dictionary returned by ``validate``.
     
     (This is a recursive function, so you shouldn't use the ``levels`` or
     ``results`` arguments - they are used by the function.
     
     Returns a list of keys that failed. Each member of the list is a tuple :
     ::
     
         ([list of sections...], key, result)
     
     If ``validate`` was called with ``preserve_errors=False`` (the default)
     then ``result`` will always be ``False``.
 
     *list of sections* is a flattened list of sections that the key was found
     in.
     
     If the section was missing then key will be ``None``.
     
     If the value (or section) was missing then ``result`` will be ``False``.
     
     If ``validate`` was called with ``preserve_errors=True`` and a value
     was present, but failed the check, then ``result`` will be the exception
     object returned. You can use this as a string that describes the failure.
     
     For example *The value "3" is of the wrong type*.
     
     >>> import validate
     >>> vtor = validate.Validator()
     >>> my_ini = '''
     ...     option1 = True
     ...     [section1]
     ...     option1 = True
     ...     [section2]
     ...     another_option = Probably
     ...     [section3]
     ...     another_option = True
     ...     [[section3b]]
     ...     value = 3
     ...     value2 = a
     ...     value3 = 11
     ...     '''
     >>> my_cfg = '''
     ...     option1 = boolean()
     ...     option2 = boolean()
     ...     option3 = boolean(default=Bad_value)
     ...     [section1]
     ...     option1 = boolean()
     ...     option2 = boolean()
     ...     option3 = boolean(default=Bad_value)
     ...     [section2]
     ...     another_option = boolean()
     ...     [section3]
     ...     another_option = boolean()
     ...     [[section3b]]
     ...     value = integer
     ...     value2 = integer
     ...     value3 = integer(0, 10)
     ...         [[[section3b-sub]]]
     ...         value = string
     ...     [section4]
     ...     another_option = boolean()
     ...     '''
     >>> cs = my_cfg.split('\\n')
     >>> ini = my_ini.split('\\n')
     >>> cfg = ConfigObj(ini, configspec=cs)
     >>> res = cfg.validate(vtor, preserve_errors=True)
     >>> errors = []
     >>> for entry in flatten_errors(cfg, res):
     ...     section_list, key, error = entry
     ...     section_list.insert(0, '[root]')
     ...     if key is not None:
     ...        section_list.append(key)
     ...     else:
     ...         section_list.append('[missing]')
     ...     section_string = ', '.join(section_list)
     ...     errors.append((section_string, ' = ', error))
     >>> errors.sort()
     >>> for entry in errors:
     ...     print entry[0], entry[1], (entry[2] or 0)
     [root], option2  =  0
     [root], option3  =  the value "Bad_value" is of the wrong type.
     [root], section1, option2  =  0
     [root], section1, option3  =  the value "Bad_value" is of the wrong type.
     [root], section2, another_option  =  the value "Probably" is of the wrong type.
     [root], section3, section3b, section3b-sub, [missing]  =  0
     [root], section3, section3b, value2  =  the value "a" is of the wrong type.
     [root], section3, section3b, value3  =  the value "11" is too big.
     [root], section4, [missing]  =  0
     """
     if levels is None:
         # first time called
         levels = []
         results = []
     if res is True:
         return results
     if res is False:
         results.append((levels[:], None, False))
         if levels:
             levels.pop()
         return results
-    for (key, val) in res.items():
+    for (key, val) in res.iteritems():
         if val == True:
             continue
         if isinstance(cfg.get(key), dict):
             # Go down one level
             levels.append(key)
             flatten_errors(cfg[key], val, levels, results)
             continue
         results.append((levels[:], key, val))
     #
     # Go up one level
     if levels:
         levels.pop()
     #
     return results
 
 
 """*A programming language is a medium of expression.* - Paul Graham"""

IPython/external/pretty.py

@@ -1,705 +1,692
 # -*- coding: utf-8 -*-
 """
     pretty
     ~~
 
     Python advanced pretty printer.  This pretty printer is intended to
     replace the old `pprint` python module which does not allow developers
     to provide their own pretty print callbacks.
 
     This module is based on ruby's `prettyprint.rb` library by `Tanaka Akira`.
 
 
     Example Usage
     =============
 
     To directly print the representation of an object use `pprint`::
 
         from pretty import pprint
         pprint(complex_object)
 
     To get a string of the output use `pretty`::
 
         from pretty import pretty
         string = pretty(complex_object)
 
 
     Extending
     =========
 
     The pretty library allows developers to add pretty printing rules for their
     own objects.  This process is straightforward.  All you have to do is to
     add a `__pretty__` method to your object and call the methods on the
     pretty printer passed::
 
         class MyObject(object):
 
             def __pretty__(self, p, cycle):
                 ...
 
     Depending on the python version you want to support you have two
     possibilities.  The following list shows the python 2.5 version and the
     compatibility one.
 
 
     Here the example implementation of a `__pretty__` method for a list
     subclass for python 2.5 and higher (python 2.5 requires the with statement
     __future__ import)::
 
         class MyList(list):
 
             def __pretty__(self, p, cycle):
                 if cycle:
                     p.text('MyList(...)')
                 else:
                     with p.group(8, 'MyList([', '])'):
                         for idx, item in enumerate(self):
                             if idx:
                                 p.text(',')
                                 p.breakable()
                             p.pretty(item)
 
     The `cycle` parameter is `True` if pretty detected a cycle.  You *have* to
     react to that or the result is an infinite loop.  `p.text()` just adds
     non breaking text to the output, `p.breakable()` either adds a whitespace
     or breaks here.  If you pass it an argument it's used instead of the
     default space.  `p.pretty` prettyprints another object using the pretty print
     method.
 
     The first parameter to the `group` function specifies the extra indentation
     of the next line.  In this example the next item will either be not
     breaked (if the items are short enough) or aligned with the right edge of
     the opening bracked of `MyList`.
 
     If you want to support python 2.4 and lower you can use this code::
 
         class MyList(list):
 
             def __pretty__(self, p, cycle):
                 if cycle:
                     p.text('MyList(...)')
                 else:
                     p.begin_group(8, 'MyList([')
                     for idx, item in enumerate(self):
                         if idx:
                             p.text(',')
                             p.breakable()
                         p.pretty(item)
                     p.end_group(8, '])')
 
     If you just want to indent something you can use the group function
     without open / close parameters.  Under python 2.5 you can also use this
     code::
 
         with p.indent(2):
             ...
 
     Or under python2.4 you might want to modify ``p.indentation`` by hand but
     this is rather ugly.
 
     :copyright: 2007 by Armin Ronacher.
                 Portions (c) 2009 by Robert Kern.
     :license: BSD License.
 """
 import __future__
 import sys
 import types
 import re
 import datetime
 from StringIO import StringIO
 from collections import deque
 
 
 __all__ = ['pretty', 'pprint', 'PrettyPrinter', 'RepresentationPrinter',
     'for_type', 'for_type_by_name']
 
 
 _re_pattern_type = type(re.compile(''))
 
 
 def pretty(obj, verbose=False, max_width=79, newline='\n'):
     """
     Pretty print the object's representation.
     """
     stream = StringIO()
     printer = RepresentationPrinter(stream, verbose, max_width, newline)
     printer.pretty(obj)
     printer.flush()
     return stream.getvalue()
 
 
 def pprint(obj, verbose=False, max_width=79, newline='\n'):
     """
     Like `pretty` but print to stdout.
     """
     printer = RepresentationPrinter(sys.stdout, verbose, max_width, newline)
     printer.pretty(obj)
     printer.flush()
     sys.stdout.write(newline)
     sys.stdout.flush()
 
-
-# add python2.5 context managers if we have the with statement feature
-if hasattr(__future__, 'with_statement'): exec '''
 from __future__ import with_statement
 from contextlib import contextmanager
 
 class _PrettyPrinterBase(object):
 
     @contextmanager
     def indent(self, indent):
         """with statement support for indenting/dedenting."""
         self.indentation += indent
         try:
             yield
         finally:
             self.indentation -= indent
 
     @contextmanager
     def group(self, indent=0, open='', close=''):
         """like begin_group / end_group but for the with statement."""
         self.begin_group(indent, open)
         try:
             with self.indent(indent):
                 yield
         finally:
             self.end_group(indent, close)
-'''
-else:
-    class _PrettyPrinterBase(object):
-
-        def _unsupported(self, *a, **kw):
-            """unsupported operation"""
-            raise RuntimeError('not available in this python version')
-        group = indent = _unsupported
-        del _unsupported
-
 
 class PrettyPrinter(_PrettyPrinterBase):
     """
     Baseclass for the `RepresentationPrinter` prettyprinter that is used to
     generate pretty reprs of objects.  Contrary to the `RepresentationPrinter`
     this printer knows nothing about the default pprinters or the `__pretty__`
     callback method.
     """
 
     def __init__(self, output, max_width=79, newline='\n'):
         self.output = output
         self.max_width = max_width
         self.newline = newline
         self.output_width = 0
         self.buffer_width = 0
         self.buffer = deque()
 
         root_group = Group(0)
         self.group_stack = [root_group]
         self.group_queue = GroupQueue(root_group)
         self.indentation = 0
 
     def _break_outer_groups(self):
         while self.max_width < self.output_width + self.buffer_width:
             group = self.group_queue.deq()
             if not group:
                 return
             while group.breakables:
                 x = self.buffer.popleft()
                 self.output_width = x.output(self.output, self.output_width)
                 self.buffer_width -= x.width
             while self.buffer and isinstance(self.buffer[0], Text):
                 x = self.buffer.popleft()
                 self.output_width = x.output(self.output, self.output_width)
                 self.buffer_width -= x.width
 
     def text(self, obj):
         """Add literal text to the output."""
         width = len(obj)
         if self.buffer:
             text = self.buffer[-1]
             if not isinstance(text, Text):
                 text = Text()
                 self.buffer.append(text)
             text.add(obj, width)
             self.buffer_width += width
             self._break_outer_groups()
         else:
             self.output.write(obj)
             self.output_width += width
 
     def breakable(self, sep=' '):
         """
         Add a breakable separator to the output.  This does not mean that it
         will automatically break here.  If no breaking on this position takes
         place the `sep` is inserted which default to one space.
         """
         width = len(sep)
         group = self.group_stack[-1]
         if group.want_break:
             self.flush()
             self.output.write(self.newline)
             self.output.write(' ' * self.indentation)
             self.output_width = self.indentation
             self.buffer_width = 0
         else:
             self.buffer.append(Breakable(sep, width, self))
             self.buffer_width += width
             self._break_outer_groups()
 
 
     def begin_group(self, indent=0, open=''):
         """
         Begin a group.  If you want support for python < 2.5 which doesn't has
         the with statement this is the preferred way:
 
             p.begin_group(1, '{')
             ...
             p.end_group(1, '}')
 
         The python 2.5 expression would be this:
 
             with p.group(1, '{', '}'):
                 ...
 
         The first parameter specifies the indentation for the next line (usually
         the width of the opening text), the second the opening text.  All
         parameters are optional.
         """
         if open:
             self.text(open)
         group = Group(self.group_stack[-1].depth + 1)
         self.group_stack.append(group)
         self.group_queue.enq(group)
         self.indentation += indent
 
     def end_group(self, dedent=0, close=''):
         """End a group. See `begin_group` for more details."""
         self.indentation -= dedent
         group = self.group_stack.pop()
         if not group.breakables:
             self.group_queue.remove(group)
         if close:
             self.text(close)
 
     def flush(self):
         """Flush data that is left in the buffer."""
         for data in self.buffer:
             self.output_width += data.output(self.output, self.output_width)
         self.buffer.clear()
         self.buffer_width = 0
 
 
 def _get_mro(obj_class):
     """ Get a reasonable method resolution order of a class and its superclasses
     for both old-style and new-style classes.
     """
     if not hasattr(obj_class, '__mro__'):
         # Old-style class. Mix in object to make a fake new-style class.
         try:
             obj_class = type(obj_class.__name__, (obj_class, object), {})
         except TypeError:
             # Old-style extension type that does not descend from object.
             # FIXME: try to construct a more thorough MRO.
             mro = [obj_class]
         else:
             mro = obj_class.__mro__[1:-1]
     else:
         mro = obj_class.__mro__
     return mro
 
 
 class RepresentationPrinter(PrettyPrinter):
     """
     Special pretty printer that has a `pretty` method that calls the pretty
     printer for a python object.
 
     This class stores processing data on `self` so you must *never* use
     this class in a threaded environment.  Always lock it or reinstanciate
     it.
 
     Instances also have a verbose flag callbacks can access to control their
     output.  For example the default instance repr prints all attributes and
     methods that are not prefixed by an underscore if the printer is in
     verbose mode.
     """
 
     def __init__(self, output, verbose=False, max_width=79, newline='\n'):
         PrettyPrinter.__init__(self, output, max_width, newline)
         self.verbose = verbose
         self.stack = []
 
     def pretty(self, obj):
         """Pretty print the given object."""
         obj_id = id(obj)
         cycle = obj_id in self.stack
         self.stack.append(obj_id)
         self.begin_group()
         try:
             obj_class = getattr(obj, '__class__', None) or type(obj)
             if hasattr(obj_class, '__pretty__'):
                 return obj_class.__pretty__(obj, self, cycle)
             try:
                 printer = _singleton_pprinters[obj_id]
             except (TypeError, KeyError):
                 pass
             else:
                 return printer(obj, self, cycle)
             for cls in _get_mro(obj_class):
                 if cls in _type_pprinters:
                     return _type_pprinters[cls](obj, self, cycle)
                 else:
                     printer = self._in_deferred_types(cls)
                     if printer is not None:
                         return printer(obj, self, cycle)
             return _default_pprint(obj, self, cycle)
         finally:
             self.end_group()
             self.stack.pop()
 
     def _in_deferred_types(self, cls):
         """
         Check if the given class is specified in the deferred type registry.
 
         Returns the printer from the registry if it exists, and None if the
         class is not in the registry. Successful matches will be moved to the
         regular type registry for future use.
         """
         mod = getattr(cls, '__module__', None)
         name = getattr(cls, '__name__', None)
         key = (mod, name)
         printer = None
         if key in _deferred_type_pprinters:
             # Move the printer over to the regular registry.
             printer = _deferred_type_pprinters.pop(key)
             _type_pprinters[cls] = printer
         return printer
 
 
 
 class Printable(object):
 
     def output(self, stream, output_width):
         return output_width
 
 
 class Text(Printable):
 
     def __init__(self):
         self.objs = []
         self.width = 0
 
     def output(self, stream, output_width):
         for obj in self.objs:
             stream.write(obj)
         return output_width + self.width
 
     def add(self, obj, width):
         self.objs.append(obj)
         self.width += width
 
 
 class Breakable(Printable):
 
     def __init__(self, seq, width, pretty):
         self.obj = seq
         self.width = width
         self.pretty = pretty
         self.indentation = pretty.indentation
         self.group = pretty.group_stack[-1]
         self.group.breakables.append(self)
 
     def output(self, stream, output_width):
         self.group.breakables.popleft()
         if self.group.want_break:
             stream.write(self.pretty.newline)
             stream.write(' ' * self.indentation)
             return self.indentation
         if not self.group.breakables:
             self.pretty.group_queue.remove(self.group)
         stream.write(self.obj)
         return output_width + self.width
 
 
 class Group(Printable):
 
     def __init__(self, depth):
         self.depth = depth
         self.breakables = deque()
         self.want_break = False
 
 
 class GroupQueue(object):
 
     def __init__(self, *groups):
         self.queue = []
         for group in groups:
             self.enq(group)
 
     def enq(self, group):
         depth = group.depth
         while depth > len(self.queue) - 1:
             self.queue.append([])
         self.queue[depth].append(group)
 
     def deq(self):
         for stack in self.queue:
             for idx, group in enumerate(reversed(stack)):
                 if group.breakables:
                     del stack[idx]
                     group.want_break = True
                     return group
             for group in stack:
                 group.want_break = True
             del stack[:]
 
     def remove(self, group):
         try:
             self.queue[group.depth].remove(group)
         except ValueError:
             pass
 
 
 _baseclass_reprs = (object.__repr__, types.InstanceType.__repr__)
 
 
 def _default_pprint(obj, p, cycle):
     """
     The default print function.  Used if an object does not provide one and
     it's none of the builtin objects.
     """
     klass = getattr(obj, '__class__', None) or type(obj)
     if getattr(klass, '__repr__', None) not in _baseclass_reprs:
         # A user-provided repr.
         p.text(repr(obj))
         return
     p.begin_group(1, '<')
     p.pretty(klass)
     p.text(' at 0x%x' % id(obj))
     if cycle:
         p.text(' ...')
     elif p.verbose:
         first = True
         for key in dir(obj):
             if not key.startswith('_'):
                 try:
                     value = getattr(obj, key)
                 except AttributeError:
                     continue
                 if isinstance(value, types.MethodType):
                     continue
                 if not first:
                     p.text(',')
                 p.breakable()
                 p.text(key)
                 p.text('=')
                 step = len(key) + 1
                 p.indentation += step
                 p.pretty(value)
                 p.indentation -= step
                 first = False
     p.end_group(1, '>')
 
 
 def _seq_pprinter_factory(start, end):
     """
     Factory that returns a pprint function useful for sequences.  Used by
     the default pprint for tuples, dicts, lists, sets and frozensets.
     """
     def inner(obj, p, cycle):
         if cycle:
             return p.text(start + '...' + end)
         step = len(start)
         p.begin_group(step, start)
         for idx, x in enumerate(obj):
             if idx:
                 p.text(',')
                 p.breakable()
             p.pretty(x)
         if len(obj) == 1 and type(obj) is tuple:
             # Special case for 1-item tuples.
             p.text(',')
         p.end_group(step, end)
     return inner
 
 
 def _dict_pprinter_factory(start, end):
     """
     Factory that returns a pprint function used by the default pprint of
     dicts and dict proxies.
     """
     def inner(obj, p, cycle):
         if cycle:
             return p.text('{...}')
         p.begin_group(1, start)
         keys = obj.keys()
         try:
             keys.sort()
         except Exception, e:
             # Sometimes the keys don't sort.
             pass
         for idx, key in enumerate(keys):
             if idx:
                 p.text(',')
                 p.breakable()
             p.pretty(key)
             p.text(': ')
             p.pretty(obj[key])
         p.end_group(1, end)
     return inner
 
 
 def _super_pprint(obj, p, cycle):
     """The pprint for the super type."""
     p.begin_group(8, '<super: ')
     p.pretty(obj.__self_class__)
     p.text(',')
     p.breakable()
     p.pretty(obj.__self__)
     p.end_group(8, '>')
 
 
 def _re_pattern_pprint(obj, p, cycle):
     """The pprint function for regular expression patterns."""
     p.text('re.compile(')
     pattern = repr(obj.pattern)
     if pattern[:1] in 'uU':
         pattern = pattern[1:]
         prefix = 'ur'
     else:
         prefix = 'r'
     pattern = prefix + pattern.replace('\\\\', '\\')
     p.text(pattern)
     if obj.flags:
         p.text(',')
         p.breakable()
         done_one = False
         for flag in ('TEMPLATE', 'IGNORECASE', 'LOCALE', 'MULTILINE', 'DOTALL',
             'UNICODE', 'VERBOSE', 'DEBUG'):
             if obj.flags & getattr(re, flag):
                 if done_one:
                     p.text('|')
                 p.text('re.' + flag)
                 done_one = True
     p.text(')')
 
 
 def _type_pprint(obj, p, cycle):
     """The pprint for classes and types."""
     if obj.__module__ in ('__builtin__', 'exceptions'):
         name = obj.__name__
     else:
         name = obj.__module__ + '.' + obj.__name__
     p.text(name)
 
 
 def _repr_pprint(obj, p, cycle):
     """A pprint that just redirects to the normal repr function."""
     p.text(repr(obj))
 
 
 def _function_pprint(obj, p, cycle):
     """Base pprint for all functions and builtin functions."""
     if obj.__module__ in ('__builtin__', 'exceptions') or not obj.__module__:
         name = obj.__name__
     else:
         name = obj.__module__ + '.' + obj.__name__
     p.text('<function %s>' % name)
 
 
 def _exception_pprint(obj, p, cycle):
     """Base pprint for all exceptions."""
     if obj.__class__.__module__ == 'exceptions':
         name = obj.__class__.__name__
     else:
         name = '%s.%s' % (
             obj.__class__.__module__,
             obj.__class__.__name__
         )
     step = len(name) + 1
     p.begin_group(step, '(')
     for idx, arg in enumerate(getattr(obj, 'args', ())):
         if idx:
             p.text(',')
             p.breakable()
         p.pretty(arg)
     p.end_group(step, ')')
 
 
 #: the exception base
 try:
     _exception_base = BaseException
 except NameError:
     _exception_base = Exception
 
 
 #: printers for builtin types
 _type_pprinters = {
     int:                        _repr_pprint,
     long:                       _repr_pprint,
     float:                      _repr_pprint,
     str:                        _repr_pprint,
     unicode:                    _repr_pprint,
     tuple:                      _seq_pprinter_factory('(', ')'),
     list:                       _seq_pprinter_factory('[', ']'),
     dict:                       _dict_pprinter_factory('{', '}'),
     types.DictProxyType:        _dict_pprinter_factory('<dictproxy {', '}>'),
     set:                        _seq_pprinter_factory('set([', '])'),
     frozenset:                  _seq_pprinter_factory('frozenset([', '])'),
     super:                      _super_pprint,
     _re_pattern_type:           _re_pattern_pprint,
     type:                       _type_pprint,
     types.ClassType:            _type_pprint,
     types.FunctionType:         _function_pprint,
     types.BuiltinFunctionType:  _function_pprint,
     types.SliceType:            _repr_pprint,
     types.MethodType:           _repr_pprint,
     xrange:                     _repr_pprint,
     datetime.datetime:          _repr_pprint,
     datetime.timedelta:         _repr_pprint,
     _exception_base:            _exception_pprint
 }
 
 #: printers for types specified by name
 _deferred_type_pprinters = {
 }
 
 def for_type(typ, func):
     """
     Add a pretty printer for a given type.
     """
     oldfunc = _type_pprinters.get(typ, None)
     if func is not None:
         # To support easy restoration of old pprinters, we need to ignore Nones.
         _type_pprinters[typ] = func
     return oldfunc
 
 def for_type_by_name(type_module, type_name, func):
     """
     Add a pretty printer for a type specified by the module and name of a type
     rather than the type object itself.
     """
     key = (type_module, type_name)
     oldfunc = _deferred_type_pprinters.get(key, None)
     if func is not None:
         # To support easy restoration of old pprinters, we need to ignore Nones.
         _deferred_type_pprinters[key] = func
     return oldfunc
 
 
 #: printers for the default singletons
 _singleton_pprinters = dict.fromkeys(map(id, [None, True, False, Ellipsis,
                                       NotImplemented]), _repr_pprint)
 
 
 if __name__ == '__main__':
     from random import randrange
     class Foo(object):
         def __init__(self):
             self.foo = 1
             self.bar = re.compile(r'\s+')
             self.blub = dict.fromkeys(range(30), randrange(1, 40))
             self.hehe = 23424.234234
             self.list = ["blub", "blah", self]
 
         def get_foo(self):
             print "foo"
 
     pprint(Foo(), verbose=True)

IPython/external/pyparsing.py

@@ -1,3707 +1,3708
+# -*- coding: utf-8 -*-
 # module pyparsing.py
 #
 # Copyright (c) 2003-2009  Paul T. McGuire
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
 # "Software"), to deal in the Software without restriction, including
 # without limitation the rights to use, copy, modify, merge, publish,
 # distribute, sublicense, and/or sell copies of the Software, and to
 # permit persons to whom the Software is furnished to do so, subject to
 # the following conditions:
 #
 # The above copyright notice and this permission notice shall be
 # included in all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #
 #from __future__ import generators
 
 __doc__ = \
 """
 pyparsing module - Classes and methods to define and execute parsing grammars
 
 The pyparsing module is an alternative approach to creating and executing simple grammars,
 vs. the traditional lex/yacc approach, or the use of regular expressions.  With pyparsing, you
 don't need to learn a new syntax for defining grammars or matching expressions - the parsing module
 provides a library of classes that you use to construct the grammar directly in Python.
 
 Here is a program to parse "Hello, World!" (or any greeting of the form "<salutation>, <addressee>!")::
 
     from pyparsing import Word, alphas
 
     # define grammar of a greeting
     greet = Word( alphas ) + "," + Word( alphas ) + "!"
 
     hello = "Hello, World!"
     print hello, "->", greet.parseString( hello )
 
 The program outputs the following::
 
     Hello, World! -> ['Hello', ',', 'World', '!']
 
 The Python representation of the grammar is quite readable, owing to the self-explanatory
 class names, and the use of '+', '|' and '^' operators.
 
 The parsed results returned from parseString() can be accessed as a nested list, a dictionary, or an
 object with named attributes.
 
 The pyparsing module handles some of the problems that are typically vexing when writing text parsers:
  - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello  ,  World  !", etc.)
  - quoted strings
  - embedded comments
 """
 
 __version__ = "1.5.2"
 __versionTime__ = "17 February 2009 19:45"
 __author__ = "Paul McGuire <[e-mail]>"
 
 import string
 from weakref import ref as wkref
 import copy
 import sys
 import warnings
 import re
 import sre_constants
 #~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) )
 
 __all__ = [
 'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty',
 'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal',
 'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or',
 'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException',
 'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException',
 'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', 'Upcase',
 'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore',
 'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col',
 'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString',
 'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'getTokensEndLoc', 'hexnums',
 'htmlComment', 'javaStyleComment', 'keepOriginalText', 'line', 'lineEnd', 'lineStart', 'lineno',
 'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral',
 'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables',
 'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', 
 'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd',
 'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute',
 'indentedBlock', 'originalTextFor',
 ]
 
 
 """
 Detect if we are running version 3.X and make appropriate changes
 Robert A. Clark
 """
 if sys.version_info[0] > 2:
     _PY3K = True
     _MAX_INT = sys.maxsize
     basestring = str
 else:
     _PY3K = False
     _MAX_INT = sys.maxint
 
 if not _PY3K:
     def _ustr(obj):
         """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries
            str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It
            then < returns the unicode object | encodes it with the default encoding | ... >.
         """
         if isinstance(obj,unicode):
             return obj
 
         try:
             # If this works, then _ustr(obj) has the same behaviour as str(obj), so
             # it won't break any existing code.
             return str(obj)
 
         except UnicodeEncodeError:
             # The Python docs (http://docs.python.org/ref/customization.html#l2h-182)
             # state that "The return value must be a string object". However, does a
             # unicode object (being a subclass of basestring) count as a "string
             # object"?
             # If so, then return a unicode object:
             return unicode(obj)
             # Else encode it... but how? There are many choices... :)
             # Replace unprintables with escape codes?
             #return unicode(obj).encode(sys.getdefaultencoding(), 'backslashreplace_errors')
             # Replace unprintables with question marks?
             #return unicode(obj).encode(sys.getdefaultencoding(), 'replace')
             # ...
 else:
     _ustr = str
     unichr = chr
 
 if not _PY3K:
 	def _str2dict(strg):
 	    return dict( [(c,0) for c in strg] )
 else:
 	_str2dict = set
 
 def _xml_escape(data):
     """Escape &, <, >, ", ', etc. in a string of data."""
 
     # ampersand must be replaced first
     from_symbols = '&><"\''
     to_symbols = ['&'+s+';' for s in "amp gt lt quot apos".split()]
     for from_,to_ in zip(from_symbols, to_symbols):
         data = data.replace(from_, to_)
     return data
 
 class _Constants(object):
     pass
 
 if not _PY3K:
     alphas     = string.lowercase + string.uppercase
 else:
     alphas     = string.ascii_lowercase + string.ascii_uppercase
 nums       = string.digits
 hexnums    = nums + "ABCDEFabcdef"
 alphanums  = alphas + nums
 _bslash = chr(92)
 printables = "".join( [ c for c in string.printable if c not in string.whitespace ] )
 
 class ParseBaseException(Exception):
     """base exception class for all parsing runtime exceptions"""
     # Performance tuning: we construct a *lot* of these, so keep this
     # constructor as small and fast as possible
     def __init__( self, pstr, loc=0, msg=None, elem=None ):
         self.loc = loc
         if msg is None:
             self.msg = pstr
             self.pstr = ""
         else:
             self.msg = msg
             self.pstr = pstr
         self.parserElement = elem
 
     def __getattr__( self, aname ):
         """supported attributes by name are:
             - lineno - returns the line number of the exception text
             - col - returns the column number of the exception text
             - line - returns the line containing the exception text
         """
         if( aname == "lineno" ):
             return lineno( self.loc, self.pstr )
         elif( aname in ("col", "column") ):
             return col( self.loc, self.pstr )
         elif( aname == "line" ):
             return line( self.loc, self.pstr )
         else:
             raise AttributeError(aname)
 
     def __str__( self ):
         return "%s (at char %d), (line:%d, col:%d)" % \
                 ( self.msg, self.loc, self.lineno, self.column )
     def __repr__( self ):
         return _ustr(self)
     def markInputline( self, markerString = ">!<" ):
         """Extracts the exception line from the input string, and marks
            the location of the exception with a special symbol.
         """
         line_str = self.line
         line_column = self.column - 1
         if markerString:
             line_str = "".join( [line_str[:line_column],
                                 markerString, line_str[line_column:]])
         return line_str.strip()
     def __dir__(self):
         return "loc msg pstr parserElement lineno col line " \
                "markInputLine __str__ __repr__".split()
 
 class ParseException(ParseBaseException):
     """exception thrown when parse expressions don't match class;
        supported attributes by name are:
         - lineno - returns the line number of the exception text
         - col - returns the column number of the exception text
         - line - returns the line containing the exception text
     """
     pass
 
 class ParseFatalException(ParseBaseException):
     """user-throwable exception thrown when inconsistent parse content
        is found; stops all parsing immediately"""
     pass
 
 class ParseSyntaxException(ParseFatalException):
     """just like ParseFatalException, but thrown internally when an
        ErrorStop indicates that parsing is to stop immediately because
        an unbacktrackable syntax error has been found"""
     def __init__(self, pe):
         super(ParseSyntaxException, self).__init__(
                                     pe.pstr, pe.loc, pe.msg, pe.parserElement)
 
 #~ class ReparseException(ParseBaseException):
     #~ """Experimental class - parse actions can raise this exception to cause
        #~ pyparsing to reparse the input string:
         #~ - with a modified input string, and/or
         #~ - with a modified start location
        #~ Set the values of the ReparseException in the constructor, and raise the
        #~ exception in a parse action to cause pyparsing to use the new string/location.
        #~ Setting the values as None causes no change to be made.
        #~ """
     #~ def __init_( self, newstring, restartLoc ):
         #~ self.newParseText = newstring
         #~ self.reparseLoc = restartLoc
 
 class RecursiveGrammarException(Exception):
     """exception thrown by validate() if the grammar could be improperly recursive"""
     def __init__( self, parseElementList ):
         self.parseElementTrace = parseElementList
 
     def __str__( self ):
         return "RecursiveGrammarException: %s" % self.parseElementTrace
 
 class _ParseResultsWithOffset(object):
     def __init__(self,p1,p2):
         self.tup = (p1,p2)
     def __getitem__(self,i):
         return self.tup[i]
     def __repr__(self):
         return repr(self.tup)
     def setOffset(self,i):
         self.tup = (self.tup[0],i)
 
 class ParseResults(object):
     """Structured parse results, to provide multiple means of access to the parsed data:
        - as a list (len(results))
        - by list index (results[0], results[1], etc.)
        - by attribute (results.<resultsName>)
        """
     __slots__ = ( "__toklist", "__tokdict", "__doinit", "__name", "__parent", "__accumNames", "__weakref__" )
     def __new__(cls, toklist, name=None, asList=True, modal=True ):
         if isinstance(toklist, cls):
             return toklist
         retobj = object.__new__(cls)
         retobj.__doinit = True
         return retobj
 
     # Performance tuning: we construct a *lot* of these, so keep this
     # constructor as small and fast as possible
     def __init__( self, toklist, name=None, asList=True, modal=True ):
         if self.__doinit:
             self.__doinit = False
             self.__name = None
             self.__parent = None
             self.__accumNames = {}
             if isinstance(toklist, list):
                 self.__toklist = toklist[:]
             else:
                 self.__toklist = [toklist]
             self.__tokdict = dict()
 
         if name:
             if not modal:
                 self.__accumNames[name] = 0
             if isinstance(name,int):
                 name = _ustr(name) # will always return a str, but use _ustr for consistency
             self.__name = name
             if not toklist in (None,'',[]):
                 if isinstance(toklist,basestring):
                     toklist = [ toklist ]
                 if asList:
                     if isinstance(toklist,ParseResults):
                         self[name] = _ParseResultsWithOffset(toklist.copy(),0)
                     else:
                         self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]),0)
                     self[name].__name = name
                 else:
                     try:
                         self[name] = toklist[0]
                     except (KeyError,TypeError,IndexError):
                         self[name] = toklist
 
     def __getitem__( self, i ):
         if isinstance( i, (int,slice) ):
             return self.__toklist[i]
         else:
             if i not in self.__accumNames:
                 return self.__tokdict[i][-1][0]
             else:
                 return ParseResults([ v[0] for v in self.__tokdict[i] ])
 
     def __setitem__( self, k, v ):
         if isinstance(v,_ParseResultsWithOffset):
             self.__tokdict[k] = self.__tokdict.get(k,list()) + [v]
             sub = v[0]
         elif isinstance(k,int):
             self.__toklist[k] = v
             sub = v
         else:
             self.__tokdict[k] = self.__tokdict.get(k,list()) + [_ParseResultsWithOffset(v,0)]
             sub = v
         if isinstance(sub,ParseResults):
             sub.__parent = wkref(self)
 
     def __delitem__( self, i ):
         if isinstance(i,(int,slice)):
             mylen = len( self.__toklist )
             del self.__toklist[i]
 
             # convert int to slice
             if isinstance(i, int):
                 if i < 0:
                     i += mylen
                 i = slice(i, i+1)
             # get removed indices
             removed = list(range(*i.indices(mylen)))
             removed.reverse()
             # fixup indices in token dictionary
             for name in self.__tokdict:
                 occurrences = self.__tokdict[name]
                 for j in removed:
                     for k, (value, position) in enumerate(occurrences):
                         occurrences[k] = _ParseResultsWithOffset(value, position - (position > j))
         else:
             del self.__tokdict[i]
 
     def __contains__( self, k ):
         return k in self.__tokdict
 
     def __len__( self ): return len( self.__toklist )
     def __bool__(self): return len( self.__toklist ) > 0
     __nonzero__ = __bool__
     def __iter__( self ): return iter( self.__toklist )
     def __reversed__( self ): return iter( reversed(self.__toklist) )
     def keys( self ):
         """Returns all named result keys."""
         return self.__tokdict.keys()
 
     def pop( self, index=-1 ):
         """Removes and returns item at specified index (default=last).
            Will work with either numeric indices or dict-key indicies."""
         ret = self[index]
         del self[index]
         return ret
 
     def get(self, key, defaultValue=None):
         """Returns named result matching the given key, or if there is no
            such name, then returns the given defaultValue or None if no
            defaultValue is specified."""
         if key in self:
             return self[key]
         else:
             return defaultValue
 
     def insert( self, index, insStr ):
         self.__toklist.insert(index, insStr)
         # fixup indices in token dictionary
         for name in self.__tokdict:
             occurrences = self.__tokdict[name]
             for k, (value, position) in enumerate(occurrences):
                 occurrences[k] = _ParseResultsWithOffset(value, position + (position > index))
 
     def items( self ):
         """Returns all named result keys and values as a list of tuples."""
         return [(k,self[k]) for k in self.__tokdict]
 
     def values( self ):
         """Returns all named result values."""
-        return [ v[-1][0] for v in self.__tokdict.values() ]
+        return [ v[-1][0] for v in self.__tokdict.itervalues() ]
 
     def __getattr__( self, name ):
         if name not in self.__slots__:
             if name in self.__tokdict:
                 if name not in self.__accumNames:
                     return self.__tokdict[name][-1][0]
                 else:
                     return ParseResults([ v[0] for v in self.__tokdict[name] ])
             else:
                 return ""
         return None
 
     def __add__( self, other ):
         ret = self.copy()
         ret += other
         return ret
 
     def __iadd__( self, other ):
         if other.__tokdict:
             offset = len(self.__toklist)
             addoffset = ( lambda a: (a<0 and offset) or (a+offset) )
-            otheritems = other.__tokdict.items()
+            otheritems = other.__tokdict.iteritems()
             otherdictitems = [(k, _ParseResultsWithOffset(v[0],addoffset(v[1])) )
                                 for (k,vlist) in otheritems for v in vlist]
             for k,v in otherdictitems:
                 self[k] = v
                 if isinstance(v[0],ParseResults):
                     v[0].__parent = wkref(self)
             
         self.__toklist += other.__toklist
         self.__accumNames.update( other.__accumNames )
         del other
         return self
 
     def __repr__( self ):
         return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) )
 
     def __str__( self ):
         out = "["
         sep = ""
         for i in self.__toklist:
             if isinstance(i, ParseResults):
                 out += sep + _ustr(i)
             else:
                 out += sep + repr(i)
             sep = ", "
         out += "]"
         return out
 
     def _asStringList( self, sep='' ):
         out = []
         for item in self.__toklist:
             if out and sep:
                 out.append(sep)
             if isinstance( item, ParseResults ):
                 out += item._asStringList()
             else:
                 out.append( _ustr(item) )
         return out
 
     def asList( self ):
         """Returns the parse results as a nested list of matching tokens, all converted to strings."""
         out = []
         for res in self.__toklist:
             if isinstance(res,ParseResults):
                 out.append( res.asList() )
             else:
                 out.append( res )
         return out
 
     def asDict( self ):
         """Returns the named parse results as dictionary."""
         return dict( self.items() )
 
     def copy( self ):
         """Returns a new copy of a ParseResults object."""
         ret = ParseResults( self.__toklist )
         ret.__tokdict = self.__tokdict.copy()
         ret.__parent = self.__parent
         ret.__accumNames.update( self.__accumNames )
         ret.__name = self.__name
         return ret
 
     def asXML( self, doctag=None, namedItemsOnly=False, indent="", formatted=True ):
         """Returns the parse results as XML. Tags are created for tokens and lists that have defined results names."""
         nl = "\n"
         out = []
-        namedItems = dict( [ (v[1],k) for (k,vlist) in self.__tokdict.items()
+        namedItems = dict([(v[1],k) for (k,vlist) in self.__tokdict.iteritems()
                                                             for v in vlist ] )
         nextLevelIndent = indent + "  "
 
         # collapse out indents if formatting is not desired
         if not formatted:
             indent = ""
             nextLevelIndent = ""
             nl = ""
 
         selfTag = None
         if doctag is not None:
             selfTag = doctag
         else:
             if self.__name:
                 selfTag = self.__name
 
         if not selfTag:
             if namedItemsOnly:
                 return ""
             else:
                 selfTag = "ITEM"
 
         out += [ nl, indent, "<", selfTag, ">" ]
 
         worklist = self.__toklist
         for i,res in enumerate(worklist):
             if isinstance(res,ParseResults):
                 if i in namedItems:
                     out += [ res.asXML(namedItems[i],
                                         namedItemsOnly and doctag is None,
                                         nextLevelIndent,
                                         formatted)]
                 else:
                     out += [ res.asXML(None,
                                         namedItemsOnly and doctag is None,
                                         nextLevelIndent,
                                         formatted)]
             else:
                 # individual token, see if there is a name for it
                 resTag = None
                 if i in namedItems:
                     resTag = namedItems[i]
                 if not resTag:
                     if namedItemsOnly:
                         continue
                     else:
                         resTag = "ITEM"
                 xmlBodyText = _xml_escape(_ustr(res))
                 out += [ nl, nextLevelIndent, "<", resTag, ">",
                                                 xmlBodyText,
                                                 "</", resTag, ">" ]
 
         out += [ nl, indent, "</", selfTag, ">" ]
         return "".join(out)
 
     def __lookup(self,sub):
-        for k,vlist in self.__tokdict.items():
+        for k,vlist in self.__tokdict.iteritems():
             for v,loc in vlist:
                 if sub is v:
                     return k
         return None
 
     def getName(self):
         """Returns the results name for this token expression."""
         if self.__name:
             return self.__name
         elif self.__parent:
             par = self.__parent()
             if par:
                 return par.__lookup(self)
             else:
                 return None
         elif (len(self) == 1 and
                len(self.__tokdict) == 1 and
                self.__tokdict.values()[0][0][1] in (0,-1)):
             return self.__tokdict.keys()[0]
         else:
             return None
 
     def dump(self,indent='',depth=0):
         """Diagnostic method for listing out the contents of a ParseResults.
            Accepts an optional indent argument so that this string can be embedded
            in a nested display of other data."""
         out = []
         out.append( indent+_ustr(self.asList()) )
         keys = self.items()
         keys.sort()
         for k,v in keys:
             if out:
                 out.append('\n')
             out.append( "%s%s- %s: " % (indent,('  '*depth), k) )
             if isinstance(v,ParseResults):
                 if v.keys():
                     #~ out.append('\n')
                     out.append( v.dump(indent,depth+1) )
                     #~ out.append('\n')
                 else:
                     out.append(_ustr(v))
             else:
                 out.append(_ustr(v))
         #~ out.append('\n')
         return "".join(out)
 
     # add support for pickle protocol
     def __getstate__(self):
         return ( self.__toklist,
                  ( self.__tokdict.copy(),
                    self.__parent is not None and self.__parent() or None,
                    self.__accumNames,
                    self.__name ) )
 
     def __setstate__(self,state):
         self.__toklist = state[0]
         self.__tokdict, \
         par, \
         inAccumNames, \
         self.__name = state[1]
         self.__accumNames = {}
         self.__accumNames.update(inAccumNames)
         if par is not None:
             self.__parent = wkref(par)
         else:
             self.__parent = None
 
     def __dir__(self):
         return dir(super(ParseResults,self)) + self.keys()
 
 def col (loc,strg):
     """Returns current column within a string, counting newlines as line separators.
    The first column is number 1.
 
    Note: the default parsing behavior is to expand tabs in the input string
    before starting the parsing process.  See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information
    on parsing strings containing <TAB>s, and suggested methods to maintain a
    consistent view of the parsed string, the parse location, and line and column
    positions within the parsed string.
    """
     return (loc<len(strg) and strg[loc] == '\n') and 1 or loc - strg.rfind("\n", 0, loc)
 
 def lineno(loc,strg):
     """Returns current line number within a string, counting newlines as line separators.
    The first line is number 1.
 
    Note: the default parsing behavior is to expand tabs in the input string
    before starting the parsing process.  See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information
    on parsing strings containing <TAB>s, and suggested methods to maintain a
    consistent view of the parsed string, the parse location, and line and column
    positions within the parsed string.
    """
     return strg.count("\n",0,loc) + 1
 
 def line( loc, strg ):
     """Returns the line of text containing loc within a string, counting newlines as line separators.
        """
     lastCR = strg.rfind("\n", 0, loc)
     nextCR = strg.find("\n", loc)
     if nextCR > 0:
         return strg[lastCR+1:nextCR]
     else:
         return strg[lastCR+1:]
 
 def _defaultStartDebugAction( instring, loc, expr ):
     print ("Match " + _ustr(expr) + " at loc " + _ustr(loc) + "(%d,%d)" % ( lineno(loc,instring), col(loc,instring) ))
 
 def _defaultSuccessDebugAction( instring, startloc, endloc, expr, toks ):
     print ("Matched " + _ustr(expr) + " -> " + str(toks.asList()))
 
 def _defaultExceptionDebugAction( instring, loc, expr, exc ):
     print ("Exception raised:" + _ustr(exc))
 
 def nullDebugAction(*args):
     """'Do-nothing' debug action, to suppress debugging output during parsing."""
     pass
 
 class ParserElement(object):
     """Abstract base level parser element class."""
     DEFAULT_WHITE_CHARS = " \n\t\r"
 
     def setDefaultWhitespaceChars( chars ):
         """Overrides the default whitespace chars
         """
         ParserElement.DEFAULT_WHITE_CHARS = chars
     setDefaultWhitespaceChars = staticmethod(setDefaultWhitespaceChars)
 
     def __init__( self, savelist=False ):
         self.parseAction = list()
         self.failAction = None
         #~ self.name = "<unknown>"  # don't define self.name, let subclasses try/except upcall
         self.strRepr = None
         self.resultsName = None
         self.saveAsList = savelist
         self.skipWhitespace = True
         self.whiteChars = ParserElement.DEFAULT_WHITE_CHARS
         self.copyDefaultWhiteChars = True
         self.mayReturnEmpty = False # used when checking for left-recursion
         self.keepTabs = False
         self.ignoreExprs = list()
         self.debug = False
         self.streamlined = False
         self.mayIndexError = True # used to optimize exception handling for subclasses that don't advance parse index
         self.errmsg = ""
         self.modalResults = True # used to mark results names as modal (report only last) or cumulative (list all)
         self.debugActions = ( None, None, None ) #custom debug actions
         self.re = None
         self.callPreparse = True # used to avoid redundant calls to preParse
         self.callDuringTry = False
 
     def copy( self ):
         """Make a copy of this ParserElement.  Useful for defining different parse actions
            for the same parsing pattern, using copies of the original parse element."""
         cpy = copy.copy( self )
         cpy.parseAction = self.parseAction[:]
         cpy.ignoreExprs = self.ignoreExprs[:]
         if self.copyDefaultWhiteChars:
             cpy.whiteChars = ParserElement.DEFAULT_WHITE_CHARS
         return cpy
 
     def setName( self, name ):
         """Define name for this expression, for use in debugging."""
         self.name = name
         self.errmsg = "Expected " + self.name
         if hasattr(self,"exception"):
             self.exception.msg = self.errmsg
         return self
 
     def setResultsName( self, name, listAllMatches=False ):
         """Define name for referencing matching tokens as a nested attribute
            of the returned parse results.
            NOTE: this returns a *copy* of the original ParserElement object;
            this is so that the client can define a basic element, such as an
            integer, and reference it in multiple places with different names.
         """
         newself = self.copy()
         newself.resultsName = name
         newself.modalResults = not listAllMatches
         return newself
 
     def setBreak(self,breakFlag = True):
         """Method to invoke the Python pdb debugger when this element is
            about to be parsed. Set breakFlag to True to enable, False to
            disable.
         """
         if breakFlag:
             _parseMethod = self._parse
             def breaker(instring, loc, doActions=True, callPreParse=True):
                 import pdb
                 pdb.set_trace()
                 return _parseMethod( instring, loc, doActions, callPreParse )
             breaker._originalParseMethod = _parseMethod
             self._parse = breaker
         else:
             if hasattr(self._parse,"_originalParseMethod"):
                 self._parse = self._parse._originalParseMethod
         return self
 
     def _normalizeParseActionArgs( f ):
         """Internal method used to decorate parse actions that take fewer than 3 arguments,
            so that all parse actions can be called as f(s,l,t)."""
         STAR_ARGS = 4
 
         try:
             restore = None
             if isinstance(f,type):
                 restore = f
                 f = f.__init__
             if not _PY3K:
                 codeObj = f.func_code
             else:
                 codeObj = f.code
             if codeObj.co_flags & STAR_ARGS:
                 return f
             numargs = codeObj.co_argcount
             if not _PY3K:
                 if hasattr(f,"im_self"):
                     numargs -= 1
             else:
                 if hasattr(f,"__self__"):
                     numargs -= 1
             if restore:
                 f = restore
         except AttributeError:
             try:
                 if not _PY3K:
                     call_im_func_code = f.__call__.im_func.func_code
                 else:
                     call_im_func_code = f.__code__
 
                 # not a function, must be a callable object, get info from the
                 # im_func binding of its bound __call__ method
                 if call_im_func_code.co_flags & STAR_ARGS:
                     return f
                 numargs = call_im_func_code.co_argcount
                 if not _PY3K:
                     if hasattr(f.__call__,"im_self"):
                         numargs -= 1
                 else:
                     if hasattr(f.__call__,"__self__"):
                         numargs -= 0
             except AttributeError:
                 if not _PY3K:
                     call_func_code = f.__call__.func_code
                 else:
                     call_func_code = f.__call__.__code__
                 # not a bound method, get info directly from __call__ method
                 if call_func_code.co_flags & STAR_ARGS:
                     return f
                 numargs = call_func_code.co_argcount
                 if not _PY3K:
                     if hasattr(f.__call__,"im_self"):
                         numargs -= 1
                 else:
                     if hasattr(f.__call__,"__self__"):
                         numargs -= 1
 
 
         #~ print ("adding function %s with %d args" % (f.func_name,numargs))
         if numargs == 3:
             return f
         else:
             if numargs > 3:
                 def tmp(s,l,t):
                     return f(f.__call__.__self__, s,l,t)
             if numargs == 2:
                 def tmp(s,l,t):
                     return f(l,t)
             elif numargs == 1:
                 def tmp(s,l,t):
                     return f(t)
             else: #~ numargs == 0:
                 def tmp(s,l,t):
                     return f()
             try:
                 tmp.__name__ = f.__name__
             except (AttributeError,TypeError):
                 # no need for special handling if attribute doesnt exist
                 pass
             try:
                 tmp.__doc__ = f.__doc__
             except (AttributeError,TypeError):
                 # no need for special handling if attribute doesnt exist
                 pass
             try:
                 tmp.__dict__.update(f.__dict__)
             except (AttributeError,TypeError):
                 # no need for special handling if attribute doesnt exist
                 pass
             return tmp
     _normalizeParseActionArgs = staticmethod(_normalizeParseActionArgs)
 
     def setParseAction( self, *fns, **kwargs ):
         """Define action to perform when successfully matching parse element definition.
            Parse action fn is a callable method with 0-3 arguments, called as fn(s,loc,toks),
            fn(loc,toks), fn(toks), or just fn(), where:
             - s   = the original string being parsed (see note below)
             - loc = the location of the matching substring
             - toks = a list of the matched tokens, packaged as a ParseResults object
            If the functions in fns modify the tokens, they can return them as the return
            value from fn, and the modified list of tokens will replace the original.
            Otherwise, fn does not need to return any value.
 
            Note: the default parsing behavior is to expand tabs in the input string
            before starting the parsing process.  See L{I{parseString}<parseString>} for more information
            on parsing strings containing <TAB>s, and suggested methods to maintain a
            consistent view of the parsed string, the parse location, and line and column
            positions within the parsed string.
            """
         self.parseAction = list(map(self._normalizeParseActionArgs, list(fns)))
         self.callDuringTry = ("callDuringTry" in kwargs and kwargs["callDuringTry"])
         return self
 
     def addParseAction( self, *fns, **kwargs ):
         """Add parse action to expression's list of parse actions. See L{I{setParseAction}<setParseAction>}."""
         self.parseAction += list(map(self._normalizeParseActionArgs, list(fns)))
         self.callDuringTry = self.callDuringTry or ("callDuringTry" in kwargs and kwargs["callDuringTry"])
         return self
 
     def setFailAction( self, fn ):
         """Define action to perform if parsing fails at this expression.
            Fail acton fn is a callable function that takes the arguments
            fn(s,loc,expr,err) where:
             - s = string being parsed
             - loc = location where expression match was attempted and failed
             - expr = the parse expression that failed
             - err = the exception thrown
            The function returns no value.  It may throw ParseFatalException
            if it is desired to stop parsing immediately."""
         self.failAction = fn
         return self
 
     def _skipIgnorables( self, instring, loc ):
         exprsFound = True
         while exprsFound:
             exprsFound = False
             for e in self.ignoreExprs:
                 try:
                     while 1:
                         loc,dummy = e._parse( instring, loc )
                         exprsFound = True
                 except ParseException:
                     pass
         return loc
 
     def preParse( self, instring, loc ):
         if self.ignoreExprs:
             loc = self._skipIgnorables( instring, loc )
 
         if self.skipWhitespace:
             wt = self.whiteChars
             instrlen = len(instring)
             while loc < instrlen and instring[loc] in wt:
                 loc += 1
 
         return loc
 
     def parseImpl( self, instring, loc, doActions=True ):
         return loc, []
 
     def postParse( self, instring, loc, tokenlist ):
         return tokenlist
 
     #~ @profile
     def _parseNoCache( self, instring, loc, doActions=True, callPreParse=True ):
         debugging = ( self.debug ) #and doActions )
 
         if debugging or self.failAction:
             #~ print ("Match",self,"at loc",loc,"(%d,%d)" % ( lineno(loc,instring), col(loc,instring) ))
             if (self.debugActions[0] ):
                 self.debugActions[0]( instring, loc, self )
             if callPreParse and self.callPreparse:
                 preloc = self.preParse( instring, loc )
             else:
                 preloc = loc
             tokensStart = loc
             try:
                 try:
                     loc,tokens = self.parseImpl( instring, preloc, doActions )
                 except IndexError:
                     raise ParseException( instring, len(instring), self.errmsg, self )
             except ParseBaseException, err:
                 #~ print ("Exception raised:", err)
                 if self.debugActions[2]:
                     self.debugActions[2]( instring, tokensStart, self, err )
                 if self.failAction:
                     self.failAction( instring, tokensStart, self, err )
                 raise
         else:
             if callPreParse and self.callPreparse:
                 preloc = self.preParse( instring, loc )
             else:
                 preloc = loc
             tokensStart = loc
             if self.mayIndexError or loc >= len(instring):
                 try:
                     loc,tokens = self.parseImpl( instring, preloc, doActions )
                 except IndexError:
                     raise ParseException( instring, len(instring), self.errmsg, self )
             else:
                 loc,tokens = self.parseImpl( instring, preloc, doActions )
 
         tokens = self.postParse( instring, loc, tokens )
 
         retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults )
         if self.parseAction and (doActions or self.callDuringTry):
             if debugging:
                 try:
                     for fn in self.parseAction:
                         tokens = fn( instring, tokensStart, retTokens )
                         if tokens is not None:
                             retTokens = ParseResults( tokens,
                                                       self.resultsName,
                                                       asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
                                                       modal=self.modalResults )
                 except ParseBaseException, err:
                     #~ print "Exception raised in user parse action:", err
                     if (self.debugActions[2] ):
                         self.debugActions[2]( instring, tokensStart, self, err )
                     raise
             else:
                 for fn in self.parseAction:
                     tokens = fn( instring, tokensStart, retTokens )
                     if tokens is not None:
                         retTokens = ParseResults( tokens,
                                                   self.resultsName,
                                                   asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
                                                   modal=self.modalResults )
 
         if debugging:
             #~ print ("Matched",self,"->",retTokens.asList())
             if (self.debugActions[1] ):
                 self.debugActions[1]( instring, tokensStart, loc, self, retTokens )
 
         return loc, retTokens
 
     def tryParse( self, instring, loc ):
         try:
             return self._parse( instring, loc, doActions=False )[0]
         except ParseFatalException:
             raise ParseException( instring, loc, self.errmsg, self)
 
     # this method gets repeatedly called during backtracking with the same arguments -
     # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression
     def _parseCache( self, instring, loc, doActions=True, callPreParse=True ):
         lookup = (self,instring,loc,callPreParse,doActions)
         if lookup in ParserElement._exprArgCache:
             value = ParserElement._exprArgCache[ lookup ]
             if isinstance(value,Exception):
                 raise value
             return value
         else:
             try:
                 value = self._parseNoCache( instring, loc, doActions, callPreParse )
                 ParserElement._exprArgCache[ lookup ] = (value[0],value[1].copy())
                 return value
             except ParseBaseException, pe:
                 ParserElement._exprArgCache[ lookup ] = pe
                 raise
 
     _parse = _parseNoCache
 
     # argument cache for optimizing repeated calls when backtracking through recursive expressions
     _exprArgCache = {}
     def resetCache():
         ParserElement._exprArgCache.clear()
     resetCache = staticmethod(resetCache)
 
     _packratEnabled = False
     def enablePackrat():
         """Enables "packrat" parsing, which adds memoizing to the parsing logic.
            Repeated parse attempts at the same string location (which happens
            often in many complex grammars) can immediately return a cached value,
            instead of re-executing parsing/validating code.  Memoizing is done of
            both valid results and parsing exceptions.
 
            This speedup may break existing programs that use parse actions that
            have side-effects.  For this reason, packrat parsing is disabled when
            you first import pyparsing.  To activate the packrat feature, your
            program must call the class method ParserElement.enablePackrat().  If
            your program uses psyco to "compile as you go", you must call
            enablePackrat before calling psyco.full().  If you do not do this,
            Python will crash.  For best results, call enablePackrat() immediately
            after importing pyparsing.
         """
         if not ParserElement._packratEnabled:
             ParserElement._packratEnabled = True
             ParserElement._parse = ParserElement._parseCache
     enablePackrat = staticmethod(enablePackrat)
 
     def parseString( self, instring, parseAll=False ):
         """Execute the parse expression with the given string.
            This is the main interface to the client code, once the complete
            expression has been built.
 
            If you want the grammar to require that the entire input string be
            successfully parsed, then set parseAll to True (equivalent to ending
            the grammar with StringEnd()).
 
            Note: parseString implicitly calls expandtabs() on the input string,
            in order to report proper column numbers in parse actions.
            If the input string contains tabs and
            the grammar uses parse actions that use the loc argument to index into the
            string being parsed, you can ensure you have a consistent view of the input
            string by:
             - calling parseWithTabs on your grammar before calling parseString
               (see L{I{parseWithTabs}<parseWithTabs>})
             - define your parse action using the full (s,loc,toks) signature, and
               reference the input string using the parse action's s argument
             - explictly expand the tabs in your input string before calling
               parseString
         """
         ParserElement.resetCache()
         if not self.streamlined:
             self.streamline()
             #~ self.saveAsList = True
         for e in self.ignoreExprs:
             e.streamline()
         if not self.keepTabs:
             instring = instring.expandtabs()
         try:
             loc, tokens = self._parse( instring, 0 )
             if parseAll:
                 loc = self.preParse( instring, loc )
                 StringEnd()._parse( instring, loc )
         except ParseBaseException, exc:
             # catch and re-raise exception from here, clears out pyparsing internal stack trace
             raise exc
         else:
             return tokens
 
     def scanString( self, instring, maxMatches=_MAX_INT ):
         """Scan the input string for expression matches.  Each match will return the
            matching tokens, start location, and end location.  May be called with optional
            maxMatches argument, to clip scanning after 'n' matches are found.
 
            Note that the start and end locations are reported relative to the string
            being parsed.  See L{I{parseString}<parseString>} for more information on parsing
            strings with embedded tabs."""
         if not self.streamlined:
             self.streamline()
         for e in self.ignoreExprs:
             e.streamline()
 
         if not self.keepTabs:
             instring = _ustr(instring).expandtabs()
         instrlen = len(instring)
         loc = 0
         preparseFn = self.preParse
         parseFn = self._parse
         ParserElement.resetCache()
         matches = 0
         try:
             while loc <= instrlen and matches < maxMatches:
                 try:
                     preloc = preparseFn( instring, loc )
                     nextLoc,tokens = parseFn( instring, preloc, callPreParse=False )
                 except ParseException:
                     loc = preloc+1
                 else:
                     matches += 1
                     yield tokens, preloc, nextLoc
                     loc = nextLoc
         except ParseBaseException, pe:
             raise pe
 
     def transformString( self, instring ):
         """Extension to scanString, to modify matching text with modified tokens that may
            be returned from a parse action.  To use transformString, define a grammar and
            attach a parse action to it that modifies the returned token list.
            Invoking transformString() on a target string will then scan for matches,
            and replace the matched text patterns according to the logic in the parse
            action.  transformString() returns the resulting transformed string."""
         out = []
         lastE = 0
         # force preservation of <TAB>s, to minimize unwanted transformation of string, and to
         # keep string locs straight between transformString and scanString
         self.keepTabs = True
         try:
             for t,s,e in self.scanString( instring ):
                 out.append( instring[lastE:s] )
                 if t:
                     if isinstance(t,ParseResults):
                         out += t.asList()
                     elif isinstance(t,list):
                         out += t
                     else:
                         out.append(t)
                 lastE = e
             out.append(instring[lastE:])
             return "".join(map(_ustr,out))
         except ParseBaseException, pe:
             raise pe
 
     def searchString( self, instring, maxMatches=_MAX_INT ):
         """Another extension to scanString, simplifying the access to the tokens found
            to match the given parse expression.  May be called with optional
            maxMatches argument, to clip searching after 'n' matches are found.
         """
         try:
             return ParseResults([ t for t,s,e in self.scanString( instring, maxMatches ) ])
         except ParseBaseException, pe:
             raise pe
 
     def __add__(self, other ):
         """Implementation of + operator - returns And"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return And( [ self, other ] )
 
     def __radd__(self, other ):
         """Implementation of + operator when left operand is not a ParserElement"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return other + self
 
     def __sub__(self, other):
         """Implementation of - operator, returns And with error stop"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return And( [ self, And._ErrorStop(), other ] )
 
     def __rsub__(self, other ):
         """Implementation of - operator when left operand is not a ParserElement"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return other - self
 
     def __mul__(self,other):
         if isinstance(other,int):
             minElements, optElements = other,0
         elif isinstance(other,tuple):
             other = (other + (None, None))[:2]
             if other[0] is None:
                 other = (0, other[1])
             if isinstance(other[0],int) and other[1] is None:
                 if other[0] == 0:
                     return ZeroOrMore(self)
                 if other[0] == 1:
                     return OneOrMore(self)
                 else:
                     return self*other[0] + ZeroOrMore(self)
             elif isinstance(other[0],int) and isinstance(other[1],int):
                 minElements, optElements = other
                 optElements -= minElements
             else:
                 raise TypeError("cannot multiply 'ParserElement' and ('%s','%s') objects", type(other[0]),type(other[1]))
         else:
             raise TypeError("cannot multiply 'ParserElement' and '%s' objects", type(other))
 
         if minElements < 0:
             raise ValueError("cannot multiply ParserElement by negative value")
         if optElements < 0:
             raise ValueError("second tuple value must be greater or equal to first tuple value")
         if minElements == optElements == 0:
             raise ValueError("cannot multiply ParserElement by 0 or (0,0)")
 
         if (optElements):
             def makeOptionalList(n):
                 if n>1:
                     return Optional(self + makeOptionalList(n-1))
                 else:
                     return Optional(self)
             if minElements:
                 if minElements == 1:
                     ret = self + makeOptionalList(optElements)
                 else:
                     ret = And([self]*minElements) + makeOptionalList(optElements)
             else:
                 ret = makeOptionalList(optElements)
         else:
             if minElements == 1:
                 ret = self
             else:
                 ret = And([self]*minElements)
         return ret
 
     def __rmul__(self, other):
         return self.__mul__(other)
 
     def __or__(self, other ):
         """Implementation of | operator - returns MatchFirst"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return MatchFirst( [ self, other ] )
 
     def __ror__(self, other ):
         """Implementation of | operator when left operand is not a ParserElement"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return other | self
 
     def __xor__(self, other ):
         """Implementation of ^ operator - returns Or"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return Or( [ self, other ] )
 
     def __rxor__(self, other ):
         """Implementation of ^ operator when left operand is not a ParserElement"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return other ^ self
 
     def __and__(self, other ):
         """Implementation of & operator - returns Each"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return Each( [ self, other ] )
 
     def __rand__(self, other ):
         """Implementation of & operator when left operand is not a ParserElement"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return other & self
 
     def __invert__( self ):
         """Implementation of ~ operator - returns NotAny"""
         return NotAny( self )
 
     def __call__(self, name):
         """Shortcut for setResultsName, with listAllMatches=default::
              userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno")
            could be written as::
              userdata = Word(alphas)("name") + Word(nums+"-")("socsecno")
            """
         return self.setResultsName(name)
 
     def suppress( self ):
         """Suppresses the output of this ParserElement; useful to keep punctuation from
            cluttering up returned output.
         """
         return Suppress( self )
 
     def leaveWhitespace( self ):
         """Disables the skipping of whitespace before matching the characters in the
            ParserElement's defined pattern.  This is normally only used internally by
            the pyparsing module, but may be needed in some whitespace-sensitive grammars.
         """
         self.skipWhitespace = False
         return self
 
     def setWhitespaceChars( self, chars ):
         """Overrides the default whitespace chars
         """
         self.skipWhitespace = True
         self.whiteChars = chars
         self.copyDefaultWhiteChars = False
         return self
 
     def parseWithTabs( self ):
         """Overrides default behavior to expand <TAB>s to spaces before parsing the input string.
            Must be called before parseString when the input grammar contains elements that
            match <TAB> characters."""
         self.keepTabs = True
         return self
 
     def ignore( self, other ):
         """Define expression to be ignored (e.g., comments) while doing pattern
            matching; may be called repeatedly, to define multiple comment or other
            ignorable patterns.
         """
         if isinstance( other, Suppress ):
             if other not in self.ignoreExprs:
                 self.ignoreExprs.append( other )
         else:
             self.ignoreExprs.append( Suppress( other ) )
         return self
 
     def setDebugActions( self, startAction, successAction, exceptionAction ):
         """Enable display of debugging messages while doing pattern matching."""
         self.debugActions = (startAction or _defaultStartDebugAction,
                              successAction or _defaultSuccessDebugAction,
                              exceptionAction or _defaultExceptionDebugAction)
         self.debug = True
         return self
 
     def setDebug( self, flag=True ):
         """Enable display of debugging messages while doing pattern matching.
            Set flag to True to enable, False to disable."""
         if flag:
             self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction )
         else:
             self.debug = False
         return self
 
     def __str__( self ):
         return self.name
 
     def __repr__( self ):
         return _ustr(self)
 
     def streamline( self ):
         self.streamlined = True
         self.strRepr = None
         return self
 
     def checkRecursion( self, parseElementList ):
         pass
 
     def validate( self, validateTrace=[] ):
         """Check defined expressions for valid structure, check for infinite recursive definitions."""
         self.checkRecursion( [] )
 
     def parseFile( self, file_or_filename, parseAll=False ):
         """Execute the parse expression on the given file or filename.
            If a filename is specified (instead of a file object),
            the entire file is opened, read, and closed before parsing.
         """
         try:
             file_contents = file_or_filename.read()
         except AttributeError:
             f = open(file_or_filename, "rb")
             file_contents = f.read()
             f.close()
         try:
             return self.parseString(file_contents, parseAll)
         except ParseBaseException, exc:
             # catch and re-raise exception from here, clears out pyparsing internal stack trace
             raise exc
 
     def getException(self):
         return ParseException("",0,self.errmsg,self)
 
     def __getattr__(self,aname):
         if aname == "myException":
             self.myException = ret = self.getException();
             return ret;
         else:
             raise AttributeError("no such attribute " + aname)
 
     def __eq__(self,other):
         if isinstance(other, ParserElement):
             return self is other or self.__dict__ == other.__dict__
         elif isinstance(other, basestring):
             try:
                 self.parseString(_ustr(other), parseAll=True)
                 return True
             except ParseBaseException:
                 return False
         else:
             return super(ParserElement,self)==other
 
     def __ne__(self,other):
         return not (self == other)
 
     def __hash__(self):
         return hash(id(self))
 
     def __req__(self,other):
         return self == other
 
     def __rne__(self,other):
         return not (self == other)
 
 
 class Token(ParserElement):
     """Abstract ParserElement subclass, for defining atomic matching patterns."""
     def __init__( self ):
         super(Token,self).__init__( savelist=False )
         #self.myException = ParseException("",0,"",self)
 
     def setName(self, name):
         s = super(Token,self).setName(name)
         self.errmsg = "Expected " + self.name
         #s.myException.msg = self.errmsg
         return s
 
 
 class Empty(Token):
     """An empty token, will always match."""
     def __init__( self ):
         super(Empty,self).__init__()
         self.name = "Empty"
         self.mayReturnEmpty = True
         self.mayIndexError = False
 
 
 class NoMatch(Token):
     """A token that will never match."""
     def __init__( self ):
         super(NoMatch,self).__init__()
         self.name = "NoMatch"
         self.mayReturnEmpty = True
         self.mayIndexError = False
         self.errmsg = "Unmatchable token"
         #self.myException.msg = self.errmsg
 
     def parseImpl( self, instring, loc, doActions=True ):
         exc = self.myException
         exc.loc = loc
         exc.pstr = instring
         raise exc
 
 
 class Literal(Token):
     """Token to exactly match a specified string."""
     def __init__( self, matchString ):
         super(Literal,self).__init__()
         self.match = matchString
         self.matchLen = len(matchString)
         try:
             self.firstMatchChar = matchString[0]
         except IndexError:
             warnings.warn("null string passed to Literal; use Empty() instead",
                             SyntaxWarning, stacklevel=2)
             self.__class__ = Empty
         self.name = '"%s"' % _ustr(self.match)
         self.errmsg = "Expected " + self.name
         self.mayReturnEmpty = False
         #self.myException.msg = self.errmsg
         self.mayIndexError = False
 
     # Performance tuning: this routine gets called a *lot*
     # if this is a single character match string  and the first character matches,
     # short-circuit as quickly as possible, and avoid calling startswith
     #~ @profile
     def parseImpl( self, instring, loc, doActions=True ):
         if (instring[loc] == self.firstMatchChar and
             (self.matchLen==1 or instring.startswith(self.match,loc)) ):
             return loc+self.matchLen, self.match
         #~ raise ParseException( instring, loc, self.errmsg )
         exc = self.myException
         exc.loc = loc
         exc.pstr = instring
         raise exc
 _L = Literal
 
 class Keyword(Token):
     """Token to exactly match a specified string as a keyword, that is, it must be
        immediately followed by a non-keyword character.  Compare with Literal::
          Literal("if") will match the leading 'if' in 'ifAndOnlyIf'.
          Keyword("if") will not; it will only match the leading 'if in 'if x=1', or 'if(y==2)'
        Accepts two optional constructor arguments in addition to the keyword string:
        identChars is a string of characters that would be valid identifier characters,
        defaulting to all alphanumerics + "_" and "$"; caseless allows case-insensitive
        matching, default is False.
     """
     DEFAULT_KEYWORD_CHARS = alphanums+"_$"
 
     def __init__( self, matchString, identChars=DEFAULT_KEYWORD_CHARS, caseless=False ):
         super(Keyword,self).__init__()
         self.match = matchString
         self.matchLen = len(matchString)
         try:
             self.firstMatchChar = matchString[0]
         except IndexError:
             warnings.warn("null string passed to Keyword; use Empty() instead",
                             SyntaxWarning, stacklevel=2)
         self.name = '"%s"' % self.match
         self.errmsg = "Expected " + self.name
         self.mayReturnEmpty = False
         #self.myException.msg = self.errmsg
         self.mayIndexError = False
         self.caseless = caseless
         if caseless:
             self.caselessmatch = matchString.upper()
             identChars = identChars.upper()
         self.identChars = _str2dict(identChars)
 
     def parseImpl( self, instring, loc, doActions=True ):
         if self.caseless:
             if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
                  (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) and
                  (loc == 0 or instring[loc-1].upper() not in self.identChars) ):
                 return loc+self.matchLen, self.match
         else:
             if (instring[loc] == self.firstMatchChar and
                 (self.matchLen==1 or instring.startswith(self.match,loc)) and
                 (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen] not in self.identChars) and
                 (loc == 0 or instring[loc-1] not in self.identChars) ):
                 return loc+self.matchLen, self.match
         #~ raise ParseException( instring, loc, self.errmsg )
         exc = self.myException
         exc.loc = loc
         exc.pstr = instring
         raise exc
 
     def copy(self):
         c = super(Keyword,self).copy()
         c.identChars = Keyword.DEFAULT_KEYWORD_CHARS
         return c
 
     def setDefaultKeywordChars( chars ):
         """Overrides the default Keyword chars
         """
         Keyword.DEFAULT_KEYWORD_CHARS = chars
     setDefaultKeywordChars = staticmethod(setDefaultKeywordChars)
 
 class CaselessLiteral(Literal):
     """Token to match a specified string, ignoring case of letters.
        Note: the matched results will always be in the case of the given
        match string, NOT the case of the input text.
     """
     def __init__( self, matchString ):
         super(CaselessLiteral,self).__init__( matchString.upper() )
         # Preserve the defining literal.
         self.returnString = matchString
         self.name = "'%s'" % self.returnString
         self.errmsg = "Expected " + self.name
         #self.myException.msg = self.errmsg
 
     def parseImpl( self, instring, loc, doActions=True ):
         if instring[ loc:loc+self.matchLen ].upper() == self.match:
             return loc+self.matchLen, self.returnString
         #~ raise ParseException( instring, loc, self.errmsg )
         exc = self.myException
         exc.loc = loc
         exc.pstr = instring
         raise exc
 
 class CaselessKeyword(Keyword):
     def __init__( self, matchString, identChars=Keyword.DEFAULT_KEYWORD_CHARS ):
         super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True )
 
     def parseImpl( self, instring, loc, doActions=True ):
         if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
              (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ):
             return loc+self.matchLen, self.match
         #~ raise ParseException( instring, loc, self.errmsg )
         exc = self.myException
         exc.loc = loc
         exc.pstr = instring
         raise exc
 
 class Word(Token):
     """Token for matching words composed of allowed character sets.
        Defined with string containing all allowed initial characters,
        an optional string containing allowed body characters (if omitted,
        defaults to the initial character set), and an optional minimum,
        maximum, and/or exact length.  The default value for min is 1 (a
        minimum value < 1 is not valid); the default values for max and exact
        are 0, meaning no maximum or exact length restriction.
     """
     def __init__( self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False ):
         super(Word,self).__init__()
         self.initCharsOrig = initChars
         self.initChars = _str2dict(initChars)
         if bodyChars :
             self.bodyCharsOrig = bodyChars
             self.bodyChars = _str2dict(bodyChars)
         else:
             self.bodyCharsOrig = initChars
             self.bodyChars = _str2dict(initChars)
 
         self.maxSpecified = max > 0
 
         if min < 1:
             raise ValueError("cannot specify a minimum length < 1; use Optional(Word()) if zero-length word is permitted")
 
         self.minLen = min
 
         if max > 0:
             self.maxLen = max
         else:
             self.maxLen = _MAX_INT
 
         if exact > 0:
             self.maxLen = exact
             self.minLen = exact
 
         self.name = _ustr(self)
         self.errmsg = "Expected " + self.name
         #self.myException.msg = self.errmsg
         self.mayIndexError = False
         self.asKeyword = asKeyword
 
         if ' ' not in self.initCharsOrig+self.bodyCharsOrig and (min==1 and max==0 and exact==0):
             if self.bodyCharsOrig == self.initCharsOrig:
                 self.reString = "[%s]+" % _escapeRegexRangeChars(self.initCharsOrig)
             elif len(self.bodyCharsOrig) == 1:
                 self.reString = "%s[%s]*" % \
                                       (re.escape(self.initCharsOrig),
                                       _escapeRegexRangeChars(self.bodyCharsOrig),)
             else:
                 self.reString = "[%s][%s]*" % \
                                       (_escapeRegexRangeChars(self.initCharsOrig),
                                       _escapeRegexRangeChars(self.bodyCharsOrig),)
             if self.asKeyword:
                 self.reString = r"\b"+self.reString+r"\b"
             try:
                 self.re = re.compile( self.reString )
             except:
                 self.re = None
 
     def parseImpl( self, instring, loc, doActions=True ):
         if self.re:
             result = self.re.match(instring,loc)
             if not result:
                 exc = self.myException
                 exc.loc = loc
                 exc.pstr = instring
                 raise exc
 
             loc = result.end()
             return loc,result.group()
 
         if not(instring[ loc ] in self.initChars):
             #~ raise ParseException( instring, loc, self.errmsg )
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
         start = loc
         loc += 1
         instrlen = len(instring)
         bodychars = self.bodyChars
         maxloc = start + self.maxLen
         maxloc = min( maxloc, instrlen )
         while loc < maxloc and instring[loc] in bodychars:
             loc += 1
 
         throwException = False
         if loc - start < self.minLen:
             throwException = True
         if self.maxSpecified and loc < instrlen and instring[loc] in bodychars:
             throwException = True
         if self.asKeyword:
             if (start>0 and instring[start-1] in bodychars) or (loc<instrlen and instring[loc] in bodychars):
                 throwException = True
 
         if throwException:
             #~ raise ParseException( instring, loc, self.errmsg )
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
 
         return loc, instring[start:loc]
 
     def __str__( self ):
         try:
             return super(Word,self).__str__()
         except:
             pass
 
 
         if self.strRepr is None:
 
             def charsAsStr(s):
                 if len(s)>4:
                     return s[:4]+"..."
                 else:
                     return s
 
             if ( self.initCharsOrig != self.bodyCharsOrig ):
                 self.strRepr = "W:(%s,%s)" % ( charsAsStr(self.initCharsOrig), charsAsStr(self.bodyCharsOrig) )
             else:
                 self.strRepr = "W:(%s)" % charsAsStr(self.initCharsOrig)
 
         return self.strRepr
 
 
 class Regex(Token):
     """Token for matching strings that match a given regular expression.
        Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module.
     """
     def __init__( self, pattern, flags=0):
         """The parameters pattern and flags are passed to the re.compile() function as-is. See the Python re module for an explanation of the acceptable patterns and flags."""
         super(Regex,self).__init__()
 
         if len(pattern) == 0:
             warnings.warn("null string passed to Regex; use Empty() instead",
                     SyntaxWarning, stacklevel=2)
 
         self.pattern = pattern
         self.flags = flags
 
         try:
             self.re = re.compile(self.pattern, self.flags)
             self.reString = self.pattern
         except sre_constants.error:
             warnings.warn("invalid pattern (%s) passed to Regex" % pattern,
                 SyntaxWarning, stacklevel=2)
             raise
 
         self.name = _ustr(self)
         self.errmsg = "Expected " + self.name
         #self.myException.msg = self.errmsg
         self.mayIndexError = False
         self.mayReturnEmpty = True
 
     def parseImpl( self, instring, loc, doActions=True ):
         result = self.re.match(instring,loc)
         if not result:
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
 
         loc = result.end()
         d = result.groupdict()
         ret = ParseResults(result.group())
         if d:
             for k in d:
                 ret[k] = d[k]
         return loc,ret
 
     def __str__( self ):
         try:
             return super(Regex,self).__str__()
         except:
             pass
 
         if self.strRepr is None:
             self.strRepr = "Re:(%s)" % repr(self.pattern)
 
         return self.strRepr
 
 
 class QuotedString(Token):
     """Token for matching strings that are delimited by quoting characters.
     """
     def __init__( self, quoteChar, escChar=None, escQuote=None, multiline=False, unquoteResults=True, endQuoteChar=None):
         """
            Defined with the following parameters:
             - quoteChar - string of one or more characters defining the quote delimiting string
             - escChar - character to escape quotes, typically backslash (default=None)
             - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=None)
             - multiline - boolean indicating whether quotes can span multiple lines (default=False)
             - unquoteResults - boolean indicating whether the matched text should be unquoted (default=True)
             - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=None => same as quoteChar)
         """
         super(QuotedString,self).__init__()
 
         # remove white space from quote chars - wont work anyway
         quoteChar = quoteChar.strip()
         if len(quoteChar) == 0:
             warnings.warn("quoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
             raise SyntaxError()
 
         if endQuoteChar is None:
             endQuoteChar = quoteChar
         else:
             endQuoteChar = endQuoteChar.strip()
             if len(endQuoteChar) == 0:
                 warnings.warn("endQuoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
                 raise SyntaxError()
 
         self.quoteChar = quoteChar
         self.quoteCharLen = len(quoteChar)
         self.firstQuoteChar = quoteChar[0]
         self.endQuoteChar = endQuoteChar
         self.endQuoteCharLen = len(endQuoteChar)
         self.escChar = escChar
         self.escQuote = escQuote
         self.unquoteResults = unquoteResults
 
         if multiline:
             self.flags = re.MULTILINE | re.DOTALL
             self.pattern = r'%s(?:[^%s%s]' % \
                 ( re.escape(self.quoteChar),
                   _escapeRegexRangeChars(self.endQuoteChar[0]),
                   (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
         else:
             self.flags = 0
             self.pattern = r'%s(?:[^%s\n\r%s]' % \
                 ( re.escape(self.quoteChar),
                   _escapeRegexRangeChars(self.endQuoteChar[0]),
                   (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
         if len(self.endQuoteChar) > 1:
             self.pattern += (
                 '|(?:' + ')|(?:'.join(["%s[^%s]" % (re.escape(self.endQuoteChar[:i]),
                                                _escapeRegexRangeChars(self.endQuoteChar[i]))
                                     for i in range(len(self.endQuoteChar)-1,0,-1)]) + ')'
                 )
         if escQuote:
             self.pattern += (r'|(?:%s)' % re.escape(escQuote))
         if escChar:
             self.pattern += (r'|(?:%s.)' % re.escape(escChar))
             self.escCharReplacePattern = re.escape(self.escChar)+"(.)"
         self.pattern += (r')*%s' % re.escape(self.endQuoteChar))
 
         try:
             self.re = re.compile(self.pattern, self.flags)
             self.reString = self.pattern
         except sre_constants.error:
             warnings.warn("invalid pattern (%s) passed to Regex" % self.pattern,
                 SyntaxWarning, stacklevel=2)
             raise
 
         self.name = _ustr(self)
         self.errmsg = "Expected " + self.name
         #self.myException.msg = self.errmsg
         self.mayIndexError = False
         self.mayReturnEmpty = True
 
     def parseImpl( self, instring, loc, doActions=True ):
         result = instring[loc] == self.firstQuoteChar and self.re.match(instring,loc) or None
         if not result:
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
 
         loc = result.end()
         ret = result.group()
 
         if self.unquoteResults:
 
             # strip off quotes
             ret = ret[self.quoteCharLen:-self.endQuoteCharLen]
 
             if isinstance(ret,basestring):
                 # replace escaped characters
                 if self.escChar:
                     ret = re.sub(self.escCharReplacePattern,"\g<1>",ret)
 
                 # replace escaped quotes
                 if self.escQuote:
                     ret = ret.replace(self.escQuote, self.endQuoteChar)
 
         return loc, ret
 
     def __str__( self ):
         try:
             return super(QuotedString,self).__str__()
         except:
             pass
 
         if self.strRepr is None:
             self.strRepr = "quoted string, starting with %s ending with %s" % (self.quoteChar, self.endQuoteChar)
 
         return self.strRepr
 
 
 class CharsNotIn(Token):
     """Token for matching words composed of characters *not* in a given set.
        Defined with string containing all disallowed characters, and an optional
        minimum, maximum, and/or exact length.  The default value for min is 1 (a
        minimum value < 1 is not valid); the default values for max and exact
        are 0, meaning no maximum or exact length restriction.
     """
     def __init__( self, notChars, min=1, max=0, exact=0 ):
         super(CharsNotIn,self).__init__()
         self.skipWhitespace = False
         self.notChars = notChars
 
         if min < 1:
             raise ValueError("cannot specify a minimum length < 1; use Optional(CharsNotIn()) if zero-length char group is permitted")
 
         self.minLen = min
 
         if max > 0:
             self.maxLen = max
         else:
             self.maxLen = _MAX_INT
 
         if exact > 0:
             self.maxLen = exact
             self.minLen = exact
 
         self.name = _ustr(self)
         self.errmsg = "Expected " + self.name
         self.mayReturnEmpty = ( self.minLen == 0 )
         #self.myException.msg = self.errmsg
         self.mayIndexError = False
 
     def parseImpl( self, instring, loc, doActions=True ):
         if instring[loc] in self.notChars:
             #~ raise ParseException( instring, loc, self.errmsg )
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
 
         start = loc
         loc += 1
         notchars = self.notChars
         maxlen = min( start+self.maxLen, len(instring) )
         while loc < maxlen and \
               (instring[loc] not in notchars):
             loc += 1
 
         if loc - start < self.minLen:
             #~ raise ParseException( instring, loc, self.errmsg )
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
 
         return loc, instring[start:loc]
 
     def __str__( self ):
         try:
             return super(CharsNotIn, self).__str__()
         except:
             pass
 
         if self.strRepr is None:
             if len(self.notChars) > 4:
                 self.strRepr = "!W:(%s...)" % self.notChars[:4]
             else:
                 self.strRepr = "!W:(%s)" % self.notChars
 
         return self.strRepr
 
 class White(Token):
     """Special matching class for matching whitespace.  Normally, whitespace is ignored
        by pyparsing grammars.  This class is included when some whitespace structures
        are significant.  Define with a string containing the whitespace characters to be
        matched; default is " \\t\\r\\n".  Also takes optional min, max, and exact arguments,
        as defined for the Word class."""
     whiteStrs = {
         " " : "<SPC>",
         "\t": "<TAB>",
         "\n": "<LF>",
         "\r": "<CR>",
         "\f": "<FF>",
         }
     def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0):
         super(White,self).__init__()
         self.matchWhite = ws
         self.setWhitespaceChars( "".join([c for c in self.whiteChars if c not in self.matchWhite]) )
         #~ self.leaveWhitespace()
         self.name = ("".join([White.whiteStrs[c] for c in self.matchWhite]))
         self.mayReturnEmpty = True
         self.errmsg = "Expected " + self.name
         #self.myException.msg = self.errmsg
 
         self.minLen = min
 
         if max > 0:
             self.maxLen = max
         else:
             self.maxLen = _MAX_INT
 
         if exact > 0:
             self.maxLen = exact
             self.minLen = exact
 
     def parseImpl( self, instring, loc, doActions=True ):
         if not(instring[ loc ] in self.matchWhite):
             #~ raise ParseException( instring, loc, self.errmsg )
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
         start = loc
         loc += 1
         maxloc = start + self.maxLen
         maxloc = min( maxloc, len(instring) )
         while loc < maxloc and instring[loc] in self.matchWhite:
             loc += 1
 
         if loc - start < self.minLen:
             #~ raise ParseException( instring, loc, self.errmsg )
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
 
         return loc, instring[start:loc]
 
 
 class _PositionToken(Token):
     def __init__( self ):
         super(_PositionToken,self).__init__()
         self.name=self.__class__.__name__
         self.mayReturnEmpty = True
         self.mayIndexError = False
 
 class GoToColumn(_PositionToken):
     """Token to advance to a specific column of input text; useful for tabular report scraping."""
     def __init__( self, colno ):
         super(GoToColumn,self).__init__()
         self.col = colno
 
     def preParse( self, instring, loc ):
         if col(loc,instring) != self.col:
             instrlen = len(instring)
             if self.ignoreExprs:
                 loc = self._skipIgnorables( instring, loc )
             while loc < instrlen and instring[loc].isspace() and col( loc, instring ) != self.col :
                 loc += 1
         return loc
 
     def parseImpl( self, instring, loc, doActions=True ):
         thiscol = col( loc, instring )
         if thiscol > self.col:
             raise ParseException( instring, loc, "Text not in expected column", self )
         newloc = loc + self.col - thiscol
         ret = instring[ loc: newloc ]
         return newloc, ret
 
 class LineStart(_PositionToken):
     """Matches if current position is at the beginning of a line within the parse string"""
     def __init__( self ):
         super(LineStart,self).__init__()
         self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") )
         self.errmsg = "Expected start of line"
         #self.myException.msg = self.errmsg
 
     def preParse( self, instring, loc ):
         preloc = super(LineStart,self).preParse(instring,loc)
         if instring[preloc] == "\n":
             loc += 1
         return loc
 
     def parseImpl( self, instring, loc, doActions=True ):
         if not( loc==0 or
             (loc == self.preParse( instring, 0 )) or
             (instring[loc-1] == "\n") ): #col(loc, instring) != 1:
             #~ raise ParseException( instring, loc, "Expected start of line" )
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
         return loc, []
 
 class LineEnd(_PositionToken):
     """Matches if current position is at the end of a line within the parse string"""
     def __init__( self ):
         super(LineEnd,self).__init__()
         self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") )
         self.errmsg = "Expected end of line"
         #self.myException.msg = self.errmsg
 
     def parseImpl( self, instring, loc, doActions=True ):
         if loc<len(instring):
             if instring[loc] == "\n":
                 return loc+1, "\n"
             else:
                 #~ raise ParseException( instring, loc, "Expected end of line" )
                 exc = self.myException
                 exc.loc = loc
                 exc.pstr = instring
                 raise exc
         elif loc == len(instring):
             return loc+1, []
         else:
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
 
 class StringStart(_PositionToken):
     """Matches if current position is at the beginning of the parse string"""
     def __init__( self ):
         super(StringStart,self).__init__()
         self.errmsg = "Expected start of text"
         #self.myException.msg = self.errmsg
 
     def parseImpl( self, instring, loc, doActions=True ):
         if loc != 0:
             # see if entire string up to here is just whitespace and ignoreables
             if loc != self.preParse( instring, 0 ):
                 #~ raise ParseException( instring, loc, "Expected start of text" )
                 exc = self.myException
                 exc.loc = loc
                 exc.pstr = instring
                 raise exc
         return loc, []
 
 class StringEnd(_PositionToken):
     """Matches if current position is at the end of the parse string"""
     def __init__( self ):
         super(StringEnd,self).__init__()
         self.errmsg = "Expected end of text"
         #self.myException.msg = self.errmsg
 
     def parseImpl( self, instring, loc, doActions=True ):
         if loc < len(instring):
             #~ raise ParseException( instring, loc, "Expected end of text" )
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
         elif loc == len(instring):
             return loc+1, []
         elif loc > len(instring):
             return loc, []
         else:
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
 
 class WordStart(_PositionToken):
     """Matches if the current position is at the beginning of a Word, and
        is not preceded by any character in a given set of wordChars
        (default=printables). To emulate the \b behavior of regular expressions,
        use WordStart(alphanums). WordStart will also match at the beginning of
        the string being parsed, or at the beginning of a line.
     """
     def __init__(self, wordChars = printables):
         super(WordStart,self).__init__()
         self.wordChars = _str2dict(wordChars)
         self.errmsg = "Not at the start of a word"
 
     def parseImpl(self, instring, loc, doActions=True ):
         if loc != 0:
             if (instring[loc-1] in self.wordChars or
                 instring[loc] not in self.wordChars):
                 exc = self.myException
                 exc.loc = loc
                 exc.pstr = instring
                 raise exc
         return loc, []
 
 class WordEnd(_PositionToken):
     """Matches if the current position is at the end of a Word, and
        is not followed by any character in a given set of wordChars
        (default=printables). To emulate the \b behavior of regular expressions,
        use WordEnd(alphanums). WordEnd will also match at the end of
        the string being parsed, or at the end of a line.
     """
     def __init__(self, wordChars = printables):
         super(WordEnd,self).__init__()
         self.wordChars = _str2dict(wordChars)
         self.skipWhitespace = False
         self.errmsg = "Not at the end of a word"
 
     def parseImpl(self, instring, loc, doActions=True ):
         instrlen = len(instring)
         if instrlen>0 and loc<instrlen:
             if (instring[loc] in self.wordChars or
                 instring[loc-1] not in self.wordChars):
                 #~ raise ParseException( instring, loc, "Expected end of word" )
                 exc = self.myException
                 exc.loc = loc
                 exc.pstr = instring
                 raise exc
         return loc, []
 
 
 class ParseExpression(ParserElement):
     """Abstract subclass of ParserElement, for combining and post-processing parsed tokens."""
     def __init__( self, exprs, savelist = False ):
         super(ParseExpression,self).__init__(savelist)
         if isinstance( exprs, list ):
             self.exprs = exprs
         elif isinstance( exprs, basestring ):
             self.exprs = [ Literal( exprs ) ]
         else:
             try:
                 self.exprs = list( exprs )
             except TypeError:
                 self.exprs = [ exprs ]
         self.callPreparse = False
 
     def __getitem__( self, i ):
         return self.exprs[i]
 
     def append( self, other ):
         self.exprs.append( other )
         self.strRepr = None
         return self
 
     def leaveWhitespace( self ):
         """Extends leaveWhitespace defined in base class, and also invokes leaveWhitespace on
            all contained expressions."""
         self.skipWhitespace = False
         self.exprs = [ e.copy() for e in self.exprs ]
         for e in self.exprs:
             e.leaveWhitespace()
         return self
 
     def ignore( self, other ):
         if isinstance( other, Suppress ):
             if other not in self.ignoreExprs:
                 super( ParseExpression, self).ignore( other )
                 for e in self.exprs:
                     e.ignore( self.ignoreExprs[-1] )
         else:
             super( ParseExpression, self).ignore( other )
             for e in self.exprs:
                 e.ignore( self.ignoreExprs[-1] )
         return self
 
     def __str__( self ):
         try:
             return super(ParseExpression,self).__str__()
         except:
             pass
 
         if self.strRepr is None:
             self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.exprs) )
         return self.strRepr
 
     def streamline( self ):
         super(ParseExpression,self).streamline()
 
         for e in self.exprs:
             e.streamline()
 
         # collapse nested And's of the form And( And( And( a,b), c), d) to And( a,b,c,d )
         # but only if there are no parse actions or resultsNames on the nested And's
         # (likewise for Or's and MatchFirst's)
         if ( len(self.exprs) == 2 ):
             other = self.exprs[0]
             if ( isinstance( other, self.__class__ ) and
                   not(other.parseAction) and
                   other.resultsName is None and
                   not other.debug ):
                 self.exprs = other.exprs[:] + [ self.exprs[1] ]
                 self.strRepr = None
                 self.mayReturnEmpty |= other.mayReturnEmpty
                 self.mayIndexError  |= other.mayIndexError
 
             other = self.exprs[-1]
             if ( isinstance( other, self.__class__ ) and
                   not(other.parseAction) and
                   other.resultsName is None and
                   not other.debug ):
                 self.exprs = self.exprs[:-1] + other.exprs[:]
                 self.strRepr = None
                 self.mayReturnEmpty |= other.mayReturnEmpty
                 self.mayIndexError  |= other.mayIndexError
 
         return self
 
     def setResultsName( self, name, listAllMatches=False ):
         ret = super(ParseExpression,self).setResultsName(name,listAllMatches)
         return ret
 
     def validate( self, validateTrace=[] ):
         tmp = validateTrace[:]+[self]
         for e in self.exprs:
             e.validate(tmp)
         self.checkRecursion( [] )
 
 class And(ParseExpression):
     """Requires all given ParseExpressions to be found in the given order.
        Expressions may be separated by whitespace.
        May be constructed using the '+' operator.
     """
 
     class _ErrorStop(Empty):
         def __init__(self, *args, **kwargs):
             super(Empty,self).__init__(*args, **kwargs)
             self.leaveWhitespace()
 
     def __init__( self, exprs, savelist = True ):
         super(And,self).__init__(exprs, savelist)
         self.mayReturnEmpty = True
         for e in self.exprs:
             if not e.mayReturnEmpty:
                 self.mayReturnEmpty = False
                 break
         self.setWhitespaceChars( exprs[0].whiteChars )
         self.skipWhitespace = exprs[0].skipWhitespace
         self.callPreparse = True
 
     def parseImpl( self, instring, loc, doActions=True ):
         # pass False as last arg to _parse for first element, since we already
         # pre-parsed the string as part of our And pre-parsing
         loc, resultlist = self.exprs[0]._parse( instring, loc, doActions, callPreParse=False )
         errorStop = False
         for e in self.exprs[1:]:
             if isinstance(e, And._ErrorStop):
                 errorStop = True
                 continue
             if errorStop:
                 try:
                     loc, exprtokens = e._parse( instring, loc, doActions )
                 except ParseSyntaxException:
                     raise
                 except ParseBaseException, pe:
                     raise ParseSyntaxException(pe)
                 except IndexError, ie:
                     raise ParseSyntaxException( ParseException(instring, len(instring), self.errmsg, self) )
             else:
                 loc, exprtokens = e._parse( instring, loc, doActions )
             if exprtokens or exprtokens.keys():
                 resultlist += exprtokens
         return loc, resultlist
 
     def __iadd__(self, other ):
         if isinstance( other, basestring ):
             other = Literal( other )
         return self.append( other ) #And( [ self, other ] )
 
     def checkRecursion( self, parseElementList ):
         subRecCheckList = parseElementList[:] + [ self ]
         for e in self.exprs:
             e.checkRecursion( subRecCheckList )
             if not e.mayReturnEmpty:
                 break
 
     def __str__( self ):
         if hasattr(self,"name"):
             return self.name
 
         if self.strRepr is None:
             self.strRepr = "{" + " ".join( [ _ustr(e) for e in self.exprs ] ) + "}"
 
         return self.strRepr
 
 
 class Or(ParseExpression):
     """Requires that at least one ParseExpression is found.
        If two expressions match, the expression that matches the longest string will be used.
        May be constructed using the '^' operator.
     """
     def __init__( self, exprs, savelist = False ):
         super(Or,self).__init__(exprs, savelist)
         self.mayReturnEmpty = False
         for e in self.exprs:
             if e.mayReturnEmpty:
                 self.mayReturnEmpty = True
                 break
 
     def parseImpl( self, instring, loc, doActions=True ):
         maxExcLoc = -1
         maxMatchLoc = -1
         maxException = None
         for e in self.exprs:
             try:
                 loc2 = e.tryParse( instring, loc )
             except ParseException, err:
                 if err.loc > maxExcLoc:
                     maxException = err
                     maxExcLoc = err.loc
             except IndexError:
                 if len(instring) > maxExcLoc:
                     maxException = ParseException(instring,len(instring),e.errmsg,self)
                     maxExcLoc = len(instring)
             else:
                 if loc2 > maxMatchLoc:
                     maxMatchLoc = loc2
                     maxMatchExp = e
 
         if maxMatchLoc < 0:
             if maxException is not None:
                 raise maxException
             else:
                 raise ParseException(instring, loc, "no defined alternatives to match", self)
 
         return maxMatchExp._parse( instring, loc, doActions )
 
     def __ixor__(self, other ):
         if isinstance( other, basestring ):
             other = Literal( other )
         return self.append( other ) #Or( [ self, other ] )
 
     def __str__( self ):
         if hasattr(self,"name"):
             return self.name
 
         if self.strRepr is None:
             self.strRepr = "{" + " ^ ".join( [ _ustr(e) for e in self.exprs ] ) + "}"
 
         return self.strRepr
 
     def checkRecursion( self, parseElementList ):
         subRecCheckList = parseElementList[:] + [ self ]
         for e in self.exprs:
             e.checkRecursion( subRecCheckList )
 
 
 class MatchFirst(ParseExpression):
     """Requires that at least one ParseExpression is found.
        If two expressions match, the first one listed is the one that will match.
        May be constructed using the '|' operator.
     """
     def __init__( self, exprs, savelist = False ):
         super(MatchFirst,self).__init__(exprs, savelist)
         if exprs:
             self.mayReturnEmpty = False
             for e in self.exprs:
                 if e.mayReturnEmpty:
                     self.mayReturnEmpty = True
                     break
         else:
             self.mayReturnEmpty = True
 
     def parseImpl( self, instring, loc, doActions=True ):
         maxExcLoc = -1
         maxException = None
         for e in self.exprs:
             try:
                 ret = e._parse( instring, loc, doActions )
                 return ret
             except ParseException, err:
                 if err.loc > maxExcLoc:
                     maxException = err
                     maxExcLoc = err.loc
             except IndexError:
                 if len(instring) > maxExcLoc:
                     maxException = ParseException(instring,len(instring),e.errmsg,self)
                     maxExcLoc = len(instring)
 
         # only got here if no expression matched, raise exception for match that made it the furthest
         else:
             if maxException is not None:
                 raise maxException
             else:
                 raise ParseException(instring, loc, "no defined alternatives to match", self)
 
     def __ior__(self, other ):
         if isinstance( other, basestring ):
             other = Literal( other )
         return self.append( other ) #MatchFirst( [ self, other ] )
 
     def __str__( self ):
         if hasattr(self,"name"):
             return self.name
 
         if self.strRepr is None:
             self.strRepr = "{" + " | ".join( [ _ustr(e) for e in self.exprs ] ) + "}"
 
         return self.strRepr
 
     def checkRecursion( self, parseElementList ):
         subRecCheckList = parseElementList[:] + [ self ]
         for e in self.exprs:
             e.checkRecursion( subRecCheckList )
 
 
 class Each(ParseExpression):
     """Requires all given ParseExpressions to be found, but in any order.
        Expressions may be separated by whitespace.
        May be constructed using the '&' operator.
     """
     def __init__( self, exprs, savelist = True ):
         super(Each,self).__init__(exprs, savelist)
         self.mayReturnEmpty = True
         for e in self.exprs:
             if not e.mayReturnEmpty:
                 self.mayReturnEmpty = False
                 break
         self.skipWhitespace = True
         self.initExprGroups = True
 
     def parseImpl( self, instring, loc, doActions=True ):
         if self.initExprGroups:
             self.optionals = [ e.expr for e in self.exprs if isinstance(e,Optional) ]
             self.multioptionals = [ e.expr for e in self.exprs if isinstance(e,ZeroOrMore) ]
             self.multirequired = [ e.expr for e in self.exprs if isinstance(e,OneOrMore) ]
             self.required = [ e for e in self.exprs if not isinstance(e,(Optional,ZeroOrMore,OneOrMore)) ]
             self.required += self.multirequired
             self.initExprGroups = False
         tmpLoc = loc
         tmpReqd = self.required[:]
         tmpOpt  = self.optionals[:]
         matchOrder = []
 
         keepMatching = True
         while keepMatching:
             tmpExprs = tmpReqd + tmpOpt + self.multioptionals + self.multirequired
             failed = []
             for e in tmpExprs:
                 try:
                     tmpLoc = e.tryParse( instring, tmpLoc )
                 except ParseException:
                     failed.append(e)
                 else:
                     matchOrder.append(e)
                     if e in tmpReqd:
                         tmpReqd.remove(e)
                     elif e in tmpOpt:
                         tmpOpt.remove(e)
             if len(failed) == len(tmpExprs):
                 keepMatching = False
 
         if tmpReqd:
             missing = ", ".join( [ _ustr(e) for e in tmpReqd ] )
             raise ParseException(instring,loc,"Missing one or more required elements (%s)" % missing )
 
         # add any unmatched Optionals, in case they have default values defined
         matchOrder += list(e for e in self.exprs if isinstance(e,Optional) and e.expr in tmpOpt)
 
         resultlist = []
         for e in matchOrder:
             loc,results = e._parse(instring,loc,doActions)
             resultlist.append(results)
 
         finalResults = ParseResults([])
         for r in resultlist:
             dups = {}
             for k in r.keys():
                 if k in finalResults.keys():
                     tmp = ParseResults(finalResults[k])
                     tmp += ParseResults(r[k])
                     dups[k] = tmp
             finalResults += ParseResults(r)
-            for k,v in dups.items():
+            for k,v in dups.iteritems():
                 finalResults[k] = v
         return loc, finalResults
 
     def __str__( self ):
         if hasattr(self,"name"):
             return self.name
 
         if self.strRepr is None:
             self.strRepr = "{" + " & ".join( [ _ustr(e) for e in self.exprs ] ) + "}"
 
         return self.strRepr
 
     def checkRecursion( self, parseElementList ):
         subRecCheckList = parseElementList[:] + [ self ]
         for e in self.exprs:
             e.checkRecursion( subRecCheckList )
 
 
 class ParseElementEnhance(ParserElement):
     """Abstract subclass of ParserElement, for combining and post-processing parsed tokens."""
     def __init__( self, expr, savelist=False ):
         super(ParseElementEnhance,self).__init__(savelist)
         if isinstance( expr, basestring ):
             expr = Literal(expr)
         self.expr = expr
         self.strRepr = None
         if expr is not None:
             self.mayIndexError = expr.mayIndexError
             self.mayReturnEmpty = expr.mayReturnEmpty
             self.setWhitespaceChars( expr.whiteChars )
             self.skipWhitespace = expr.skipWhitespace
             self.saveAsList = expr.saveAsList
             self.callPreparse = expr.callPreparse
             self.ignoreExprs.extend(expr.ignoreExprs)
 
     def parseImpl( self, instring, loc, doActions=True ):
         if self.expr is not None:
             return self.expr._parse( instring, loc, doActions, callPreParse=False )
         else:
             raise ParseException("",loc,self.errmsg,self)
 
     def leaveWhitespace( self ):
         self.skipWhitespace = False
         self.expr = self.expr.copy()
         if self.expr is not None:
             self.expr.leaveWhitespace()
         return self
 
     def ignore( self, other ):
         if isinstance( other, Suppress ):
             if other not in self.ignoreExprs:
                 super( ParseElementEnhance, self).ignore( other )
                 if self.expr is not None:
                     self.expr.ignore( self.ignoreExprs[-1] )
         else:
             super( ParseElementEnhance, self).ignore( other )
             if self.expr is not None:
                 self.expr.ignore( self.ignoreExprs[-1] )
         return self
 
     def streamline( self ):
         super(ParseElementEnhance,self).streamline()
         if self.expr is not None:
             self.expr.streamline()
         return self
 
     def checkRecursion( self, parseElementList ):
         if self in parseElementList:
             raise RecursiveGrammarException( parseElementList+[self] )
         subRecCheckList = parseElementList[:] + [ self ]
         if self.expr is not None:
             self.expr.checkRecursion( subRecCheckList )
 
     def validate( self, validateTrace=[] ):
         tmp = validateTrace[:]+[self]
         if self.expr is not None:
             self.expr.validate(tmp)
         self.checkRecursion( [] )
 
     def __str__( self ):
         try:
             return super(ParseElementEnhance,self).__str__()
         except:
             pass
 
         if self.strRepr is None and self.expr is not None:
             self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.expr) )
         return self.strRepr
 
 
 class FollowedBy(ParseElementEnhance):
     """Lookahead matching of the given parse expression.  FollowedBy
     does *not* advance the parsing position within the input string, it only
     verifies that the specified parse expression matches at the current
     position.  FollowedBy always returns a null token list."""
     def __init__( self, expr ):
         super(FollowedBy,self).__init__(expr)
         self.mayReturnEmpty = True
 
     def parseImpl( self, instring, loc, doActions=True ):
         self.expr.tryParse( instring, loc )
         return loc, []
 
 
 class NotAny(ParseElementEnhance):
     """Lookahead to disallow matching with the given parse expression.  NotAny
     does *not* advance the parsing position within the input string, it only
     verifies that the specified parse expression does *not* match at the current
     position.  Also, NotAny does *not* skip over leading whitespace. NotAny
     always returns a null token list.  May be constructed using the '~' operator."""
     def __init__( self, expr ):
         super(NotAny,self).__init__(expr)
         #~ self.leaveWhitespace()
         self.skipWhitespace = False  # do NOT use self.leaveWhitespace(), don't want to propagate to exprs
         self.mayReturnEmpty = True
         self.errmsg = "Found unwanted token, "+_ustr(self.expr)
         #self.myException = ParseException("",0,self.errmsg,self)
 
     def parseImpl( self, instring, loc, doActions=True ):
         try:
             self.expr.tryParse( instring, loc )
         except (ParseException,IndexError):
             pass
         else:
             #~ raise ParseException(instring, loc, self.errmsg )
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
         return loc, []
 
     def __str__( self ):
         if hasattr(self,"name"):
             return self.name
 
         if self.strRepr is None:
             self.strRepr = "~{" + _ustr(self.expr) + "}"
 
         return self.strRepr
 
 
 class ZeroOrMore(ParseElementEnhance):
     """Optional repetition of zero or more of the given expression."""
     def __init__( self, expr ):
         super(ZeroOrMore,self).__init__(expr)
         self.mayReturnEmpty = True
 
     def parseImpl( self, instring, loc, doActions=True ):
         tokens = []
         try:
             loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
             hasIgnoreExprs = ( len(self.ignoreExprs) > 0 )
             while 1:
                 if hasIgnoreExprs:
                     preloc = self._skipIgnorables( instring, loc )
                 else:
                     preloc = loc
                 loc, tmptokens = self.expr._parse( instring, preloc, doActions )
                 if tmptokens or tmptokens.keys():
                     tokens += tmptokens
         except (ParseException,IndexError):
             pass
 
         return loc, tokens
 
     def __str__( self ):
         if hasattr(self,"name"):
             return self.name
 
         if self.strRepr is None:
             self.strRepr = "[" + _ustr(self.expr) + "]..."
 
         return self.strRepr
 
     def setResultsName( self, name, listAllMatches=False ):
         ret = super(ZeroOrMore,self).setResultsName(name,listAllMatches)
         ret.saveAsList = True
         return ret
 
 
 class OneOrMore(ParseElementEnhance):
     """Repetition of one or more of the given expression."""
     def parseImpl( self, instring, loc, doActions=True ):
         # must be at least one
         loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
         try:
             hasIgnoreExprs = ( len(self.ignoreExprs) > 0 )
             while 1:
                 if hasIgnoreExprs:
                     preloc = self._skipIgnorables( instring, loc )
                 else:
                     preloc = loc
                 loc, tmptokens = self.expr._parse( instring, preloc, doActions )
                 if tmptokens or tmptokens.keys():
                     tokens += tmptokens
         except (ParseException,IndexError):
             pass
 
         return loc, tokens
 
     def __str__( self ):
         if hasattr(self,"name"):
             return self.name
 
         if self.strRepr is None:
             self.strRepr = "{" + _ustr(self.expr) + "}..."
 
         return self.strRepr
 
     def setResultsName( self, name, listAllMatches=False ):
         ret = super(OneOrMore,self).setResultsName(name,listAllMatches)
         ret.saveAsList = True
         return ret
 
 class _NullToken(object):
     def __bool__(self):
         return False
     __nonzero__ = __bool__
     def __str__(self):
         return ""
 
 _optionalNotMatched = _NullToken()
 class Optional(ParseElementEnhance):
     """Optional matching of the given expression.
        A default return string can also be specified, if the optional expression
        is not found.
     """
     def __init__( self, exprs, default=_optionalNotMatched ):
         super(Optional,self).__init__( exprs, savelist=False )
         self.defaultValue = default
         self.mayReturnEmpty = True
 
     def parseImpl( self, instring, loc, doActions=True ):
         try:
             loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
         except (ParseException,IndexError):
             if self.defaultValue is not _optionalNotMatched:
                 if self.expr.resultsName:
                     tokens = ParseResults([ self.defaultValue ])
                     tokens[self.expr.resultsName] = self.defaultValue
                 else:
                     tokens = [ self.defaultValue ]
             else:
                 tokens = []
         return loc, tokens
 
     def __str__( self ):
         if hasattr(self,"name"):
             return self.name
 
         if self.strRepr is None:
             self.strRepr = "[" + _ustr(self.expr) + "]"
 
         return self.strRepr
 
 
 class SkipTo(ParseElementEnhance):
     """Token for skipping over all undefined text until the matched expression is found.
        If include is set to true, the matched expression is also parsed (the skipped text
        and matched expression are returned as a 2-element list).  The ignore
        argument is used to define grammars (typically quoted strings and comments) that
        might contain false matches.
     """
     def __init__( self, other, include=False, ignore=None, failOn=None ):
         super( SkipTo, self ).__init__( other )
         self.ignoreExpr = ignore
         self.mayReturnEmpty = True
         self.mayIndexError = False
         self.includeMatch = include
         self.asList = False
         if failOn is not None and isinstance(failOn, basestring):
             self.failOn = Literal(failOn)
         else:
             self.failOn = failOn
         self.errmsg = "No match found for "+_ustr(self.expr)
         #self.myException = ParseException("",0,self.errmsg,self)
 
     def parseImpl( self, instring, loc, doActions=True ):
         startLoc = loc
         instrlen = len(instring)
         expr = self.expr
         failParse = False
         while loc <= instrlen:
             try:
                 if self.failOn:
                     try:
                         self.failOn.tryParse(instring, loc)
                     except ParseBaseException:
                         pass
                     else:
                         failParse = True
                         raise ParseException(instring, loc, "Found expression " + str(self.failOn))
                     failParse = False
                 if self.ignoreExpr is not None:
                     while 1:
                         try:
                             loc = self.ignoreExpr.tryParse(instring,loc)
                             print "found ignoreExpr, advance to", loc
                         except ParseBaseException:
                             break
                 expr._parse( instring, loc, doActions=False, callPreParse=False )
                 skipText = instring[startLoc:loc]
                 if self.includeMatch:
                     loc,mat = expr._parse(instring,loc,doActions,callPreParse=False)
                     if mat:
                         skipRes = ParseResults( skipText )
                         skipRes += mat
                         return loc, [ skipRes ]
                     else:
                         return loc, [ skipText ]
                 else:
                     return loc, [ skipText ]
             except (ParseException,IndexError):
                 if failParse:
                     raise
                 else:
                     loc += 1
         exc = self.myException
         exc.loc = loc
         exc.pstr = instring
         raise exc
 
 class Forward(ParseElementEnhance):
     """Forward declaration of an expression to be defined later -
        used for recursive grammars, such as algebraic infix notation.
        When the expression is known, it is assigned to the Forward variable using the '<<' operator.
 
        Note: take care when assigning to Forward not to overlook precedence of operators.
        Specifically, '|' has a lower precedence than '<<', so that::
           fwdExpr << a | b | c
        will actually be evaluated as::
           (fwdExpr << a) | b | c
        thereby leaving b and c out as parseable alternatives.  It is recommended that you
        explicitly group the values inserted into the Forward::
           fwdExpr << (a | b | c)
     """
     def __init__( self, other=None ):
         super(Forward,self).__init__( other, savelist=False )
 
     def __lshift__( self, other ):
         if isinstance( other, basestring ):
             other = Literal(other)
         self.expr = other
         self.mayReturnEmpty = other.mayReturnEmpty
         self.strRepr = None
         self.mayIndexError = self.expr.mayIndexError
         self.mayReturnEmpty = self.expr.mayReturnEmpty
         self.setWhitespaceChars( self.expr.whiteChars )
         self.skipWhitespace = self.expr.skipWhitespace
         self.saveAsList = self.expr.saveAsList
         self.ignoreExprs.extend(self.expr.ignoreExprs)
         return None
 
     def leaveWhitespace( self ):
         self.skipWhitespace = False
         return self
 
     def streamline( self ):
         if not self.streamlined:
             self.streamlined = True
             if self.expr is not None:
                 self.expr.streamline()
         return self
 
     def validate( self, validateTrace=[] ):
         if self not in validateTrace:
             tmp = validateTrace[:]+[self]
             if self.expr is not None:
                 self.expr.validate(tmp)
         self.checkRecursion([])
 
     def __str__( self ):
         if hasattr(self,"name"):
             return self.name
 
         self._revertClass = self.__class__
         self.__class__ = _ForwardNoRecurse
         try:
             if self.expr is not None:
                 retString = _ustr(self.expr)
             else:
                 retString = "None"
         finally:
             self.__class__ = self._revertClass
         return self.__class__.__name__ + ": " + retString
 
     def copy(self):
         if self.expr is not None:
             return super(Forward,self).copy()
         else:
             ret = Forward()
             ret << self
             return ret
 
 class _ForwardNoRecurse(Forward):
     def __str__( self ):
         return "..."
 
 class TokenConverter(ParseElementEnhance):
     """Abstract subclass of ParseExpression, for converting parsed results."""
     def __init__( self, expr, savelist=False ):
         super(TokenConverter,self).__init__( expr )#, savelist )
         self.saveAsList = False
 
 class Upcase(TokenConverter):
     """Converter to upper case all matching tokens."""
     def __init__(self, *args):
         super(Upcase,self).__init__(*args)
         warnings.warn("Upcase class is deprecated, use upcaseTokens parse action instead",
                        DeprecationWarning,stacklevel=2)
 
     def postParse( self, instring, loc, tokenlist ):
         return list(map( string.upper, tokenlist ))
 
 
 class Combine(TokenConverter):
     """Converter to concatenate all matching tokens to a single string.
        By default, the matching patterns must also be contiguous in the input string;
        this can be disabled by specifying 'adjacent=False' in the constructor.
     """
     def __init__( self, expr, joinString="", adjacent=True ):
         super(Combine,self).__init__( expr )
         # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself
         if adjacent:
             self.leaveWhitespace()
         self.adjacent = adjacent
         self.skipWhitespace = True
         self.joinString = joinString
 
     def ignore( self, other ):
         if self.adjacent:
             ParserElement.ignore(self, other)
         else:
             super( Combine, self).ignore( other )
         return self
 
     def postParse( self, instring, loc, tokenlist ):
         retToks = tokenlist.copy()
         del retToks[:]
         retToks += ParseResults([ "".join(tokenlist._asStringList(self.joinString)) ], modal=self.modalResults)
 
         if self.resultsName and len(retToks.keys())>0:
             return [ retToks ]
         else:
             return retToks
 
 class Group(TokenConverter):
     """Converter to return the matched tokens as a list - useful for returning tokens of ZeroOrMore and OneOrMore expressions."""
     def __init__( self, expr ):
         super(Group,self).__init__( expr )
         self.saveAsList = True
 
     def postParse( self, instring, loc, tokenlist ):
         return [ tokenlist ]
 
 class Dict(TokenConverter):
     """Converter to return a repetitive expression as a list, but also as a dictionary.
        Each element can also be referenced using the first token in the expression as its key.
        Useful for tabular report scraping when the first column can be used as a item key.
     """
     def __init__( self, exprs ):
         super(Dict,self).__init__( exprs )
         self.saveAsList = True
 
     def postParse( self, instring, loc, tokenlist ):
         for i,tok in enumerate(tokenlist):
             if len(tok) == 0:
                 continue
             ikey = tok[0]
             if isinstance(ikey,int):
                 ikey = _ustr(tok[0]).strip()
             if len(tok)==1:
                 tokenlist[ikey] = _ParseResultsWithOffset("",i)
             elif len(tok)==2 and not isinstance(tok[1],ParseResults):
                 tokenlist[ikey] = _ParseResultsWithOffset(tok[1],i)
             else:
                 dictvalue = tok.copy() #ParseResults(i)
                 del dictvalue[0]
                 if len(dictvalue)!= 1 or (isinstance(dictvalue,ParseResults) and dictvalue.keys()):
                     tokenlist[ikey] = _ParseResultsWithOffset(dictvalue,i)
                 else:
                     tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0],i)
 
         if self.resultsName:
             return [ tokenlist ]
         else:
             return tokenlist
 
 
 class Suppress(TokenConverter):
     """Converter for ignoring the results of a parsed expression."""
     def postParse( self, instring, loc, tokenlist ):
         return []
 
     def suppress( self ):
         return self
 
 
 class OnlyOnce(object):
     """Wrapper for parse actions, to ensure they are only called once."""
     def __init__(self, methodCall):
         self.callable = ParserElement._normalizeParseActionArgs(methodCall)
         self.called = False
     def __call__(self,s,l,t):
         if not self.called:
             results = self.callable(s,l,t)
             self.called = True
             return results
         raise ParseException(s,l,"")
     def reset(self):
         self.called = False
 
 def traceParseAction(f):
     """Decorator for debugging parse actions."""
     f = ParserElement._normalizeParseActionArgs(f)
     def z(*paArgs):
         thisFunc = f.func_name
         s,l,t = paArgs[-3:]
         if len(paArgs)>3:
             thisFunc = paArgs[0].__class__.__name__ + '.' + thisFunc
         sys.stderr.write( ">>entering %s(line: '%s', %d, %s)\n" % (thisFunc,line(l,s),l,t) )
         try:
             ret = f(*paArgs)
         except Exception, exc:
             sys.stderr.write( "<<leaving %s (exception: %s)\n" % (thisFunc,exc) )
             raise
         sys.stderr.write( "<<leaving %s (ret: %s)\n" % (thisFunc,ret) )
         return ret
     try:
         z.__name__ = f.__name__
     except AttributeError:
         pass
     return z
 
 #
 # global helpers
 #
 def delimitedList( expr, delim=",", combine=False ):
     """Helper to define a delimited list of expressions - the delimiter defaults to ','.
        By default, the list elements and delimiters can have intervening whitespace, and
        comments, but this can be overridden by passing 'combine=True' in the constructor.
        If combine is set to True, the matching tokens are returned as a single token
        string, with the delimiters included; otherwise, the matching tokens are returned
        as a list of tokens, with the delimiters suppressed.
     """
     dlName = _ustr(expr)+" ["+_ustr(delim)+" "+_ustr(expr)+"]..."
     if combine:
         return Combine( expr + ZeroOrMore( delim + expr ) ).setName(dlName)
     else:
         return ( expr + ZeroOrMore( Suppress( delim ) + expr ) ).setName(dlName)
 
 def countedArray( expr ):
     """Helper to define a counted list of expressions.
        This helper defines a pattern of the form::
            integer expr expr expr...
        where the leading integer tells how many expr expressions follow.
        The matched tokens returns the array of expr tokens as a list - the leading count token is suppressed.
     """
     arrayExpr = Forward()
     def countFieldParseAction(s,l,t):
         n = int(t[0])
         arrayExpr << (n and Group(And([expr]*n)) or Group(empty))
         return []
     return ( Word(nums).setName("arrayLen").setParseAction(countFieldParseAction, callDuringTry=True) + arrayExpr )
 
 def _flatten(L):
     if type(L) is not list: return [L]
     if L == []: return L
     return _flatten(L[0]) + _flatten(L[1:])
 
 def matchPreviousLiteral(expr):
     """Helper to define an expression that is indirectly defined from
        the tokens matched in a previous expression, that is, it looks
        for a 'repeat' of a previous expression.  For example::
            first = Word(nums)
            second = matchPreviousLiteral(first)
            matchExpr = first + ":" + second
        will match "1:1", but not "1:2".  Because this matches a
        previous literal, will also match the leading "1:1" in "1:10".
        If this is not desired, use matchPreviousExpr.
        Do *not* use with packrat parsing enabled.
     """
     rep = Forward()
     def copyTokenToRepeater(s,l,t):
         if t:
             if len(t) == 1:
                 rep << t[0]
             else:
                 # flatten t tokens
                 tflat = _flatten(t.asList())
                 rep << And( [ Literal(tt) for tt in tflat ] )
         else:
             rep << Empty()
     expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
     return rep
 
 def matchPreviousExpr(expr):
     """Helper to define an expression that is indirectly defined from
        the tokens matched in a previous expression, that is, it looks
        for a 'repeat' of a previous expression.  For example::
            first = Word(nums)
            second = matchPreviousExpr(first)
            matchExpr = first + ":" + second
        will match "1:1", but not "1:2".  Because this matches by
        expressions, will *not* match the leading "1:1" in "1:10";
        the expressions are evaluated first, and then compared, so
        "1" is compared with "10".
        Do *not* use with packrat parsing enabled.
     """
     rep = Forward()
     e2 = expr.copy()
     rep << e2
     def copyTokenToRepeater(s,l,t):
         matchTokens = _flatten(t.asList())
         def mustMatchTheseTokens(s,l,t):
             theseTokens = _flatten(t.asList())
             if  theseTokens != matchTokens:
                 raise ParseException("",0,"")
         rep.setParseAction( mustMatchTheseTokens, callDuringTry=True )
     expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
     return rep
 
 def _escapeRegexRangeChars(s):
     #~  escape these chars: ^-]
     for c in r"\^-]":
         s = s.replace(c,_bslash+c)
     s = s.replace("\n",r"\n")
     s = s.replace("\t",r"\t")
     return _ustr(s)
 
 def oneOf( strs, caseless=False, useRegex=True ):
     """Helper to quickly define a set of alternative Literals, and makes sure to do
        longest-first testing when there is a conflict, regardless of the input order,
        but returns a MatchFirst for best performance.
 
        Parameters:
         - strs - a string of space-delimited literals, or a list of string literals
         - caseless - (default=False) - treat all literals as caseless
         - useRegex - (default=True) - as an optimization, will generate a Regex
           object; otherwise, will generate a MatchFirst object (if caseless=True, or
           if creating a Regex raises an exception)
     """
     if caseless:
         isequal = ( lambda a,b: a.upper() == b.upper() )
         masks = ( lambda a,b: b.upper().startswith(a.upper()) )
         parseElementClass = CaselessLiteral
     else:
         isequal = ( lambda a,b: a == b )
         masks = ( lambda a,b: b.startswith(a) )
         parseElementClass = Literal
 
     if isinstance(strs,(list,tuple)):
         symbols = list(strs[:])
     elif isinstance(strs,basestring):
         symbols = strs.split()
     else:
         warnings.warn("Invalid argument to oneOf, expected string or list",
                 SyntaxWarning, stacklevel=2)
 
     i = 0
     while i < len(symbols)-1:
         cur = symbols[i]
         for j,other in enumerate(symbols[i+1:]):
             if ( isequal(other, cur) ):
                 del symbols[i+j+1]
                 break
             elif ( masks(cur, other) ):
                 del symbols[i+j+1]
                 symbols.insert(i,other)
                 cur = other
                 break
         else:
             i += 1
 
     if not caseless and useRegex:
         #~ print (strs,"->", "|".join( [ _escapeRegexChars(sym) for sym in symbols] ))
         try:
             if len(symbols)==len("".join(symbols)):
                 return Regex( "[%s]" % "".join( [ _escapeRegexRangeChars(sym) for sym in symbols] ) )
             else:
                 return Regex( "|".join( [ re.escape(sym) for sym in symbols] ) )
         except:
             warnings.warn("Exception creating Regex for oneOf, building MatchFirst",
                     SyntaxWarning, stacklevel=2)
 
 
     # last resort, just use MatchFirst
     return MatchFirst( [ parseElementClass(sym) for sym in symbols ] )
 
 def dictOf( key, value ):
     """Helper to easily and clearly define a dictionary by specifying the respective patterns
        for the key and value.  Takes care of defining the Dict, ZeroOrMore, and Group tokens
        in the proper order.  The key pattern can include delimiting markers or punctuation,
        as long as they are suppressed, thereby leaving the significant key text.  The value
        pattern can include named results, so that the Dict results can include named token
        fields.
     """
     return Dict( ZeroOrMore( Group ( key + value ) ) )
 
 def originalTextFor(expr, asString=True):
     """Helper to return the original, untokenized text for a given expression.  Useful to
        restore the parsed fields of an HTML start tag into the raw tag text itself, or to
        revert separate tokens with intervening whitespace back to the original matching
        input text. Simpler to use than the parse action keepOriginalText, and does not
        require the inspect module to chase up the call stack.  By default, returns a 
        string containing the original parsed text.  
        
        If the optional asString argument is passed as False, then the return value is a 
        ParseResults containing any results names that were originally matched, and a 
        single token containing the original matched text from the input string.  So if 
        the expression passed to originalTextFor contains expressions with defined
        results names, you must set asString to False if you want to preserve those
        results name values."""
     locMarker = Empty().setParseAction(lambda s,loc,t: loc)
     matchExpr = locMarker("_original_start") + expr + locMarker("_original_end")
     if asString:
         extractText = lambda s,l,t: s[t._original_start:t._original_end]
     else:
         def extractText(s,l,t):
             del t[:]
             t.insert(0, s[t._original_start:t._original_end])
             del t["_original_start"]
             del t["_original_end"]
     matchExpr.setParseAction(extractText)
     return matchExpr
     
 # convenience constants for positional expressions
 empty       = Empty().setName("empty")
 lineStart   = LineStart().setName("lineStart")
 lineEnd     = LineEnd().setName("lineEnd")
 stringStart = StringStart().setName("stringStart")
 stringEnd   = StringEnd().setName("stringEnd")
 
 _escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1])
 _printables_less_backslash = "".join([ c for c in printables if c not in  r"\]" ])
 _escapedHexChar = Combine( Suppress(_bslash + "0x") + Word(hexnums) ).setParseAction(lambda s,l,t:unichr(int(t[0],16)))
 _escapedOctChar = Combine( Suppress(_bslash) + Word("0","01234567") ).setParseAction(lambda s,l,t:unichr(int(t[0],8)))
 _singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | Word(_printables_less_backslash,exact=1)
 _charRange = Group(_singleChar + Suppress("-") + _singleChar)
 _reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]"
 
 _expanded = lambda p: (isinstance(p,ParseResults) and ''.join([ unichr(c) for c in range(ord(p[0]),ord(p[1])+1) ]) or p)
 
 def srange(s):
     r"""Helper to easily define string ranges for use in Word construction.  Borrows
        syntax from regexp '[]' string range definitions::
           srange("[0-9]")   -> "0123456789"
           srange("[a-z]")   -> "abcdefghijklmnopqrstuvwxyz"
           srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_"
        The input string must be enclosed in []'s, and the returned string is the expanded
        character set joined into a single string.
        The values enclosed in the []'s may be::
           a single character
           an escaped character with a leading backslash (such as \- or \])
           an escaped hex character with a leading '\0x' (\0x21, which is a '!' character)
           an escaped octal character with a leading '\0' (\041, which is a '!' character)
           a range of any of the above, separated by a dash ('a-z', etc.)
           any combination of the above ('aeiouy', 'a-zA-Z0-9_$', etc.)
     """
     try:
         return "".join([_expanded(part) for part in _reBracketExpr.parseString(s).body])
     except:
         return ""
 
 def matchOnlyAtCol(n):
     """Helper method for defining parse actions that require matching at a specific
        column in the input text.
     """
     def verifyCol(strg,locn,toks):
         if col(locn,strg) != n:
             raise ParseException(strg,locn,"matched token not at column %d" % n)
     return verifyCol
 
 def replaceWith(replStr):
     """Helper method for common parse actions that simply return a literal value.  Especially
        useful when used with transformString().
     """
     def _replFunc(*args):
         return [replStr]
     return _replFunc
 
 def removeQuotes(s,l,t):
     """Helper parse action for removing quotation marks from parsed quoted strings.
        To use, add this parse action to quoted string using::
          quotedString.setParseAction( removeQuotes )
     """
     return t[0][1:-1]
 
 def upcaseTokens(s,l,t):
     """Helper parse action to convert tokens to upper case."""
     return [ tt.upper() for tt in map(_ustr,t) ]
 
 def downcaseTokens(s,l,t):
     """Helper parse action to convert tokens to lower case."""
     return [ tt.lower() for tt in map(_ustr,t) ]
 
 def keepOriginalText(s,startLoc,t):
     """Helper parse action to preserve original parsed text,
        overriding any nested parse actions."""
     try:
         endloc = getTokensEndLoc()
     except ParseException:
         raise ParseFatalException("incorrect usage of keepOriginalText - may only be called as a parse action")
     del t[:]
     t += ParseResults(s[startLoc:endloc])
     return t
 
 def getTokensEndLoc():
     """Method to be called from within a parse action to determine the end
        location of the parsed tokens."""
     import inspect
     fstack = inspect.stack()
     try:
         # search up the stack (through intervening argument normalizers) for correct calling routine
         for f in fstack[2:]:
             if f[3] == "_parseNoCache":
                 endloc = f[0].f_locals["loc"]
                 return endloc
         else:
             raise ParseFatalException("incorrect usage of getTokensEndLoc - may only be called from within a parse action")
     finally:
         del fstack
 
 def _makeTags(tagStr, xml):
     """Internal helper to construct opening and closing tag expressions, given a tag name"""
     if isinstance(tagStr,basestring):
         resname = tagStr
         tagStr = Keyword(tagStr, caseless=not xml)
     else:
         resname = tagStr.name
 
     tagAttrName = Word(alphas,alphanums+"_-:")
     if (xml):
         tagAttrValue = dblQuotedString.copy().setParseAction( removeQuotes )
         openTag = Suppress("<") + tagStr + \
                 Dict(ZeroOrMore(Group( tagAttrName + Suppress("=") + tagAttrValue ))) + \
                 Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
     else:
         printablesLessRAbrack = "".join( [ c for c in printables if c not in ">" ] )
         tagAttrValue = quotedString.copy().setParseAction( removeQuotes ) | Word(printablesLessRAbrack)
         openTag = Suppress("<") + tagStr + \
                 Dict(ZeroOrMore(Group( tagAttrName.setParseAction(downcaseTokens) + \
                 Optional( Suppress("=") + tagAttrValue ) ))) + \
                 Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
     closeTag = Combine(_L("</") + tagStr + ">")
 
     openTag = openTag.setResultsName("start"+"".join(resname.replace(":"," ").title().split())).setName("<%s>" % tagStr)
     closeTag = closeTag.setResultsName("end"+"".join(resname.replace(":"," ").title().split())).setName("</%s>" % tagStr)
 
     return openTag, closeTag
 
 def makeHTMLTags(tagStr):
     """Helper to construct opening and closing tag expressions for HTML, given a tag name"""
     return _makeTags( tagStr, False )
 
 def makeXMLTags(tagStr):
     """Helper to construct opening and closing tag expressions for XML, given a tag name"""
     return _makeTags( tagStr, True )
 
 def withAttribute(*args,**attrDict):
     """Helper to create a validating parse action to be used with start tags created
        with makeXMLTags or makeHTMLTags. Use withAttribute to qualify a starting tag
        with a required attribute value, to avoid false matches on common tags such as
        <TD> or <DIV>.
 
        Call withAttribute with a series of attribute names and values. Specify the list
        of filter attributes names and values as:
         - keyword arguments, as in (class="Customer",align="right"), or
         - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") )
        For attribute names with a namespace prefix, you must use the second form.  Attribute
        names are matched insensitive to upper/lower case.
 
        To verify that the attribute exists, but without specifying a value, pass
        withAttribute.ANY_VALUE as the value.
        """
     if args:
         attrs = args[:]
     else:
-        attrs = attrDict.items()
+        attrs = attrDict.iteritems()
     attrs = [(k,v) for k,v in attrs]
     def pa(s,l,tokens):
         for attrName,attrValue in attrs:
             if attrName not in tokens:
                 raise ParseException(s,l,"no matching attribute " + attrName)
             if attrValue != withAttribute.ANY_VALUE and tokens[attrName] != attrValue:
                 raise ParseException(s,l,"attribute '%s' has value '%s', must be '%s'" %
                                             (attrName, tokens[attrName], attrValue))
     return pa
 withAttribute.ANY_VALUE = object()
 
 opAssoc = _Constants()
 opAssoc.LEFT = object()
 opAssoc.RIGHT = object()
 
 def operatorPrecedence( baseExpr, opList ):
     """Helper method for constructing grammars of expressions made up of
        operators working in a precedence hierarchy.  Operators may be unary or
        binary, left- or right-associative.  Parse actions can also be attached
        to operator expressions.
 
        Parameters:
         - baseExpr - expression representing the most basic element for the nested
         - opList - list of tuples, one for each operator precedence level in the
           expression grammar; each tuple is of the form
           (opExpr, numTerms, rightLeftAssoc, parseAction), where:
            - opExpr is the pyparsing expression for the operator;
               may also be a string, which will be converted to a Literal;
               if numTerms is 3, opExpr is a tuple of two expressions, for the
               two operators separating the 3 terms
            - numTerms is the number of terms for this operator (must
               be 1, 2, or 3)
            - rightLeftAssoc is the indicator whether the operator is
               right or left associative, using the pyparsing-defined
               constants opAssoc.RIGHT and opAssoc.LEFT.
            - parseAction is the parse action to be associated with
               expressions matching this operator expression (the
               parse action tuple member may be omitted)
     """
     ret = Forward()
     lastExpr = baseExpr | ( Suppress('(') + ret + Suppress(')') )
     for i,operDef in enumerate(opList):
         opExpr,arity,rightLeftAssoc,pa = (operDef + (None,))[:4]
         if arity == 3:
             if opExpr is None or len(opExpr) != 2:
                 raise ValueError("if numterms=3, opExpr must be a tuple or list of two expressions")
             opExpr1, opExpr2 = opExpr
         thisExpr = Forward()#.setName("expr%d" % i)
         if rightLeftAssoc == opAssoc.LEFT:
             if arity == 1:
                 matchExpr = FollowedBy(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) )
             elif arity == 2:
                 if opExpr is not None:
                     matchExpr = FollowedBy(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) )
                 else:
                     matchExpr = FollowedBy(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) )
             elif arity == 3:
                 matchExpr = FollowedBy(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \
                             Group( lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr )
             else:
                 raise ValueError("operator must be unary (1), binary (2), or ternary (3)")
         elif rightLeftAssoc == opAssoc.RIGHT:
             if arity == 1:
                 # try to avoid LR with this extra test
                 if not isinstance(opExpr, Optional):
                     opExpr = Optional(opExpr)
                 matchExpr = FollowedBy(opExpr.expr + thisExpr) + Group( opExpr + thisExpr )
             elif arity == 2:
                 if opExpr is not None:
                     matchExpr = FollowedBy(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) )
                 else:
                     matchExpr = FollowedBy(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) )
             elif arity == 3:
                 matchExpr = FollowedBy(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \
                             Group( lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr )
             else:
                 raise ValueError("operator must be unary (1), binary (2), or ternary (3)")
         else:
             raise ValueError("operator must indicate right or left associativity")
         if pa:
             matchExpr.setParseAction( pa )
         thisExpr << ( matchExpr | lastExpr )
         lastExpr = thisExpr
     ret << lastExpr
     return ret
 
 dblQuotedString = Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\x[0-9a-fA-F]+)|(?:\\.))*"').setName("string enclosed in double quotes")
 sglQuotedString = Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\x[0-9a-fA-F]+)|(?:\\.))*'").setName("string enclosed in single quotes")
 quotedString = Regex(r'''(?:"(?:[^"\n\r\\]|(?:"")|(?:\\x[0-9a-fA-F]+)|(?:\\.))*")|(?:'(?:[^'\n\r\\]|(?:'')|(?:\\x[0-9a-fA-F]+)|(?:\\.))*')''').setName("quotedString using single or double quotes")
 unicodeString = Combine(_L('u') + quotedString.copy())
 
 def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString):
     """Helper method for defining nested lists enclosed in opening and closing
        delimiters ("(" and ")" are the default).
 
        Parameters:
         - opener - opening character for a nested list (default="("); can also be a pyparsing expression
         - closer - closing character for a nested list (default=")"); can also be a pyparsing expression
         - content - expression for items within the nested lists (default=None)
         - ignoreExpr - expression for ignoring opening and closing delimiters (default=quotedString)
 
        If an expression is not provided for the content argument, the nested
        expression will capture all whitespace-delimited content between delimiters
        as a list of separate values.
 
        Use the ignoreExpr argument to define expressions that may contain
        opening or closing characters that should not be treated as opening
        or closing characters for nesting, such as quotedString or a comment
        expression.  Specify multiple expressions using an Or or MatchFirst.
        The default is quotedString, but if no expressions are to be ignored,
        then pass None for this argument.
     """
     if opener == closer:
         raise ValueError("opening and closing strings cannot be the same")
     if content is None:
         if isinstance(opener,basestring) and isinstance(closer,basestring):
             if len(opener) == 1 and len(closer)==1:
                 if ignoreExpr is not None:
                     content = (Combine(OneOrMore(~ignoreExpr +
                                     CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS,exact=1))
                                 ).setParseAction(lambda t:t[0].strip()))
                 else:
                     content = (empty+CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS
                                 ).setParseAction(lambda t:t[0].strip()))
             else:
                 if ignoreExpr is not None:
                     content = (Combine(OneOrMore(~ignoreExpr + 
                                     ~Literal(opener) + ~Literal(closer) +
                                     CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1))
                                 ).setParseAction(lambda t:t[0].strip()))
                 else:
                     content = (Combine(OneOrMore(~Literal(opener) + ~Literal(closer) +
                                     CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1))
                                 ).setParseAction(lambda t:t[0].strip()))
         else:
             raise ValueError("opening and closing arguments must be strings if no content expression is given")
     ret = Forward()
     if ignoreExpr is not None:
         ret << Group( Suppress(opener) + ZeroOrMore( ignoreExpr | ret | content ) + Suppress(closer) )
     else:
         ret << Group( Suppress(opener) + ZeroOrMore( ret | content )  + Suppress(closer) )
     return ret
 
 def indentedBlock(blockStatementExpr, indentStack, indent=True):
     """Helper method for defining space-delimited indentation blocks, such as
        those used to define block statements in Python source code.
 
        Parameters:
         - blockStatementExpr - expression defining syntax of statement that
             is repeated within the indented block
         - indentStack - list created by caller to manage indentation stack
             (multiple statementWithIndentedBlock expressions within a single grammar
             should share a common indentStack)
         - indent - boolean indicating whether block must be indented beyond the
             the current level; set to False for block of left-most statements
             (default=True)
 
        A valid block must contain at least one blockStatement.
     """
     def checkPeerIndent(s,l,t):
         if l >= len(s): return
         curCol = col(l,s)
         if curCol != indentStack[-1]:
             if curCol > indentStack[-1]:
                 raise ParseFatalException(s,l,"illegal nesting")
             raise ParseException(s,l,"not a peer entry")
 
     def checkSubIndent(s,l,t):
         curCol = col(l,s)
         if curCol > indentStack[-1]:
             indentStack.append( curCol )
         else:
             raise ParseException(s,l,"not a subentry")
 
     def checkUnindent(s,l,t):
         if l >= len(s): return
         curCol = col(l,s)
         if not(indentStack and curCol < indentStack[-1] and curCol <= indentStack[-2]):
             raise ParseException(s,l,"not an unindent")
         indentStack.pop()
 
     NL = OneOrMore(LineEnd().setWhitespaceChars("\t ").suppress())
     INDENT = Empty() + Empty().setParseAction(checkSubIndent)
     PEER   = Empty().setParseAction(checkPeerIndent)
     UNDENT = Empty().setParseAction(checkUnindent)
     if indent:
         smExpr = Group( Optional(NL) +
             FollowedBy(blockStatementExpr) +
             INDENT + (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) + UNDENT)
     else:
         smExpr = Group( Optional(NL) +
             (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) )
     blockStatementExpr.ignore(_bslash + LineEnd())
     return smExpr
 
 alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]")
 punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]")
 
 anyOpenTag,anyCloseTag = makeHTMLTags(Word(alphas,alphanums+"_:"))
 commonHTMLEntity = Combine(_L("&") + oneOf("gt lt amp nbsp quot").setResultsName("entity") +";").streamline()
 _htmlEntityMap = dict(zip("gt lt amp nbsp quot".split(),'><& "'))
 replaceHTMLEntity = lambda t : t.entity in _htmlEntityMap and _htmlEntityMap[t.entity] or None
 
 # it's easy to get these comment structures wrong - they're very common, so may as well make them available
 cStyleComment = Regex(r"/\*(?:[^*]*\*+)+?/").setName("C style comment")
 
 htmlComment = Regex(r"<!--[\s\S]*?-->")
 restOfLine = Regex(r".*").leaveWhitespace()
 dblSlashComment = Regex(r"\/\/(\\\n|.)*").setName("// comment")
 cppStyleComment = Regex(r"/(?:\*(?:[^*]*\*+)+?/|/[^\n]*(?:\n[^\n]*)*?(?:(?<!\\)|\Z))").setName("C++ style comment")
 
 javaStyleComment = cppStyleComment
 pythonStyleComment = Regex(r"#.*").setName("Python style comment")
 _noncomma = "".join( [ c for c in printables if c != "," ] )
 _commasepitem = Combine(OneOrMore(Word(_noncomma) +
                                   Optional( Word(" \t") +
                                             ~Literal(",") + ~LineEnd() ) ) ).streamline().setName("commaItem")
 commaSeparatedList = delimitedList( Optional( quotedString | _commasepitem, default="") ).setName("commaSeparatedList")
 
 
 if __name__ == "__main__":
 
     def test( teststring ):
         try:
             tokens = simpleSQL.parseString( teststring )
             tokenlist = tokens.asList()
             print (teststring + "->"   + str(tokenlist))
             print ("tokens = "         + str(tokens))
             print ("tokens.columns = " + str(tokens.columns))
             print ("tokens.tables = "  + str(tokens.tables))
             print (tokens.asXML("SQL",True))
         except ParseBaseException,err:
             print (teststring + "->")
             print (err.line)
             print (" "*(err.column-1) + "^")
             print (err)
         print()
 
     selectToken    = CaselessLiteral( "select" )
     fromToken      = CaselessLiteral( "from" )
 
     ident          = Word( alphas, alphanums + "_$" )
     columnName     = delimitedList( ident, ".", combine=True ).setParseAction( upcaseTokens )
     columnNameList = Group( delimitedList( columnName ) )#.setName("columns")
     tableName      = delimitedList( ident, ".", combine=True ).setParseAction( upcaseTokens )
     tableNameList  = Group( delimitedList( tableName ) )#.setName("tables")
     simpleSQL      = ( selectToken + \
                      ( '*' | columnNameList ).setResultsName( "columns" ) + \
                      fromToken + \
                      tableNameList.setResultsName( "tables" ) )
 
     test( "SELECT * from XYZZY, ABC" )
     test( "select * from SYS.XYZZY" )
     test( "Select A from Sys.dual" )
     test( "Select AA,BB,CC from Sys.dual" )
     test( "Select A, B, C from Sys.dual" )
     test( "Select A, B, C from Sys.dual" )
     test( "Xelect A, B, C from Sys.dual" )
     test( "Select A, B, C frox Sys.dual" )
     test( "Select" )
     test( "Select ^^^ frox Sys.dual" )
     test( "Select A, B, C from Sys.dual, Table2   " )

IPython/kernel/launcher.py

@@ -1,834 +1,834
 #!/usr/bin/env python
 # encoding: utf-8
 """
 Facilities for launching IPython processes asynchronously.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import re
 import sys
 
 from IPython.config.configurable import Configurable
 from IPython.external import Itpl
 from IPython.utils.traitlets import Str, Int, List, Unicode
 from IPython.utils.path import get_ipython_module_path
 from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError
 from IPython.kernel.twistedutil import (
     gatherBoth,
     make_deferred,
     sleep_deferred
 )
 from IPython.kernel.winhpcjob import (
     IPControllerTask, IPEngineTask,
     IPControllerJob, IPEngineSetJob
 )
 
 from twisted.internet import reactor, defer
 from twisted.internet.defer import inlineCallbacks
 from twisted.internet.protocol import ProcessProtocol
 from twisted.internet.utils import getProcessOutput
 from twisted.internet.error import ProcessDone, ProcessTerminated
 from twisted.python import log
 from twisted.python.failure import Failure
 
 
 #-----------------------------------------------------------------------------
 # Paths to the kernel apps
 #-----------------------------------------------------------------------------
 
 
 ipcluster_cmd_argv = pycmd2argv(get_ipython_module_path(
     'IPython.kernel.ipclusterapp'
 ))
 
 ipengine_cmd_argv = pycmd2argv(get_ipython_module_path(
     'IPython.kernel.ipengineapp'
 ))
 
 ipcontroller_cmd_argv = pycmd2argv(get_ipython_module_path(
     'IPython.kernel.ipcontrollerapp'
 ))
 
 #-----------------------------------------------------------------------------
 # Base launchers and errors
 #-----------------------------------------------------------------------------
 
 
 class LauncherError(Exception):
     pass
 
 
 class ProcessStateError(LauncherError):
     pass
 
 
 class UnknownStatus(LauncherError):
     pass
 
 
 class BaseLauncher(Configurable):
     """An asbtraction for starting, stopping and signaling a process."""
 
     # In all of the launchers, the work_dir is where child processes will be
     # run. This will usually be the cluster_dir, but may not be. any work_dir
     # passed into the __init__ method will override the config value.
     # This should not be used to set the work_dir for the actual engine
     # and controller. Instead, use their own config files or the
     # controller_args, engine_args attributes of the launchers to add
     # the --work-dir option.
     work_dir = Unicode(u'')
 
     def __init__(self, work_dir=u'', config=None):
         super(BaseLauncher, self).__init__(work_dir=work_dir, config=config)
         self.state = 'before' # can be before, running, after
         self.stop_deferreds = []
         self.start_data = None
         self.stop_data = None
 
     @property
     def args(self):
         """A list of cmd and args that will be used to start the process.
 
         This is what is passed to :func:`spawnProcess` and the first element
         will be the process name.
         """
         return self.find_args()
 
     def find_args(self):
         """The ``.args`` property calls this to find the args list.
 
         Subcommand should implement this to construct the cmd and args.
         """
         raise NotImplementedError('find_args must be implemented in a subclass')
 
     @property
     def arg_str(self):
         """The string form of the program arguments."""
         return ' '.join(self.args)
 
     @property
     def running(self):
         """Am I running."""
         if self.state == 'running':
             return True
         else:
             return False
 
     def start(self):
         """Start the process.
 
         This must return a deferred that fires with information about the
         process starting (like a pid, job id, etc.).
         """
         return defer.fail(
             Failure(NotImplementedError(
                 'start must be implemented in a subclass')
             )
         )
 
     def stop(self):
         """Stop the process and notify observers of stopping.
 
         This must return a deferred that fires with information about the
         processing stopping, like errors that occur while the process is
         attempting to be shut down. This deferred won't fire when the process
         actually stops. To observe the actual process stopping, see
         :func:`observe_stop`.
         """
         return defer.fail(
             Failure(NotImplementedError(
                 'stop must be implemented in a subclass')
             )
         )
 
     def observe_stop(self):
         """Get a deferred that will fire when the process stops.
 
         The deferred will fire with data that contains information about
         the exit status of the process.
         """
         if self.state=='after':
             return defer.succeed(self.stop_data)
         else:
             d = defer.Deferred()
             self.stop_deferreds.append(d)
             return d
 
     def notify_start(self, data):
         """Call this to trigger startup actions.
 
         This logs the process startup and sets the state to 'running'.  It is
         a pass-through so it can be used as a callback.
         """
 
         log.msg('Process %r started: %r' % (self.args[0], data))
         self.start_data = data
         self.state = 'running'
         return data
 
     def notify_stop(self, data):
         """Call this to trigger process stop actions.
 
         This logs the process stopping and sets the state to 'after'. Call
         this to trigger all the deferreds from :func:`observe_stop`."""
 
         log.msg('Process %r stopped: %r' % (self.args[0], data))
         self.stop_data = data
         self.state = 'after'
         for i in range(len(self.stop_deferreds)):
             d = self.stop_deferreds.pop()
             d.callback(data)
         return data
 
     def signal(self, sig):
         """Signal the process.
 
         Return a semi-meaningless deferred after signaling the process.
 
         Parameters
         ----------
         sig : str or int
             'KILL', 'INT', etc., or any signal number
         """
         return defer.fail(
             Failure(NotImplementedError(
                 'signal must be implemented in a subclass')
             )
         )
 
 
 #-----------------------------------------------------------------------------
 # Local process launchers
 #-----------------------------------------------------------------------------
 
 
 class LocalProcessLauncherProtocol(ProcessProtocol):
     """A ProcessProtocol to go with the LocalProcessLauncher."""
 
     def __init__(self, process_launcher):
         self.process_launcher = process_launcher
         self.pid = None
 
     def connectionMade(self):
         self.pid = self.transport.pid
         self.process_launcher.notify_start(self.transport.pid)
 
     def processEnded(self, status):
         value = status.value
         if isinstance(value, ProcessDone):
             self.process_launcher.notify_stop(
                 {'exit_code':0,
                  'signal':None,
                  'status':None,
                  'pid':self.pid
                 }
             )
         elif isinstance(value, ProcessTerminated):
             self.process_launcher.notify_stop(
                 {'exit_code':value.exitCode,
                  'signal':value.signal,
                  'status':value.status,
                  'pid':self.pid
                 }
             )
         else:
             raise UnknownStatus("Unknown exit status, this is probably a "
                                 "bug in Twisted")
 
     def outReceived(self, data):
         log.msg(data)
 
     def errReceived(self, data):
         log.err(data)
 
 
 class LocalProcessLauncher(BaseLauncher):
     """Start and stop an external process in an asynchronous manner.
 
     This will launch the external process with a working directory of
     ``self.work_dir``.
     """
 
     # This is used to to construct self.args, which is passed to 
     # spawnProcess.
     cmd_and_args = List([])
 
     def __init__(self, work_dir=u'', config=None):
         super(LocalProcessLauncher, self).__init__(
             work_dir=work_dir, config=config
         )
         self.process_protocol = None
         self.start_deferred = None
 
     def find_args(self):
         return self.cmd_and_args
 
     def start(self):
         if self.state == 'before':
             self.process_protocol = LocalProcessLauncherProtocol(self)
             self.start_deferred = defer.Deferred()
             self.process_transport = reactor.spawnProcess(
                 self.process_protocol,
                 str(self.args[0]),  # twisted expects these to be str, not unicode
                 [str(a) for a in self.args],  # str expected, not unicode
                 env=os.environ,
                 path=self.work_dir  # start in the work_dir
             )
             return self.start_deferred
         else:
             s = 'The process was already started and has state: %r' % self.state
             return defer.fail(ProcessStateError(s))
 
     def notify_start(self, data):
         super(LocalProcessLauncher, self).notify_start(data)
         self.start_deferred.callback(data)
 
     def stop(self):
         return self.interrupt_then_kill()
 
     @make_deferred
     def signal(self, sig):
         if self.state == 'running':
             self.process_transport.signalProcess(sig)
 
     @inlineCallbacks
     def interrupt_then_kill(self, delay=2.0):
         """Send INT, wait a delay and then send KILL."""
         yield self.signal('INT')
         yield sleep_deferred(delay)
         yield self.signal('KILL')
 
 
 class LocalControllerLauncher(LocalProcessLauncher):
     """Launch a controller as a regular external process."""
 
     controller_cmd = List(ipcontroller_cmd_argv, config=True)
     # Command line arguments to ipcontroller.
     controller_args = List(['--log-to-file','--log-level', '40'], config=True)
 
     def find_args(self):
         return self.controller_cmd + self.controller_args
 
     def start(self, cluster_dir):
         """Start the controller by cluster_dir."""
         self.controller_args.extend(['--cluster-dir', cluster_dir])
         self.cluster_dir = unicode(cluster_dir)
         log.msg("Starting LocalControllerLauncher: %r" % self.args)
         return super(LocalControllerLauncher, self).start()
 
 
 class LocalEngineLauncher(LocalProcessLauncher):
     """Launch a single engine as a regular externall process."""
 
     engine_cmd = List(ipengine_cmd_argv, config=True)
     # Command line arguments for ipengine.
     engine_args = List(
         ['--log-to-file','--log-level', '40'], config=True
     )
 
     def find_args(self):
         return self.engine_cmd + self.engine_args
 
     def start(self, cluster_dir):
         """Start the engine by cluster_dir."""
         self.engine_args.extend(['--cluster-dir', cluster_dir])
         self.cluster_dir = unicode(cluster_dir)
         return super(LocalEngineLauncher, self).start()
 
 
 class LocalEngineSetLauncher(BaseLauncher):
     """Launch a set of engines as regular external processes."""
 
     # Command line arguments for ipengine.
     engine_args = List(
         ['--log-to-file','--log-level', '40'], config=True
     )
 
     def __init__(self, work_dir=u'', config=None):
         super(LocalEngineSetLauncher, self).__init__(
             work_dir=work_dir, config=config
         )
         self.launchers = []
 
     def start(self, n, cluster_dir):
         """Start n engines by profile or cluster_dir."""
         self.cluster_dir = unicode(cluster_dir)
         dlist = []
         for i in range(n):
             el = LocalEngineLauncher(work_dir=self.work_dir, config=self.config)
             # Copy the engine args over to each engine launcher.
             import copy
             el.engine_args = copy.deepcopy(self.engine_args)
             d = el.start(cluster_dir)
             if i==0:
                 log.msg("Starting LocalEngineSetLauncher: %r" % el.args)
             self.launchers.append(el)
             dlist.append(d)
         # The consumeErrors here could be dangerous
         dfinal = gatherBoth(dlist, consumeErrors=True)
         dfinal.addCallback(self.notify_start)
         return dfinal
 
     def find_args(self):
         return ['engine set']
 
     def signal(self, sig):
         dlist = []
         for el in self.launchers:
             d = el.signal(sig)
             dlist.append(d)
         dfinal = gatherBoth(dlist, consumeErrors=True)
         return dfinal
 
     def interrupt_then_kill(self, delay=1.0):
         dlist = []
         for el in self.launchers:
             d = el.interrupt_then_kill(delay)
             dlist.append(d)
         dfinal = gatherBoth(dlist, consumeErrors=True)
         return dfinal
 
     def stop(self):
         return self.interrupt_then_kill()
 
     def observe_stop(self):
         dlist = [el.observe_stop() for el in self.launchers]
         dfinal = gatherBoth(dlist, consumeErrors=False)
         dfinal.addCallback(self.notify_stop)
         return dfinal
 
 
 #-----------------------------------------------------------------------------
 # MPIExec launchers
 #-----------------------------------------------------------------------------
 
 
 class MPIExecLauncher(LocalProcessLauncher):
     """Launch an external process using mpiexec."""
 
     # The mpiexec command to use in starting the process.
     mpi_cmd = List(['mpiexec'], config=True)
     # The command line arguments to pass to mpiexec.
     mpi_args = List([], config=True)
     # The program to start using mpiexec.
     program = List(['date'], config=True)
     # The command line argument to the program.
     program_args = List([], config=True)
     # The number of instances of the program to start.
     n = Int(1, config=True)
 
     def find_args(self):
         """Build self.args using all the fields."""
         return self.mpi_cmd + ['-n', self.n] + self.mpi_args + \
                self.program + self.program_args
 
     def start(self, n):
         """Start n instances of the program using mpiexec."""
         self.n = n
         return super(MPIExecLauncher, self).start()
 
 
 class MPIExecControllerLauncher(MPIExecLauncher):
     """Launch a controller using mpiexec."""
 
     controller_cmd = List(ipcontroller_cmd_argv, config=True)
     # Command line arguments to ipcontroller.
     controller_args = List(['--log-to-file','--log-level', '40'], config=True)
     n = Int(1, config=False)
 
     def start(self, cluster_dir):
         """Start the controller by cluster_dir."""
         self.controller_args.extend(['--cluster-dir', cluster_dir])
         self.cluster_dir = unicode(cluster_dir)
         log.msg("Starting MPIExecControllerLauncher: %r" % self.args)
         return super(MPIExecControllerLauncher, self).start(1)
 
     def find_args(self):
         return self.mpi_cmd + ['-n', self.n] + self.mpi_args + \
                self.controller_cmd + self.controller_args
 
 
 class MPIExecEngineSetLauncher(MPIExecLauncher):
 
     engine_cmd = List(ipengine_cmd_argv, config=True)
     # Command line arguments for ipengine.
     engine_args = List(
         ['--log-to-file','--log-level', '40'], config=True
     )
     n = Int(1, config=True)
 
     def start(self, n, cluster_dir):
         """Start n engines by profile or cluster_dir."""
         self.engine_args.extend(['--cluster-dir', cluster_dir])
         self.cluster_dir = unicode(cluster_dir)
         self.n = n
         log.msg('Starting MPIExecEngineSetLauncher: %r' % self.args)
         return super(MPIExecEngineSetLauncher, self).start(n)
 
     def find_args(self):
         return self.mpi_cmd + ['-n', self.n] + self.mpi_args + \
                self.engine_cmd + self.engine_args
 
 
 #-----------------------------------------------------------------------------
 # SSH launchers
 #-----------------------------------------------------------------------------
 
 # TODO: Get SSH Launcher working again.
 
 class SSHLauncher(BaseLauncher):
     """A minimal launcher for ssh.
 
     To be useful this will probably have to be extended to use the ``sshx``
     idea for environment variables.  There could be other things this needs
     as well.
     """
 
     ssh_cmd = List(['ssh'], config=True)
     ssh_args = List([], config=True)
     program = List(['date'], config=True)
     program_args = List([], config=True)
     hostname = Str('', config=True)
     user = Str('', config=True)
     location = Str('')
 
     def _hostname_changed(self, name, old, new):
         self.location = '%s@%s' % (self.user, new)
 
     def _user_changed(self, name, old, new):
         self.location = '%s@%s' % (new, self.hostname)
 
     def find_args(self):
         return self.ssh_cmd + self.ssh_args + [self.location] + \
                self.program + self.program_args
 
     def start(self, n, hostname=None, user=None):
         if hostname is not None:
             self.hostname = hostname
         if user is not None:
             self.user = user
         return super(SSHLauncher, self).start()
 
 
 class SSHControllerLauncher(SSHLauncher):
     pass
 
 
 class SSHEngineSetLauncher(BaseLauncher):
     pass
 
 
 #-----------------------------------------------------------------------------
 # Windows HPC Server 2008 scheduler launchers
 #-----------------------------------------------------------------------------
 
 
 # This is only used on Windows.
 def find_job_cmd():
     if os.name=='nt':
         try:
             return find_cmd('job')
         except FindCmdError:
             return 'job'
     else:
         return 'job'
 
 
 class WindowsHPCLauncher(BaseLauncher):
 
     # A regular expression used to get the job id from the output of the 
     # submit_command.
     job_id_regexp = Str(r'\d+', config=True)
     # The filename of the instantiated job script.
     job_file_name = Unicode(u'ipython_job.xml', config=True)
     # The full path to the instantiated job script. This gets made dynamically
     # by combining the work_dir with the job_file_name.
     job_file = Unicode(u'')
     # The hostname of the scheduler to submit the job to
     scheduler = Str('', config=True)
     job_cmd = Str(find_job_cmd(), config=True)
 
     def __init__(self, work_dir=u'', config=None):
         super(WindowsHPCLauncher, self).__init__(
             work_dir=work_dir, config=config
         )
 
     @property
     def job_file(self):
         return os.path.join(self.work_dir, self.job_file_name)
 
     def write_job_file(self, n):
         raise NotImplementedError("Implement write_job_file in a subclass.")
 
     def find_args(self):
         return ['job.exe']
         
     def parse_job_id(self, output):
         """Take the output of the submit command and return the job id."""
         m = re.search(self.job_id_regexp, output)
         if m is not None:
             job_id = m.group()
         else:
             raise LauncherError("Job id couldn't be determined: %s" % output)
         self.job_id = job_id
         log.msg('Job started with job id: %r' % job_id)
         return job_id
 
     @inlineCallbacks
     def start(self, n):
         """Start n copies of the process using the Win HPC job scheduler."""
         self.write_job_file(n)
         args = [
             'submit',
             '/jobfile:%s' % self.job_file,
             '/scheduler:%s' % self.scheduler
         ]
         log.msg("Starting Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
         # Twisted will raise DeprecationWarnings if we try to pass unicode to this
         output = yield getProcessOutput(str(self.job_cmd),
             [str(a) for a in args],
             env=dict((str(k),str(v)) for k,v in os.environ.items()),
             path=self.work_dir
         )
         job_id = self.parse_job_id(output)
         self.notify_start(job_id)
         defer.returnValue(job_id)
 
     @inlineCallbacks
     def stop(self):
         args = [
             'cancel',
             self.job_id,
             '/scheduler:%s' % self.scheduler
         ]
         log.msg("Stopping Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
         try:
             # Twisted will raise DeprecationWarnings if we try to pass unicode to this
             output = yield getProcessOutput(str(self.job_cmd),
                 [str(a) for a in args],
-                env=dict((str(k),str(v)) for k,v in os.environ.items()),
+                env=dict((str(k),str(v)) for k,v in os.environ.iteritems()),
                 path=self.work_dir
             )
         except:
             output = 'The job already appears to be stoppped: %r' % self.job_id
         self.notify_stop(output)  # Pass the output of the kill cmd
         defer.returnValue(output)
 
 
 class WindowsHPCControllerLauncher(WindowsHPCLauncher):
 
     job_file_name = Unicode(u'ipcontroller_job.xml', config=True)
     extra_args = List([], config=False)
 
     def write_job_file(self, n):
         job = IPControllerJob(config=self.config)
 
         t = IPControllerTask(config=self.config)
         # The tasks work directory is *not* the actual work directory of 
         # the controller. It is used as the base path for the stdout/stderr
         # files that the scheduler redirects to.
         t.work_directory = self.cluster_dir
         # Add the --cluster-dir and from self.start().
         t.controller_args.extend(self.extra_args)
         job.add_task(t)
 
         log.msg("Writing job description file: %s" % self.job_file)
         job.write(self.job_file)
 
     @property
     def job_file(self):
         return os.path.join(self.cluster_dir, self.job_file_name)
 
     def start(self, cluster_dir):
         """Start the controller by cluster_dir."""
         self.extra_args = ['--cluster-dir', cluster_dir]
         self.cluster_dir = unicode(cluster_dir)
         return super(WindowsHPCControllerLauncher, self).start(1)
 
 
 class WindowsHPCEngineSetLauncher(WindowsHPCLauncher):
 
     job_file_name = Unicode(u'ipengineset_job.xml', config=True)
     extra_args = List([], config=False)
 
     def write_job_file(self, n):
         job = IPEngineSetJob(config=self.config)
 
         for i in range(n):
             t = IPEngineTask(config=self.config)
             # The tasks work directory is *not* the actual work directory of 
             # the engine. It is used as the base path for the stdout/stderr
             # files that the scheduler redirects to.
             t.work_directory = self.cluster_dir
             # Add the --cluster-dir and from self.start().
             t.engine_args.extend(self.extra_args)
             job.add_task(t)
 
         log.msg("Writing job description file: %s" % self.job_file)
         job.write(self.job_file)
 
     @property
     def job_file(self):
         return os.path.join(self.cluster_dir, self.job_file_name)
 
     def start(self, n, cluster_dir):
         """Start the controller by cluster_dir."""
         self.extra_args = ['--cluster-dir', cluster_dir]
         self.cluster_dir = unicode(cluster_dir)
         return super(WindowsHPCEngineSetLauncher, self).start(n)
 
 
 #-----------------------------------------------------------------------------
 # Batch (PBS) system launchers
 #-----------------------------------------------------------------------------
 
 # TODO: Get PBS launcher working again.
 
 class BatchSystemLauncher(BaseLauncher):
     """Launch an external process using a batch system.
 
     This class is designed to work with UNIX batch systems like PBS, LSF,
     GridEngine, etc.  The overall model is that there are different commands
     like qsub, qdel, etc. that handle the starting and stopping of the process.
 
     This class also has the notion of a batch script. The ``batch_template``
     attribute can be set to a string that is a template for the batch script.
     This template is instantiated using Itpl. Thus the template can use
     ${n} fot the number of instances. Subclasses can add additional variables
     to the template dict.
     """
 
     # Subclasses must fill these in.  See PBSEngineSet
     # The name of the command line program used to submit jobs.
     submit_command = Str('', config=True)
     # The name of the command line program used to delete jobs.
     delete_command = Str('', config=True)
     # A regular expression used to get the job id from the output of the 
     # submit_command.
     job_id_regexp = Str('', config=True)
     # The string that is the batch script template itself.
     batch_template = Str('', config=True)
     # The filename of the instantiated batch script.
     batch_file_name = Unicode(u'batch_script', config=True)
     # The full path to the instantiated batch script.
     batch_file = Unicode(u'')
 
     def __init__(self, work_dir=u'', config=None):
         super(BatchSystemLauncher, self).__init__(
             work_dir=work_dir, config=config
         )
         self.batch_file = os.path.join(self.work_dir, self.batch_file_name)
         self.context = {}
 
     def parse_job_id(self, output):
         """Take the output of the submit command and return the job id."""
         m = re.match(self.job_id_regexp, output)
         if m is not None:
             job_id = m.group()
         else:
             raise LauncherError("Job id couldn't be determined: %s" % output)
         self.job_id = job_id
         log.msg('Job started with job id: %r' % job_id)
         return job_id
 
     def write_batch_script(self, n):
         """Instantiate and write the batch script to the work_dir."""
         self.context['n'] = n
         script_as_string = Itpl.itplns(self.batch_template, self.context)
         log.msg('Writing instantiated batch script: %s' % self.batch_file)
         f = open(self.batch_file, 'w')
         f.write(script_as_string)
         f.close()
 
     @inlineCallbacks
     def start(self, n):
         """Start n copies of the process using a batch system."""
         self.write_batch_script(n)
         output = yield getProcessOutput(self.submit_command,
             [self.batch_file], env=os.environ)
         job_id = self.parse_job_id(output)
         self.notify_start(job_id)
         defer.returnValue(job_id)
 
     @inlineCallbacks
     def stop(self):
         output = yield getProcessOutput(self.delete_command,
             [self.job_id], env=os.environ
         )
         self.notify_stop(output)  # Pass the output of the kill cmd
         defer.returnValue(output)
 
 
 class PBSLauncher(BatchSystemLauncher):
     """A BatchSystemLauncher subclass for PBS."""
 
     submit_command = Str('qsub', config=True)
     delete_command = Str('qdel', config=True)
     job_id_regexp = Str(r'\d+', config=True)
     batch_template = Str('', config=True)
     batch_file_name = Unicode(u'pbs_batch_script', config=True)
     batch_file = Unicode(u'')
 
 
 class PBSControllerLauncher(PBSLauncher):
     """Launch a controller using PBS."""
 
     batch_file_name = Unicode(u'pbs_batch_script_controller', config=True)
 
     def start(self, cluster_dir):
         """Start the controller by profile or cluster_dir."""
         # Here we save profile and cluster_dir in the context so they
         # can be used in the batch script template as ${profile} and
         # ${cluster_dir}
         self.context['cluster_dir'] = cluster_dir
         self.cluster_dir = unicode(cluster_dir)
         log.msg("Starting PBSControllerLauncher: %r" % self.args)
         return super(PBSControllerLauncher, self).start(1)
 
 
 class PBSEngineSetLauncher(PBSLauncher):
 
     batch_file_name = Unicode(u'pbs_batch_script_engines', config=True)
 
     def start(self, n, cluster_dir):
         """Start n engines by profile or cluster_dir."""
         self.program_args.extend(['--cluster-dir', cluster_dir])
         self.cluster_dir = unicode(cluster_dir)
         log.msg('Starting PBSEngineSetLauncher: %r' % self.args)
         return super(PBSEngineSetLauncher, self).start(n)
 
 
 #-----------------------------------------------------------------------------
 # A launcher for ipcluster itself!
 #-----------------------------------------------------------------------------
 
 
 class IPClusterLauncher(LocalProcessLauncher):
     """Launch the ipcluster program in an external process."""
 
     ipcluster_cmd = List(ipcluster_cmd_argv, config=True)
     # Command line arguments to pass to ipcluster.
     ipcluster_args = List(
         ['--clean-logs', '--log-to-file', '--log-level', '40'], config=True)
     ipcluster_subcommand = Str('start')
     ipcluster_n = Int(2)
 
     def find_args(self):
         return self.ipcluster_cmd + [self.ipcluster_subcommand] + \
             ['-n', repr(self.ipcluster_n)] + self.ipcluster_args
 
     def start(self):
         log.msg("Starting ipcluster: %r" % self.args)
         return super(IPClusterLauncher, self).start()
 

IPython/kernel/tests/test_pendingdeferred.py

@@ -1,186 +1,186
 #!/usr/bin/env python
 # encoding: utf-8
 
 """Tests for pendingdeferred.py"""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 # Tell nose to skip this module
 __test__ = {}
 
 from twisted.internet import defer
 from twisted.python import failure
 
 from IPython.testing.util import DeferredTestCase
 import IPython.kernel.pendingdeferred as pd
 from IPython.kernel import error
 from IPython.kernel.util import printer
 
 
 class Foo(object):
 
     def bar(self, bahz):
         return defer.succeed('blahblah: %s' % bahz)
 
 class TwoPhaseFoo(pd.PendingDeferredManager):
 
     def __init__(self, foo):
         self.foo = foo
         pd.PendingDeferredManager.__init__(self)
 
     @pd.two_phase
     def bar(self, bahz):
         return self.foo.bar(bahz)
 
 class PendingDeferredManagerTest(DeferredTestCase):
 
     def setUp(self):
         self.pdm = pd.PendingDeferredManager()
     
     def tearDown(self):
         pass
 
     def testBasic(self):
         dDict = {}
         # Create 10 deferreds and save them
         for i in range(10):
             d = defer.Deferred()
             did = self.pdm.save_pending_deferred(d)
             dDict[did] = d
         # Make sure they are begin saved
-        for k in dDict.keys():
+        for k in dDict.iterkeys():
             self.assert_(self.pdm.quick_has_id(k))
         # Get the pending deferred (block=True), then callback with 'foo' and compare
         for did in dDict.keys()[0:5]:
             d = self.pdm.get_pending_deferred(did,block=True)
             dDict[did].callback('foo')
             d.addCallback(lambda r: self.assert_(r=='foo'))
         # Get the pending deferreds with (block=False) and make sure ResultNotCompleted is raised
         for did in dDict.keys()[5:10]:
             d = self.pdm.get_pending_deferred(did,block=False)
             d.addErrback(lambda f: self.assertRaises(error.ResultNotCompleted, f.raiseException))
         # Now callback the last 5, get them and compare.
         for did in dDict.keys()[5:10]:
             dDict[did].callback('foo')
             d = self.pdm.get_pending_deferred(did,block=False)
             d.addCallback(lambda r: self.assert_(r=='foo'))
 
     def test_save_then_delete(self):
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         self.assert_(self.pdm.quick_has_id(did))
         self.pdm.delete_pending_deferred(did)
         self.assert_(not self.pdm.quick_has_id(did))
 
     def test_save_get_delete(self):
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         d2 = self.pdm.get_pending_deferred(did,True)
         d2.addErrback(lambda f: self.assertRaises(error.AbortedPendingDeferredError, f.raiseException))
         self.pdm.delete_pending_deferred(did)
         return d2
 
     def test_double_get(self):
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         d2 = self.pdm.get_pending_deferred(did,True)
         d3 = self.pdm.get_pending_deferred(did,True)
         d3.addErrback(lambda f: self.assertRaises(error.InvalidDeferredID, f.raiseException))
 
     def test_get_after_callback(self):
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         d.callback('foo')
         d2 = self.pdm.get_pending_deferred(did,True)
         d2.addCallback(lambda r: self.assertEquals(r,'foo'))
         self.assert_(not self.pdm.quick_has_id(did))
 
     def test_get_before_callback(self):
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         d2 = self.pdm.get_pending_deferred(did,True)
         d.callback('foo')
         d2.addCallback(lambda r: self.assertEquals(r,'foo'))
         self.assert_(not self.pdm.quick_has_id(did))
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         d2 = self.pdm.get_pending_deferred(did,True)
         d2.addCallback(lambda r: self.assertEquals(r,'foo'))
         d.callback('foo')
         self.assert_(not self.pdm.quick_has_id(did))
 
     def test_get_after_errback(self):
         class MyError(Exception):
             pass
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         d.errback(failure.Failure(MyError('foo')))
         d2 = self.pdm.get_pending_deferred(did,True)
         d2.addErrback(lambda f: self.assertRaises(MyError, f.raiseException))
         self.assert_(not self.pdm.quick_has_id(did))
 
     def test_get_before_errback(self):
         class MyError(Exception):
             pass
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         d2 = self.pdm.get_pending_deferred(did,True)
         d.errback(failure.Failure(MyError('foo')))
         d2.addErrback(lambda f: self.assertRaises(MyError, f.raiseException))
         self.assert_(not self.pdm.quick_has_id(did))
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         d2 = self.pdm.get_pending_deferred(did,True)
         d2.addErrback(lambda f: self.assertRaises(MyError, f.raiseException))
         d.errback(failure.Failure(MyError('foo')))
         self.assert_(not self.pdm.quick_has_id(did))
     
     def test_noresult_noblock(self):
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         d2 = self.pdm.get_pending_deferred(did,False)
         d2.addErrback(lambda f: self.assertRaises(error.ResultNotCompleted, f.raiseException))
 
     def test_with_callbacks(self):
         d = defer.Deferred()
         d.addCallback(lambda r: r+' foo')
         d.addCallback(lambda r: r+' bar')
         did = self.pdm.save_pending_deferred(d)
         d2 = self.pdm.get_pending_deferred(did,True)
         d.callback('bam')
         d2.addCallback(lambda r: self.assertEquals(r,'bam foo bar'))
 
     def test_with_errbacks(self):
         class MyError(Exception):
             pass
         d = defer.Deferred()
         d.addCallback(lambda r: 'foo')
         d.addErrback(lambda f: 'caught error')
         did = self.pdm.save_pending_deferred(d)
         d2 = self.pdm.get_pending_deferred(did,True)
         d.errback(failure.Failure(MyError('bam')))
         d2.addErrback(lambda f: self.assertRaises(MyError, f.raiseException))
 
     def test_nested_deferreds(self):
         d = defer.Deferred()
         d2 = defer.Deferred()
         d.addCallback(lambda r: d2)
         did = self.pdm.save_pending_deferred(d)
         d.callback('foo')
         d3 = self.pdm.get_pending_deferred(did,False)
         d3.addErrback(lambda f: self.assertRaises(error.ResultNotCompleted, f.raiseException))
         d2.callback('bar')
         d3 = self.pdm.get_pending_deferred(did,False)
         d3.addCallback(lambda r: self.assertEquals(r,'bar'))
 

IPython/kernel/winhpcjob.py

@@ -1,316 +1,316
 #!/usr/bin/env python
 # encoding: utf-8
 """
 Job and task components for writing .xml files that the Windows HPC Server 
 2008 can use to start jobs.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
 
 import os
 import re
 import uuid
 
 from xml.etree import ElementTree as ET
 
 from IPython.config.configurable import Configurable
 from IPython.utils.traitlets import (
     Str, Int, List, Instance,
     Enum, Bool, CStr
 )
 
 #-----------------------------------------------------------------------------
 # Job and Task classes
 #-----------------------------------------------------------------------------
 
 
 def as_str(value):
     if isinstance(value, str):
         return value
     elif isinstance(value, bool):
         if value:
             return 'true'
         else:
             return 'false'
     elif isinstance(value, (int, float)):
         return repr(value)
     else:
         return value
 
 
 def indent(elem, level=0):
     i = "\n" + level*"  "
     if len(elem):
         if not elem.text or not elem.text.strip():
             elem.text = i + "  "
         if not elem.tail or not elem.tail.strip():
             elem.tail = i
         for elem in elem:
             indent(elem, level+1)
         if not elem.tail or not elem.tail.strip():
             elem.tail = i
     else:
         if level and (not elem.tail or not elem.tail.strip()):
             elem.tail = i
 
 
 def find_username():
     domain = os.environ.get('USERDOMAIN')
     username = os.environ.get('USERNAME','')
     if domain is None:
         return username
     else:
         return '%s\\%s' % (domain, username)
 
 
 class WinHPCJob(Configurable):
 
     job_id = Str('')
     job_name = Str('MyJob', config=True)
     min_cores = Int(1, config=True)
     max_cores = Int(1, config=True)
     min_sockets = Int(1, config=True)
     max_sockets = Int(1, config=True)
     min_nodes = Int(1, config=True)
     max_nodes = Int(1, config=True)
     unit_type = Str("Core", config=True)
     auto_calculate_min = Bool(True, config=True)
     auto_calculate_max = Bool(True, config=True)
     run_until_canceled = Bool(False, config=True)
     is_exclusive = Bool(False, config=True)
     username = Str(find_username(), config=True)
     job_type = Str('Batch', config=True)
     priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
         default_value='Highest', config=True)
     requested_nodes = Str('', config=True)
     project = Str('IPython', config=True)
     xmlns = Str('http://schemas.microsoft.com/HPCS2008/scheduler/')
     version = Str("2.000")
     tasks = List([])
 
     @property
     def owner(self):
         return self.username
 
     def _write_attr(self, root, attr, key):
         s = as_str(getattr(self, attr, ''))
         if s:
             root.set(key, s)
 
     def as_element(self):
         # We have to add _A_ type things to get the right order than
         # the MSFT XML parser expects.
         root = ET.Element('Job')
         self._write_attr(root, 'version', '_A_Version')
         self._write_attr(root, 'job_name', '_B_Name')
         self._write_attr(root, 'unit_type', '_C_UnitType')
         self._write_attr(root, 'min_cores', '_D_MinCores')
         self._write_attr(root, 'max_cores', '_E_MaxCores')
         self._write_attr(root, 'min_sockets', '_F_MinSockets')
         self._write_attr(root, 'max_sockets', '_G_MaxSockets')
         self._write_attr(root, 'min_nodes', '_H_MinNodes')
         self._write_attr(root, 'max_nodes', '_I_MaxNodes')
         self._write_attr(root, 'run_until_canceled', '_J_RunUntilCanceled')
         self._write_attr(root, 'is_exclusive', '_K_IsExclusive')
         self._write_attr(root, 'username', '_L_UserName')
         self._write_attr(root, 'job_type', '_M_JobType')
         self._write_attr(root, 'priority', '_N_Priority')
         self._write_attr(root, 'requested_nodes', '_O_RequestedNodes')
         self._write_attr(root, 'auto_calculate_max', '_P_AutoCalculateMax')
         self._write_attr(root, 'auto_calculate_min', '_Q_AutoCalculateMin')
         self._write_attr(root, 'project', '_R_Project')
         self._write_attr(root, 'owner', '_S_Owner')
         self._write_attr(root, 'xmlns', '_T_xmlns')
         dependencies = ET.SubElement(root, "Dependencies")
         etasks = ET.SubElement(root, "Tasks")
         for t in self.tasks:
             etasks.append(t.as_element())
         return root
 
     def tostring(self):
         """Return the string representation of the job description XML."""
         root = self.as_element()
         indent(root)
         txt = ET.tostring(root, encoding="utf-8")
         # Now remove the tokens used to order the attributes.
         txt = re.sub(r'_[A-Z]_','',txt)
         txt = '<?xml version="1.0" encoding="utf-8"?>\n' + txt
         return txt
 
     def write(self, filename):
         """Write the XML job description to a file."""
         txt = self.tostring()
         with open(filename, 'w') as f:
             f.write(txt)
 
     def add_task(self, task):
         """Add a task to the job.
 
         Parameters
         ----------
         task : :class:`WinHPCTask`
             The task object to add.
         """
         self.tasks.append(task)
 
 
 class WinHPCTask(Configurable):
 
     task_id = Str('')
     task_name = Str('')
     version = Str("2.000")
     min_cores = Int(1, config=True)
     max_cores = Int(1, config=True)
     min_sockets = Int(1, config=True)
     max_sockets = Int(1, config=True)
     min_nodes = Int(1, config=True)
     max_nodes = Int(1, config=True)
     unit_type = Str("Core", config=True)
     command_line = CStr('', config=True)
     work_directory = CStr('', config=True)
     is_rerunnaable = Bool(True, config=True)
     std_out_file_path = CStr('', config=True)
     std_err_file_path = CStr('', config=True)
     is_parametric = Bool(False, config=True)
     environment_variables = Instance(dict, args=(), config=True)
 
     def _write_attr(self, root, attr, key):
         s = as_str(getattr(self, attr, ''))
         if s:
             root.set(key, s)
 
     def as_element(self):
         root = ET.Element('Task')
         self._write_attr(root, 'version', '_A_Version')
         self._write_attr(root, 'task_name', '_B_Name')
         self._write_attr(root, 'min_cores', '_C_MinCores')
         self._write_attr(root, 'max_cores', '_D_MaxCores')
         self._write_attr(root, 'min_sockets', '_E_MinSockets')
         self._write_attr(root, 'max_sockets', '_F_MaxSockets')
         self._write_attr(root, 'min_nodes', '_G_MinNodes')
         self._write_attr(root, 'max_nodes', '_H_MaxNodes')
         self._write_attr(root, 'command_line', '_I_CommandLine')
         self._write_attr(root, 'work_directory', '_J_WorkDirectory')
         self._write_attr(root, 'is_rerunnaable', '_K_IsRerunnable')
         self._write_attr(root, 'std_out_file_path', '_L_StdOutFilePath')
         self._write_attr(root, 'std_err_file_path', '_M_StdErrFilePath')
         self._write_attr(root, 'is_parametric', '_N_IsParametric')
         self._write_attr(root, 'unit_type', '_O_UnitType')
         root.append(self.get_env_vars())
         return root
 
     def get_env_vars(self):
         env_vars = ET.Element('EnvironmentVariables')
-        for k, v in self.environment_variables.items():
+        for k, v in self.environment_variables.iteritems():
             variable = ET.SubElement(env_vars, "Variable")
             name = ET.SubElement(variable, "Name")
             name.text = k
             value = ET.SubElement(variable, "Value")
             value.text = v
         return env_vars
 
 
 
 # By declaring these, we can configure the controller and engine separately!
 
 class IPControllerJob(WinHPCJob):
     job_name = Str('IPController', config=False)
     is_exclusive = Bool(False, config=True)
     username = Str(find_username(), config=True)
     priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
         default_value='Highest', config=True)
     requested_nodes = Str('', config=True)
     project = Str('IPython', config=True)
 
 
 class IPEngineSetJob(WinHPCJob):
     job_name = Str('IPEngineSet', config=False)
     is_exclusive = Bool(False, config=True)
     username = Str(find_username(), config=True)
     priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
         default_value='Highest', config=True)
     requested_nodes = Str('', config=True)
     project = Str('IPython', config=True)
 
 
 class IPControllerTask(WinHPCTask):
 
     task_name = Str('IPController', config=True)
     controller_cmd = List(['ipcontroller.exe'], config=True)
     controller_args = List(['--log-to-file', '--log-level', '40'], config=True)
     # I don't want these to be configurable
     std_out_file_path = CStr('', config=False)
     std_err_file_path = CStr('', config=False)
     min_cores = Int(1, config=False)
     max_cores = Int(1, config=False)
     min_sockets = Int(1, config=False)
     max_sockets = Int(1, config=False)
     min_nodes = Int(1, config=False)
     max_nodes = Int(1, config=False)
     unit_type = Str("Core", config=False)
     work_directory = CStr('', config=False)
 
     def __init__(self, config=None):
         super(IPControllerTask, self).__init__(config=config)
         the_uuid = uuid.uuid1()
         self.std_out_file_path = os.path.join('log','ipcontroller-%s.out' % the_uuid)
         self.std_err_file_path = os.path.join('log','ipcontroller-%s.err' % the_uuid)
 
     @property
     def command_line(self):
         return ' '.join(self.controller_cmd + self.controller_args)
 
 
 class IPEngineTask(WinHPCTask):
 
     task_name = Str('IPEngine', config=True)
     engine_cmd = List(['ipengine.exe'], config=True)
     engine_args = List(['--log-to-file', '--log-level', '40'], config=True)
     # I don't want these to be configurable
     std_out_file_path = CStr('', config=False)
     std_err_file_path = CStr('', config=False)
     min_cores = Int(1, config=False)
     max_cores = Int(1, config=False)
     min_sockets = Int(1, config=False)
     max_sockets = Int(1, config=False)
     min_nodes = Int(1, config=False)
     max_nodes = Int(1, config=False)
     unit_type = Str("Core", config=False)
     work_directory = CStr('', config=False)
 
     def __init__(self, config=None):
         super(IPEngineTask,self).__init__(config=config)
         the_uuid = uuid.uuid1()
         self.std_out_file_path = os.path.join('log','ipengine-%s.out' % the_uuid)
         self.std_err_file_path = os.path.join('log','ipengine-%s.err' % the_uuid)
     
     @property
     def command_line(self):
         return ' '.join(self.engine_cmd + self.engine_args)
 
 
 # j = WinHPCJob(None)
 # j.job_name = 'IPCluster'
 # j.username = 'GNET\\bgranger'
 # j.requested_nodes = 'GREEN'
 # 
 # t = WinHPCTask(None)
 # t.task_name = 'Controller'
 # t.command_line = r"\\blue\domainusers$\bgranger\Python\Python25\Scripts\ipcontroller.exe --log-to-file -p default --log-level 10"
 # t.work_directory = r"\\blue\domainusers$\bgranger\.ipython\cluster_default"
 # t.std_out_file_path = 'controller-out.txt'
 # t.std_err_file_path = 'controller-err.txt'
 # t.environment_variables['PYTHONPATH'] = r"\\blue\domainusers$\bgranger\Python\Python25\Lib\site-packages"
 # j.add_task(t)