Feedback from pull request #1245

IPython/utils/encoding.py

@@ -0,0 +1,54 b''
+# coding: utf-8
+"""
+Utilities for dealing with text encodings
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2012  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+import sys
+import locale
+
+# to deal with the possibility of sys.std* not being a stream at all
+def get_stream_enc(stream, default=None):
+    """Return the given stream's encoding or a default.
+
+    There are cases where sys.std* might not actually be a stream, so
+    check for the encoding attribute prior to returning it, and return
+    a default if it doesn't exist or evaluates as False. `default'
+    is None if not provided.
+    """
+    if not hasattr(stream, 'encoding') or not stream.encoding:
+        return default
+    else:
+        return stream.encoding
+
+# Less conservative replacement for sys.getdefaultencoding, that will try
+# to match the environment.
+# Defined here as central function, so if we find better choices, we
+# won't need to make changes all over IPython.
+def getdefaultencoding():
+    """Return IPython's guess for the default encoding for bytes as text.
+
+    Asks for stdin.encoding first, to match the calling Terminal, but that
+    is often None for subprocesses.  Fall back on locale.getpreferredencoding()
+    which should be a sensible platform default (that respects LANG environment),
+    and finally to sys.getdefaultencoding() which is the most conservative option,
+    and usually ASCII.
+    """
+    enc = get_stream_enc(sys.stdin)
+    if not enc or enc=='ascii':
+        try:
+            # There are reports of getpreferredencoding raising errors
+            # in some cases, which may well be fixed, but let's be conservative here.
+            enc = locale.getpreferredencoding()
+        except Exception:
+            pass
+    return enc or sys.getdefaultencoding()

IPython/config/loader.py

@@ -1,702 +1,703 b''
 """A simple configuration system.
 
 Authors
 -------
 * Brian Granger
 * Fernando Perez
 * Min RK
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import __builtin__ as builtin_mod
 import os
 import re
 import sys
 
 from IPython.external import argparse
 from IPython.utils.path import filefind, get_ipython_dir
 from IPython.utils import py3compat, text, warn
+from IPython.utils.encoding import getdefaultencoding
 
 #-----------------------------------------------------------------------------
 # Exceptions
 #-----------------------------------------------------------------------------
 
 
 class ConfigError(Exception):
     pass
 
 class ConfigLoaderError(ConfigError):
     pass
 
 class ConfigFileNotFound(ConfigError):
     pass
 
 class ArgumentError(ConfigLoaderError):
     pass
 
 #-----------------------------------------------------------------------------
 # Argparse fix
 #-----------------------------------------------------------------------------
 
 # Unfortunately argparse by default prints help messages to stderr instead of
 # stdout.  This makes it annoying to capture long help screens at the command
 # line, since one must know how to pipe stderr, which many users don't know how
 # to do.  So we override the print_help method with one that defaults to
 # stdout and use our class instead.
 
 class ArgumentParser(argparse.ArgumentParser):
     """Simple argparse subclass that prints help to stdout by default."""
 
     def print_help(self, file=None):
         if file is None:
             file = sys.stdout
         return super(ArgumentParser, self).print_help(file)
 
     print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
 
 #-----------------------------------------------------------------------------
 # Config class for holding config information
 #-----------------------------------------------------------------------------
 
 
 class Config(dict):
     """An attribute based dict that can do smart merges."""
 
     def __init__(self, *args, **kwds):
         dict.__init__(self, *args, **kwds)
         # This sets self.__dict__ = self, but it has to be done this way
         # because we are also overriding __setattr__.
         dict.__setattr__(self, '__dict__', self)
 
     def _merge(self, other):
         to_update = {}
         for k, v in other.iteritems():
             if not self.has_key(k):
                 to_update[k] = v
             else: # I have this key
                 if isinstance(v, Config):
                     # Recursively merge common sub Configs
                     self[k]._merge(v)
                 else:
                     # Plain updates for non-Configs
                     to_update[k] = v
 
         self.update(to_update)
 
     def _is_section_key(self, key):
         if key[0].upper()==key[0] and not key.startswith('_'):
             return True
         else:
             return False
 
     def __contains__(self, key):
         if self._is_section_key(key):
             return True
         else:
             return super(Config, self).__contains__(key)
     # .has_key is deprecated for dictionaries.
     has_key = __contains__
 
     def _has_section(self, key):
         if self._is_section_key(key):
             if super(Config, self).__contains__(key):
                 return True
         return False
 
     def copy(self):
         return type(self)(dict.copy(self))
 
     def __copy__(self):
         return self.copy()
 
     def __deepcopy__(self, memo):
         import copy
         return type(self)(copy.deepcopy(self.items()))
 
     def __getitem__(self, key):
         # We cannot use directly self._is_section_key, because it triggers
         # infinite recursion on top of PyPy. Instead, we manually fish the
         # bound method.
         is_section_key = self.__class__._is_section_key.__get__(self)
 
         # Because we use this for an exec namespace, we need to delegate
         # the lookup of names in __builtin__ to itself.  This means
         # that you can't have section or attribute names that are
         # builtins.
         try:
             return getattr(builtin_mod, key)
         except AttributeError:
             pass
         if is_section_key(key):
             try:
                 return dict.__getitem__(self, key)
             except KeyError:
                 c = Config()
                 dict.__setitem__(self, key, c)
                 return c
         else:
             return dict.__getitem__(self, key)
 
     def __setitem__(self, key, value):
         # Don't allow names in __builtin__ to be modified.
         if hasattr(builtin_mod, key):
             raise ConfigError('Config variable names cannot have the same name '
                               'as a Python builtin: %s' % key)
         if self._is_section_key(key):
             if not isinstance(value, Config):
                 raise ValueError('values whose keys begin with an uppercase '
                                  'char must be Config instances: %r, %r' % (key, value))
         else:
             dict.__setitem__(self, key, value)
 
     def __getattr__(self, key):
         try:
             return self.__getitem__(key)
         except KeyError, e:
             raise AttributeError(e)
 
     def __setattr__(self, key, value):
         try:
             self.__setitem__(key, value)
         except KeyError, e:
             raise AttributeError(e)
 
     def __delattr__(self, key):
         try:
             dict.__delitem__(self, key)
         except KeyError, e:
             raise AttributeError(e)
 
 
 #-----------------------------------------------------------------------------
 # Config loading classes
 #-----------------------------------------------------------------------------
 
 
 class ConfigLoader(object):
     """A object for loading configurations from just about anywhere.
 
     The resulting configuration is packaged as a :class:`Struct`.
 
     Notes
     -----
     A :class:`ConfigLoader` does one thing: load a config from a source
     (file, command line arguments) and returns the data as a :class:`Struct`.
     There are lots of things that :class:`ConfigLoader` does not do.  It does
     not implement complex logic for finding config files.  It does not handle
     default values or merge multiple configs.  These things need to be
     handled elsewhere.
     """
 
     def __init__(self):
         """A base class for config loaders.
 
         Examples
         --------
 
         >>> cl = ConfigLoader()
         >>> config = cl.load_config()
         >>> config
         {}
         """
         self.clear()
 
     def clear(self):
         self.config = Config()
 
     def load_config(self):
         """Load a config from somewhere, return a :class:`Config` instance.
 
         Usually, this will cause self.config to be set and then returned.
         However, in most cases, :meth:`ConfigLoader.clear` should be called
         to erase any previous state.
         """
         self.clear()
         return self.config
 
 
 class FileConfigLoader(ConfigLoader):
     """A base class for file based configurations.
 
     As we add more file based config loaders, the common logic should go
     here.
     """
     pass
 
 
 class PyFileConfigLoader(FileConfigLoader):
     """A config loader for pure python files.
 
     This calls execfile on a plain python file and looks for attributes
     that are all caps.  These attribute are added to the config Struct.
     """
 
     def __init__(self, filename, path=None):
         """Build a config loader for a filename and path.
 
         Parameters
         ----------
         filename : str
             The file name of the config file.
         path : str, list, tuple
             The path to search for the config file on, or a sequence of
             paths to try in order.
         """
         super(PyFileConfigLoader, self).__init__()
         self.filename = filename
         self.path = path
         self.full_filename = ''
         self.data = None
 
     def load_config(self):
         """Load the config from a file and return it as a Struct."""
         self.clear()
         try:
             self._find_file()
         except IOError as e:
             raise ConfigFileNotFound(str(e))
         self._read_file_as_dict()
         self._convert_to_config()
         return self.config
 
     def _find_file(self):
         """Try to find the file by searching the paths."""
         self.full_filename = filefind(self.filename, self.path)
 
     def _read_file_as_dict(self):
         """Load the config file into self.config, with recursive loading."""
         # This closure is made available in the namespace that is used
         # to exec the config file.  It allows users to call
         # load_subconfig('myconfig.py') to load config files recursively.
         # It needs to be a closure because it has references to self.path
         # and self.config.  The sub-config is loaded with the same path
         # as the parent, but it uses an empty config which is then merged
         # with the parents.
 
         # If a profile is specified, the config file will be loaded
         # from that profile
 
         def load_subconfig(fname, profile=None):
             # import here to prevent circular imports
             from IPython.core.profiledir import ProfileDir, ProfileDirError
             if profile is not None:
                 try:
                     profile_dir = ProfileDir.find_profile_dir_by_name(
                             get_ipython_dir(),
                             profile,
                     )
                 except ProfileDirError:
                     return
                 path = profile_dir.location
             else:
                 path = self.path
             loader = PyFileConfigLoader(fname, path)
             try:
                 sub_config = loader.load_config()
             except ConfigFileNotFound:
                 # Pass silently if the sub config is not there. This happens
                 # when a user s using a profile, but not the default config.
                 pass
             else:
                 self.config._merge(sub_config)
 
         # Again, this needs to be a closure and should be used in config
         # files to get the config being loaded.
         def get_config():
             return self.config
 
         namespace = dict(load_subconfig=load_subconfig, get_config=get_config)
         fs_encoding = sys.getfilesystemencoding() or 'ascii'
         conf_filename = self.full_filename.encode(fs_encoding)
         py3compat.execfile(conf_filename, namespace)
 
     def _convert_to_config(self):
         if self.data is None:
             ConfigLoaderError('self.data does not exist')
 
 
 class CommandLineConfigLoader(ConfigLoader):
     """A config loader for command line arguments.
 
     As we add more command line based loaders, the common logic should go
     here.
     """
 
     def _exec_config_str(self, lhs, rhs):
         """execute self.config.<lhs>=<rhs>
         
         * expands ~ with expanduser
         * tries to assign with raw exec, otherwise assigns with just the string,
           allowing `--C.a=foobar` and `--C.a="foobar"` to be equivalent.  *Not*
           equivalent are `--C.a=4` and `--C.a='4'`.
         """
         rhs = os.path.expanduser(rhs)
         exec_str = 'self.config.' + lhs + '=' + rhs
         try:
             # Try to see if regular Python syntax will work. This
             # won't handle strings as the quote marks are removed
             # by the system shell.
             exec exec_str in locals(), globals()
         except (NameError, SyntaxError):
             # This case happens if the rhs is a string but without
             # the quote marks. Use repr, to get quote marks, and
             # 'u' prefix and see if
             # it succeeds. If it still fails, we let it raise.
             exec_str = u'self.config.' + lhs + '= rhs'
             exec exec_str in locals(), globals()
 
     def _load_flag(self, cfg):
         """update self.config from a flag, which can be a dict or Config"""
         if isinstance(cfg, (dict, Config)):
             # don't clobber whole config sections, update
             # each section from config:
             for sec,c in cfg.iteritems():
                 self.config[sec].update(c)
         else:
             raise TypeError("Invalid flag: %r" % cfg)
 
 # raw --identifier=value pattern
 # but *also* accept '-' as wordsep, for aliases
 # accepts:  --foo=a
 #           --Class.trait=value
 #           --alias-name=value
 # rejects:  -foo=value
 #           --foo
 #           --Class.trait
 kv_pattern = re.compile(r'\-\-[A-Za-z][\w\-]*(\.[\w\-]+)*\=.*')
 
 # just flags, no assignments, with two *or one* leading '-'
 # accepts:  --foo
 #           -foo-bar-again
 # rejects:  --anything=anything
 #           --two.word
 
 flag_pattern = re.compile(r'\-\-?\w+[\-\w]*$')
 
 class KeyValueConfigLoader(CommandLineConfigLoader):
     """A config loader that loads key value pairs from the command line.
 
     This allows command line options to be gives in the following form::
 
         ipython --profile="foo" --InteractiveShell.autocall=False
     """
 
     def __init__(self, argv=None, aliases=None, flags=None):
         """Create a key value pair config loader.
 
         Parameters
         ----------
         argv : list
             A list that has the form of sys.argv[1:] which has unicode
             elements of the form u"key=value". If this is None (default),
             then sys.argv[1:] will be used.
         aliases : dict
             A dict of aliases for configurable traits.
             Keys are the short aliases, Values are the resolved trait.
             Of the form: `{'alias' : 'Configurable.trait'}`
         flags : dict
             A dict of flags, keyed by str name. Vaues can be Config objects,
             dicts, or "key=value" strings.  If Config or dict, when the flag
             is triggered, The flag is loaded as `self.config.update(m)`.
 
         Returns
         -------
         config : Config
             The resulting Config object.
 
         Examples
         --------
 
             >>> from IPython.config.loader import KeyValueConfigLoader
             >>> cl = KeyValueConfigLoader()
             >>> cl.load_config(["--A.name='brian'","--B.number=0"])
             {'A': {'name': 'brian'}, 'B': {'number': 0}}
         """
         self.clear()
         if argv is None:
             argv = sys.argv[1:]
         self.argv = argv
         self.aliases = aliases or {}
         self.flags = flags or {}
 
 
     def clear(self):
         super(KeyValueConfigLoader, self).clear()
         self.extra_args = []
 
 
     def _decode_argv(self, argv, enc=None):
         """decode argv if bytes, using stin.encoding, falling back on default enc"""
         uargv = []
         if enc is None:
-            enc = py3compat.getdefaultencoding()
+            enc = getdefaultencoding()
         for arg in argv:
             if not isinstance(arg, unicode):
                 # only decode if not already decoded
                 arg = arg.decode(enc)
             uargv.append(arg)
         return uargv
 
 
     def load_config(self, argv=None, aliases=None, flags=None):
         """Parse the configuration and generate the Config object.
 
         After loading, any arguments that are not key-value or
         flags will be stored in self.extra_args - a list of
         unparsed command-line arguments.  This is used for
         arguments such as input files or subcommands.
 
         Parameters
         ----------
         argv : list, optional
             A list that has the form of sys.argv[1:] which has unicode
             elements of the form u"key=value". If this is None (default),
             then self.argv will be used.
         aliases : dict
             A dict of aliases for configurable traits.
             Keys are the short aliases, Values are the resolved trait.
             Of the form: `{'alias' : 'Configurable.trait'}`
         flags : dict
             A dict of flags, keyed by str name. Values can be Config objects
             or dicts.  When the flag is triggered, The config is loaded as
             `self.config.update(cfg)`.
         """
         from IPython.config.configurable import Configurable
 
         self.clear()
         if argv is None:
             argv = self.argv
         if aliases is None:
             aliases = self.aliases
         if flags is None:
             flags = self.flags
 
         # ensure argv is a list of unicode strings:
         uargv = self._decode_argv(argv)
         for idx,raw in enumerate(uargv):
             # strip leading '-'
             item = raw.lstrip('-')
 
             if raw == '--':
                 # don't parse arguments after '--'
                 # this is useful for relaying arguments to scripts, e.g.
                 # ipython -i foo.py --pylab=qt -- args after '--' go-to-foo.py
                 self.extra_args.extend(uargv[idx+1:])
                 break
 
             if kv_pattern.match(raw):
                 lhs,rhs = item.split('=',1)
                 # Substitute longnames for aliases.
                 if lhs in aliases:
                     lhs = aliases[lhs]
                 if '.' not in lhs:
                     # probably a mistyped alias, but not technically illegal
                     warn.warn("Unrecognized alias: '%s', it will probably have no effect."%lhs)
                 try:
                     self._exec_config_str(lhs, rhs)
                 except Exception:
                     raise ArgumentError("Invalid argument: '%s'" % raw)
 
             elif flag_pattern.match(raw):
                 if item in flags:
                     cfg,help = flags[item]
                     self._load_flag(cfg)
                 else:
                     raise ArgumentError("Unrecognized flag: '%s'"%raw)
             elif raw.startswith('-'):
                 kv = '--'+item
                 if kv_pattern.match(kv):
                     raise ArgumentError("Invalid argument: '%s', did you mean '%s'?"%(raw, kv))
                 else:
                     raise ArgumentError("Invalid argument: '%s'"%raw)
             else:
                 # keep all args that aren't valid in a list,
                 # in case our parent knows what to do with them.
                 self.extra_args.append(item)
         return self.config
 
 class ArgParseConfigLoader(CommandLineConfigLoader):
     """A loader that uses the argparse module to load from the command line."""
 
     def __init__(self, argv=None, aliases=None, flags=None, *parser_args, **parser_kw):
         """Create a config loader for use with argparse.
 
         Parameters
         ----------
 
         argv : optional, list
           If given, used to read command-line arguments from, otherwise
           sys.argv[1:] is used.
 
         parser_args : tuple
           A tuple of positional arguments that will be passed to the
           constructor of :class:`argparse.ArgumentParser`.
 
         parser_kw : dict
           A tuple of keyword arguments that will be passed to the
           constructor of :class:`argparse.ArgumentParser`.
 
         Returns
         -------
         config : Config
             The resulting Config object.
         """
         super(CommandLineConfigLoader, self).__init__()
         self.clear()
         if argv is None:
             argv = sys.argv[1:]
         self.argv = argv
         self.aliases = aliases or {}
         self.flags = flags or {}
 
         self.parser_args = parser_args
         self.version = parser_kw.pop("version", None)
         kwargs = dict(argument_default=argparse.SUPPRESS)
         kwargs.update(parser_kw)
         self.parser_kw = kwargs
 
     def load_config(self, argv=None, aliases=None, flags=None):
         """Parse command line arguments and return as a Config object.
 
         Parameters
         ----------
 
         args : optional, list
           If given, a list with the structure of sys.argv[1:] to parse
           arguments from. If not given, the instance's self.argv attribute
           (given at construction time) is used."""
         self.clear()
         if argv is None:
             argv = self.argv
         if aliases is None:
             aliases = self.aliases
         if flags is None:
             flags = self.flags
         self._create_parser(aliases, flags)
         self._parse_args(argv)
         self._convert_to_config()
         return self.config
 
     def get_extra_args(self):
         if hasattr(self, 'extra_args'):
             return self.extra_args
         else:
             return []
 
     def _create_parser(self, aliases=None, flags=None):
         self.parser = ArgumentParser(*self.parser_args, **self.parser_kw)
         self._add_arguments(aliases, flags)
 
     def _add_arguments(self, aliases=None, flags=None):
         raise NotImplementedError("subclasses must implement _add_arguments")
 
     def _parse_args(self, args):
         """self.parser->self.parsed_data"""
         # decode sys.argv to support unicode command-line options
-        enc = py3compat.getdefaultencoding()
+        enc = getdefaultencoding()
         uargs = [py3compat.cast_unicode(a, enc) for a in args]
         self.parsed_data, self.extra_args = self.parser.parse_known_args(uargs)
 
     def _convert_to_config(self):
         """self.parsed_data->self.config"""
         for k, v in vars(self.parsed_data).iteritems():
             exec "self.config.%s = v"%k in locals(), globals()
 
 class KVArgParseConfigLoader(ArgParseConfigLoader):
     """A config loader that loads aliases and flags with argparse,
     but will use KVLoader for the rest.  This allows better parsing
     of common args, such as `ipython -c 'print 5'`, but still gets
     arbitrary config with `ipython --InteractiveShell.use_readline=False`"""
 
     def _convert_to_config(self):
         """self.parsed_data->self.config"""
         for k, v in vars(self.parsed_data).iteritems():
             self._exec_config_str(k, v)
 
     def _add_arguments(self, aliases=None, flags=None):
         self.alias_flags = {}
         # print aliases, flags
         if aliases is None:
             aliases = self.aliases
         if flags is None:
             flags = self.flags
         paa = self.parser.add_argument
         for key,value in aliases.iteritems():
             if key in flags:
                 # flags
                 nargs = '?'
             else:
                 nargs = None
             if len(key) is 1:
                 paa('-'+key, '--'+key, type=unicode, dest=value, nargs=nargs)
             else:
                 paa('--'+key, type=unicode, dest=value, nargs=nargs)
         for key, (value, help) in flags.iteritems():
             if key in self.aliases:
                 #
                 self.alias_flags[self.aliases[key]] = value
                 continue
             if len(key) is 1:
                 paa('-'+key, '--'+key, action='append_const', dest='_flags', const=value)
             else:
                 paa('--'+key, action='append_const', dest='_flags', const=value)
 
     def _convert_to_config(self):
         """self.parsed_data->self.config, parse unrecognized extra args via KVLoader."""
         # remove subconfigs list from namespace before transforming the Namespace
         if '_flags' in self.parsed_data:
             subcs = self.parsed_data._flags
             del self.parsed_data._flags
         else:
             subcs = []
 
         for k, v in vars(self.parsed_data).iteritems():
             if v is None:
                 # it was a flag that shares the name of an alias
                 subcs.append(self.alias_flags[k])
             else:
                 # eval the KV assignment
                 self._exec_config_str(k, v)
 
         for subc in subcs:
             self._load_flag(subc)
 
         if self.extra_args:
             sub_parser = KeyValueConfigLoader()
             sub_parser.load_config(self.extra_args)
             self.config._merge(sub_parser.config)
             self.extra_args = sub_parser.extra_args
 
 
 def load_pyconfig_files(config_files, path):
     """Load multiple Python config files, merging each of them in turn.
 
     Parameters
     ==========
     config_files : list of str
         List of config files names to load and merge into the config.
     path : unicode
         The full path to the location of the config files.
     """
     config = Config()
     for cf in config_files:
         loader = PyFileConfigLoader(cf, path=path)
         try:
             next_config = loader.load_config()
         except ConfigFileNotFound:
             pass
         except:
             raise
         else:
             config._merge(next_config)
     return config

IPython/core/macro.py

@@ -1,59 +1,60 b''
 """Support for interactive macros in IPython"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2005 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 import re
 import sys
 
 from IPython.utils import py3compat
+from IPython.utils.encoding import getdefaultencoding
 
 coding_declaration = re.compile(r"#\s*coding[:=]\s*([-\w.]+)")
 
 class Macro(object):
     """Simple class to store the value of macros as strings.
 
     Macro is just a callable that executes a string of IPython
     input when called.
     
     Args to macro are available in _margv list if you need them.
     """
 
     def __init__(self,code):
         """store the macro value, as a single string which can be executed"""
         lines = []
         enc = None
         for line in code.splitlines():
             coding_match = coding_declaration.match(line)
             if coding_match:
                 enc = coding_match.group(1)
             else:
                 lines.append(line)
         code = "\n".join(lines)
         if isinstance(code, bytes):
-            code = code.decode(enc or py3compat.getdefaultencoding())
+            code = code.decode(enc or getdefaultencoding())
         self.value = code + '\n'
     
     def __str__(self):
         return py3compat.unicode_to_str(self.value)
     
     def __unicode__(self):
         return self.value
 
     def __repr__(self):
         return 'IPython.macro.Macro(%s)' % repr(self.value)
     
     def __getstate__(self):
         """ needed for safe pickling via %store """
         return {'value': self.value}
     
     def __add__(self, other):
         if isinstance(other, Macro):
             return Macro(self.value + other.value)
         elif isinstance(other, basestring):
             return Macro(self.value + other)
         raise TypeError

IPython/core/magic.py

@@ -1,3798 +1,3799 b''
 # encoding: utf-8
 """Magic functions for InteractiveShell.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001 Janko Hauser <[e-mail]> and
 #  Copyright (C) 2001-2007 Fernando Perez <[e-mail]>
 #  Copyright (C) 2008-2011  The IPython Development Team
 
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import __builtin__ as builtin_mod
 import __future__
 import bdb
 import inspect
 import imp
 import io
 import os
 import sys
 import shutil
 import re
 import time
 import gc
 from StringIO import StringIO
 from getopt import getopt,GetoptError
 from pprint import pformat
 from xmlrpclib import ServerProxy
 
 # cProfile was added in Python2.5
 try:
     import cProfile as profile
     import pstats
 except ImportError:
     # profile isn't bundled by default in Debian for license reasons
     try:
         import profile,pstats
     except ImportError:
         profile = pstats = None
 
 import IPython
 from IPython.core import debugger, oinspect
 from IPython.core.error import TryNext
 from IPython.core.error import UsageError
 from IPython.core.error import StdinNotImplementedError
 from IPython.core.fakemodule import FakeModule
 from IPython.core.profiledir import ProfileDir
 from IPython.core.macro import Macro
 from IPython.core import magic_arguments, page
 from IPython.core.prefilter import ESC_MAGIC
 from IPython.core.pylabtools import mpl_runner
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import py3compat
 from IPython.utils import openpy
+from IPython.utils.encoding import getdefaultencoding
 from IPython.utils.io import file_read, nlprint
 from IPython.utils.module_paths import find_mod
 from IPython.utils.path import get_py_filename, unquote_filename
 from IPython.utils.process import arg_split, abbrev_cwd
 from IPython.utils.terminal import set_term_title
 from IPython.utils.text import LSString, SList, format_screen
 from IPython.utils.timing import clock, clock2
 from IPython.utils.warn import warn, error
 from IPython.utils.ipstruct import Struct
 from IPython.config.application import Application
 
 #-----------------------------------------------------------------------------
 # Utility functions
 #-----------------------------------------------------------------------------
 
 def on_off(tag):
     """Return an ON/OFF string for a 1/0 input. Simple utility function."""
     return ['OFF','ON'][tag]
 
 class Bunch: pass
 
 def compress_dhist(dh):
     head, tail = dh[:-10], dh[-10:]
 
     newhead = []
     done = set()
     for h in head:
         if h in done:
             continue
         newhead.append(h)
         done.add(h)
 
     return newhead + tail
 
 def needs_local_scope(func):
     """Decorator to mark magic functions which need to local scope to run."""
     func.needs_local_scope = True
     return func
 
     
 # Used for exception handling in magic_edit
 class MacroToEdit(ValueError): pass
 
 #***************************************************************************
 # Main class implementing Magic functionality
 
 # XXX - for some odd reason, if Magic is made a new-style class, we get errors
 # on construction of the main InteractiveShell object.  Something odd is going
 # on with super() calls, Configurable and the MRO... For now leave it as-is, but
 # eventually this needs to be clarified.
 # BG: This is because InteractiveShell inherits from this, but is itself a
 # Configurable. This messes up the MRO in some way. The fix is that we need to
 # make Magic a configurable that InteractiveShell does not subclass.
 
 class Magic:
     """Magic functions for InteractiveShell.
 
     Shell functions which can be reached as %function_name. All magic
     functions should accept a string, which they can parse for their own
     needs. This can make some functions easier to type, eg `%cd ../`
     vs. `%cd("../")`
 
     ALL definitions MUST begin with the prefix magic_. The user won't need it
     at the command line, but it is is needed in the definition. """
 
     # class globals
     auto_status = ['Automagic is OFF, % prefix IS needed for magic functions.',
                    'Automagic is ON, % prefix NOT needed for magic functions.']
 
 
     configurables = None
     #......................................................................
     # some utility functions
 
     def __init__(self,shell):
 
         self.options_table = {}
         if profile is None:
             self.magic_prun = self.profile_missing_notice
         self.shell = shell
         if self.configurables is None:
             self.configurables = []
 
         # namespace for holding state we may need
         self._magic_state = Bunch()
 
     def profile_missing_notice(self, *args, **kwargs):
         error("""\
 The profile module could not be found. It has been removed from the standard
 python packages because of its non-free license. To use profiling, install the
 python-profiler package from non-free.""")
 
     def default_option(self,fn,optstr):
         """Make an entry in the options_table for fn, with value optstr"""
 
         if fn not in self.lsmagic():
             error("%s is not a magic function" % fn)
         self.options_table[fn] = optstr
 
     def lsmagic(self):
         """Return a list of currently available magic functions.
 
         Gives a list of the bare names after mangling (['ls','cd', ...], not
         ['magic_ls','magic_cd',...]"""
 
         # FIXME. This needs a cleanup, in the way the magics list is built.
 
         # magics in class definition
         class_magic = lambda fn: fn.startswith('magic_') and \
                       callable(Magic.__dict__[fn])
         # in instance namespace (run-time user additions)
         inst_magic =  lambda fn: fn.startswith('magic_') and \
                      callable(self.__dict__[fn])
         # and bound magics by user (so they can access self):
         inst_bound_magic =  lambda fn: fn.startswith('magic_') and \
                            callable(self.__class__.__dict__[fn])
         magics = filter(class_magic,Magic.__dict__.keys()) + \
                  filter(inst_magic,self.__dict__.keys()) + \
                  filter(inst_bound_magic,self.__class__.__dict__.keys())
         out = []
         for fn in set(magics):
             out.append(fn.replace('magic_','',1))
         out.sort()
         return out
 
     def extract_input_lines(self, range_str, raw=False):
         """Return as a string a set of input history slices.
 
         Parameters
         ----------
         range_str : string
             The set of slices is given as a string, like "~5/6-~4/2 4:8 9",
             since this function is for use by magic functions which get their
             arguments as strings. The number before the / is the session
             number: ~n goes n back from the current session.
 
         Optional Parameters:
           - raw(False): by default, the processed input is used.  If this is
           true, the raw input history is used instead.
 
         Note that slices can be called with two notations:
 
         N:M -> standard python form, means including items N...(M-1).
 
         N-M -> include items N..M (closed endpoint)."""
         lines = self.shell.history_manager.\
                                     get_range_by_str(range_str, raw=raw)
         return "\n".join(x for _, _, x in lines)
 
     def arg_err(self,func):
         """Print docstring if incorrect arguments were passed"""
         print 'Error in arguments:'
         print oinspect.getdoc(func)
 
     def format_latex(self,strng):
         """Format a string for latex inclusion."""
 
         # Characters that need to be escaped for latex:
         escape_re = re.compile(r'(%|_|\$|#|&)',re.MULTILINE)
         # Magic command names as headers:
         cmd_name_re = re.compile(r'^(%s.*?):' % ESC_MAGIC,
                                  re.MULTILINE)
         # Magic commands
         cmd_re = re.compile(r'(?P<cmd>%s.+?\b)(?!\}\}:)' % ESC_MAGIC,
                             re.MULTILINE)
         # Paragraph continue
         par_re = re.compile(r'\\$',re.MULTILINE)
 
         # The "\n" symbol
         newline_re = re.compile(r'\\n')
 
         # Now build the string for output:
         #strng = cmd_name_re.sub(r'\n\\texttt{\\textsl{\\large \1}}:',strng)
         strng = cmd_name_re.sub(r'\n\\bigskip\n\\texttt{\\textbf{ \1}}:',
                                 strng)
         strng = cmd_re.sub(r'\\texttt{\g<cmd>}',strng)
         strng = par_re.sub(r'\\\\',strng)
         strng = escape_re.sub(r'\\\1',strng)
         strng = newline_re.sub(r'\\textbackslash{}n',strng)
         return strng
 
     def parse_options(self,arg_str,opt_str,*long_opts,**kw):
         """Parse options passed to an argument string.
 
         The interface is similar to that of getopt(), but it returns back a
         Struct with the options as keys and the stripped argument string still
         as a string.
 
         arg_str is quoted as a true sys.argv vector by using shlex.split.
         This allows us to easily expand variables, glob files, quote
         arguments, etc.
 
         Options:
           -mode: default 'string'. If given as 'list', the argument string is
           returned as a list (split on whitespace) instead of a string.
 
           -list_all: put all option values in lists. Normally only options
           appearing more than once are put in a list.
 
           -posix (True): whether to split the input line in POSIX mode or not,
           as per the conventions outlined in the shlex module from the
           standard library."""
 
         # inject default options at the beginning of the input line
         caller = sys._getframe(1).f_code.co_name.replace('magic_','')
         arg_str = '%s %s' % (self.options_table.get(caller,''),arg_str)
 
         mode = kw.get('mode','string')
         if mode not in ['string','list']:
             raise ValueError,'incorrect mode given: %s' % mode
         # Get options
         list_all = kw.get('list_all',0)
         posix = kw.get('posix', os.name == 'posix')
         strict = kw.get('strict', True)
 
         # Check if we have more than one argument to warrant extra processing:
         odict = {}  # Dictionary with options
         args = arg_str.split()
         if len(args) >= 1:
             # If the list of inputs only has 0 or 1 thing in it, there's no
             # need to look for options
             argv = arg_split(arg_str, posix, strict)
             # Do regular option processing
             try:
                 opts,args = getopt(argv,opt_str,*long_opts)
             except GetoptError,e:
                 raise UsageError('%s ( allowed: "%s" %s)' % (e.msg,opt_str,
                                         " ".join(long_opts)))
             for o,a in opts:
                 if o.startswith('--'):
                     o = o[2:]
                 else:
                     o = o[1:]
                 try:
                     odict[o].append(a)
                 except AttributeError:
                     odict[o] = [odict[o],a]
                 except KeyError:
                     if list_all:
                         odict[o] = [a]
                     else:
                         odict[o] = a
 
         # Prepare opts,args for return
         opts = Struct(odict)
         if mode == 'string':
             args = ' '.join(args)
 
         return opts,args
 
     #......................................................................
     # And now the actual magic functions
 
     # Functions for IPython shell work (vars,funcs, config, etc)
     def magic_lsmagic(self, parameter_s = ''):
         """List currently available magic functions."""
         mesc = ESC_MAGIC
         print 'Available magic functions:\n'+mesc+\
               ('  '+mesc).join(self.lsmagic())
         print '\n' + Magic.auto_status[self.shell.automagic]
         return None
 
     def magic_magic(self, parameter_s = ''):
         """Print information about the magic function system.
 
         Supported formats: -latex, -brief, -rest
         """
 
         mode = ''
         try:
             if parameter_s.split()[0] == '-latex':
                 mode = 'latex'
             if parameter_s.split()[0] == '-brief':
                 mode = 'brief'
             if parameter_s.split()[0] == '-rest':
                 mode = 'rest'
                 rest_docs = []
         except:
             pass
 
         magic_docs = []
         for fname in self.lsmagic():
             mname = 'magic_' + fname
             for space in (Magic,self,self.__class__):
                 try:
                     fn = space.__dict__[mname]
                 except KeyError:
                     pass
                 else:
                     break
             if mode == 'brief':
                 # only first line
                 if fn.__doc__:
                     fndoc = fn.__doc__.split('\n',1)[0]
                 else:
                     fndoc = 'No documentation'
             else:
                 if fn.__doc__:
                     fndoc = fn.__doc__.rstrip()
                 else:
                     fndoc = 'No documentation'
 
 
             if mode == 'rest':
                 rest_docs.append('**%s%s**::\n\n\t%s\n\n' %(ESC_MAGIC,
                                                     fname,fndoc))
 
             else:
                 magic_docs.append('%s%s:\n\t%s\n' %(ESC_MAGIC,
                                                     fname,fndoc))
 
         magic_docs = ''.join(magic_docs)
 
         if mode == 'rest':
             return "".join(rest_docs)
 
         if mode == 'latex':
             print self.format_latex(magic_docs)
             return
         else:
             magic_docs = format_screen(magic_docs)
         if mode == 'brief':
             return magic_docs
 
         outmsg = """
 IPython's 'magic' functions
 ===========================
 
 The magic function system provides a series of functions which allow you to
 control the behavior of IPython itself, plus a lot of system-type
 features. All these functions are prefixed with a % character, but parameters
 are given without parentheses or quotes.
 
 NOTE: If you have 'automagic' enabled (via the command line option or with the
 %automagic function), you don't need to type in the % explicitly.  By default,
 IPython ships with automagic on, so you should only rarely need the % escape.
 
 Example: typing '%cd mydir' (without the quotes) changes you working directory
 to 'mydir', if it exists.
 
 For a list of the available magic functions, use %lsmagic. For a description
 of any of them, type %magic_name?, e.g. '%cd?'.
 
 Currently the magic system has the following functions:\n"""
 
         mesc = ESC_MAGIC
         outmsg = ("%s\n%s\n\nSummary of magic functions (from %slsmagic):"
                   "\n\n%s%s\n\n%s" % (outmsg,
                                      magic_docs,mesc,mesc,
                                      ('  '+mesc).join(self.lsmagic()),
                                      Magic.auto_status[self.shell.automagic] ) )
         page.page(outmsg)
 
     def magic_automagic(self, parameter_s = ''):
         """Make magic functions callable without having to type the initial %.
 
         Without argumentsl toggles on/off (when off, you must call it as
         %automagic, of course).  With arguments it sets the value, and you can
         use any of (case insensitive):
 
          - on,1,True: to activate
 
          - off,0,False: to deactivate.
 
         Note that magic functions have lowest priority, so if there's a
         variable whose name collides with that of a magic fn, automagic won't
         work for that function (you get the variable instead). However, if you
         delete the variable (del var), the previously shadowed magic function
         becomes visible to automagic again."""
 
         arg = parameter_s.lower()
         if parameter_s in ('on','1','true'):
             self.shell.automagic = True
         elif parameter_s in ('off','0','false'):
             self.shell.automagic = False
         else:
             self.shell.automagic = not self.shell.automagic
         print '\n' + Magic.auto_status[self.shell.automagic]
 
     @skip_doctest
     def magic_autocall(self, parameter_s = ''):
         """Make functions callable without having to type parentheses.
 
         Usage:
 
            %autocall [mode]
 
         The mode can be one of: 0->Off, 1->Smart, 2->Full.  If not given, the
         value is toggled on and off (remembering the previous state).
 
         In more detail, these values mean:
 
         0 -> fully disabled
 
         1 -> active, but do not apply if there are no arguments on the line.
 
         In this mode, you get::
 
           In [1]: callable
           Out[1]: <built-in function callable>
 
           In [2]: callable 'hello'
           ------> callable('hello')
           Out[2]: False
 
         2 -> Active always.  Even if no arguments are present, the callable
         object is called::
 
           In [2]: float
           ------> float()
           Out[2]: 0.0
 
         Note that even with autocall off, you can still use '/' at the start of
         a line to treat the first argument on the command line as a function
         and add parentheses to it::
 
           In [8]: /str 43
           ------> str(43)
           Out[8]: '43'
 
         # all-random (note for auto-testing)
         """
 
         if parameter_s:
             arg = int(parameter_s)
         else:
             arg = 'toggle'
 
         if not arg in (0,1,2,'toggle'):
             error('Valid modes: (0->Off, 1->Smart, 2->Full')
             return
 
         if arg in (0,1,2):
             self.shell.autocall = arg
         else: # toggle
             if self.shell.autocall:
                 self._magic_state.autocall_save = self.shell.autocall
                 self.shell.autocall = 0
             else:
                 try:
                     self.shell.autocall = self._magic_state.autocall_save
                 except AttributeError:
                     self.shell.autocall = self._magic_state.autocall_save = 1
 
         print "Automatic calling is:",['OFF','Smart','Full'][self.shell.autocall]
 
 
     def magic_page(self, parameter_s=''):
         """Pretty print the object and display it through a pager.
 
         %page [options] OBJECT
 
         If no object is given, use _ (last output).
 
         Options:
 
           -r: page str(object), don't pretty-print it."""
 
         # After a function contributed by Olivier Aubert, slightly modified.
 
         # Process options/args
         opts,args = self.parse_options(parameter_s,'r')
         raw = 'r' in opts
 
         oname = args and args or '_'
         info = self._ofind(oname)
         if info['found']:
             txt = (raw and str or pformat)( info['obj'] )
             page.page(txt)
         else:
             print 'Object `%s` not found' % oname
 
     def magic_profile(self, parameter_s=''):
         """Print your currently active IPython profile."""
         from IPython.core.application import BaseIPythonApplication
         if BaseIPythonApplication.initialized():
             print BaseIPythonApplication.instance().profile
         else:
             error("profile is an application-level value, but you don't appear to be in an IPython application")
 
     def magic_pinfo(self, parameter_s='', namespaces=None):
         """Provide detailed information about an object.
 
         '%pinfo object' is just a synonym for object? or ?object."""
 
         #print 'pinfo par: <%s>' % parameter_s  # dbg
 
 
         # detail_level: 0 -> obj? , 1 -> obj??
         detail_level = 0
         # We need to detect if we got called as 'pinfo pinfo foo', which can
         # happen if the user types 'pinfo foo?' at the cmd line.
         pinfo,qmark1,oname,qmark2 = \
                re.match('(pinfo )?(\?*)(.*?)(\??$)',parameter_s).groups()
         if pinfo or qmark1 or qmark2:
             detail_level = 1
         if "*" in oname:
             self.magic_psearch(oname)
         else:
             self.shell._inspect('pinfo', oname, detail_level=detail_level,
                                 namespaces=namespaces)
 
     def magic_pinfo2(self, parameter_s='', namespaces=None):
         """Provide extra detailed information about an object.
 
         '%pinfo2 object' is just a synonym for object?? or ??object."""
         self.shell._inspect('pinfo', parameter_s, detail_level=1,
                             namespaces=namespaces)
 
     @skip_doctest
     def magic_pdef(self, parameter_s='', namespaces=None):
         """Print the definition header for any callable object.
 
         If the object is a class, print the constructor information.
 
         Examples
         --------
         ::
 
           In [3]: %pdef urllib.urlopen
           urllib.urlopen(url, data=None, proxies=None)
         """
         self._inspect('pdef',parameter_s, namespaces)
 
     def magic_pdoc(self, parameter_s='', namespaces=None):
         """Print the docstring for an object.
 
         If the given object is a class, it will print both the class and the
         constructor docstrings."""
         self._inspect('pdoc',parameter_s, namespaces)
 
     def magic_psource(self, parameter_s='', namespaces=None):
         """Print (or run through pager) the source code for an object."""
         self._inspect('psource',parameter_s, namespaces)
 
     def magic_pfile(self, parameter_s=''):
         """Print (or run through pager) the file where an object is defined.
 
         The file opens at the line where the object definition begins. IPython
         will honor the environment variable PAGER if set, and otherwise will
         do its best to print the file in a convenient form.
 
         If the given argument is not an object currently defined, IPython will
         try to interpret it as a filename (automatically adding a .py extension
         if needed). You can thus use %pfile as a syntax highlighting code
         viewer."""
 
         # first interpret argument as an object name
         out = self._inspect('pfile',parameter_s)
         # if not, try the input as a filename
         if out == 'not found':
             try:
                 filename = get_py_filename(parameter_s)
             except IOError,msg:
                 print msg
                 return
             page.page(self.shell.inspector.format(open(filename).read()))
 
     def magic_psearch(self, parameter_s=''):
         """Search for object in namespaces by wildcard.
 
         %psearch [options] PATTERN [OBJECT TYPE]
 
         Note: ? can be used as a synonym for %psearch, at the beginning or at
         the end: both a*? and ?a* are equivalent to '%psearch a*'.  Still, the
         rest of the command line must be unchanged (options come first), so
         for example the following forms are equivalent
 
         %psearch -i a* function
         -i a* function?
         ?-i a* function
 
         Arguments:
 
           PATTERN
 
           where PATTERN is a string containing * as a wildcard similar to its
           use in a shell.  The pattern is matched in all namespaces on the
           search path. By default objects starting with a single _ are not
           matched, many IPython generated objects have a single
           underscore. The default is case insensitive matching. Matching is
           also done on the attributes of objects and not only on the objects
           in a module.
 
           [OBJECT TYPE]
 
           Is the name of a python type from the types module. The name is
           given in lowercase without the ending type, ex. StringType is
           written string. By adding a type here only objects matching the
           given type are matched. Using all here makes the pattern match all
           types (this is the default).
 
         Options:
 
           -a: makes the pattern match even objects whose names start with a
           single underscore.  These names are normally omitted from the
           search.
 
           -i/-c: make the pattern case insensitive/sensitive.  If neither of
           these options are given, the default is read from your configuration
           file, with the option ``InteractiveShell.wildcards_case_sensitive``.
           If this option is not specified in your configuration file, IPython's
           internal default is to do a case sensitive search.
 
           -e/-s NAMESPACE: exclude/search a given namespace.  The pattern you
           specify can be searched in any of the following namespaces:
           'builtin', 'user', 'user_global','internal', 'alias', where
           'builtin' and 'user' are the search defaults.  Note that you should
           not use quotes when specifying namespaces.
 
           'Builtin' contains the python module builtin, 'user' contains all
           user data, 'alias' only contain the shell aliases and no python
           objects, 'internal' contains objects used by IPython.  The
           'user_global' namespace is only used by embedded IPython instances,
           and it contains module-level globals.  You can add namespaces to the
           search with -s or exclude them with -e (these options can be given
           more than once).
 
         Examples
         --------
         ::
 
           %psearch a*            -> objects beginning with an a
           %psearch -e builtin a* -> objects NOT in the builtin space starting in a
           %psearch a* function   -> all functions beginning with an a
           %psearch re.e*         -> objects beginning with an e in module re
           %psearch r*.e*         -> objects that start with e in modules starting in r
           %psearch r*.* string   -> all strings in modules beginning with r
 
         Case sensitive search::
 
           %psearch -c a*         list all object beginning with lower case a
 
         Show objects beginning with a single _::
 
           %psearch -a _*         list objects beginning with a single underscore"""
         try:
             parameter_s.encode('ascii')
         except UnicodeEncodeError:
             print 'Python identifiers can only contain ascii characters.'
             return
 
         # default namespaces to be searched
         def_search = ['user_local', 'user_global', 'builtin']
 
         # Process options/args
         opts,args = self.parse_options(parameter_s,'cias:e:',list_all=True)
         opt = opts.get
         shell = self.shell
         psearch = shell.inspector.psearch
 
         # select case options
         if opts.has_key('i'):
             ignore_case = True
         elif opts.has_key('c'):
             ignore_case = False
         else:
             ignore_case = not shell.wildcards_case_sensitive
 
         # Build list of namespaces to search from user options
         def_search.extend(opt('s',[]))
         ns_exclude = ns_exclude=opt('e',[])
         ns_search = [nm for nm in def_search if nm not in ns_exclude]
 
         # Call the actual search
         try:
             psearch(args,shell.ns_table,ns_search,
                     show_all=opt('a'),ignore_case=ignore_case)
         except:
             shell.showtraceback()
 
     @skip_doctest
     def magic_who_ls(self, parameter_s=''):
         """Return a sorted list of all interactive variables.
 
         If arguments are given, only variables of types matching these
         arguments are returned.
 
         Examples
         --------
 
         Define two variables and list them with who_ls::
 
           In [1]: alpha = 123
 
           In [2]: beta = 'test'
 
           In [3]: %who_ls
           Out[3]: ['alpha', 'beta']
 
           In [4]: %who_ls int
           Out[4]: ['alpha']
 
           In [5]: %who_ls str
           Out[5]: ['beta']
         """
 
         user_ns = self.shell.user_ns
         user_ns_hidden = self.shell.user_ns_hidden
         out = [ i for i in user_ns
                 if not i.startswith('_') \
                 and not i in user_ns_hidden ]
 
         typelist = parameter_s.split()
         if typelist:
             typeset = set(typelist)
             out = [i for i in out if type(user_ns[i]).__name__ in typeset]
 
         out.sort()
         return out
 
     @skip_doctest
     def magic_who(self, parameter_s=''):
         """Print all interactive variables, with some minimal formatting.
 
         If any arguments are given, only variables whose type matches one of
         these are printed.  For example::
 
           %who function str
 
         will only list functions and strings, excluding all other types of
         variables.  To find the proper type names, simply use type(var) at a
         command line to see how python prints type names.  For example:
 
         ::
 
           In [1]: type('hello')\\
           Out[1]: <type 'str'>
 
         indicates that the type name for strings is 'str'.
 
         ``%who`` always excludes executed names loaded through your configuration
         file and things which are internal to IPython.
 
         This is deliberate, as typically you may load many modules and the
         purpose of %who is to show you only what you've manually defined.
 
         Examples
         --------
 
         Define two variables and list them with who::
 
           In [1]: alpha = 123
 
           In [2]: beta = 'test'
 
           In [3]: %who
           alpha   beta
 
           In [4]: %who int
           alpha
 
           In [5]: %who str
           beta
         """
 
         varlist = self.magic_who_ls(parameter_s)
         if not varlist:
             if parameter_s:
                 print 'No variables match your requested type.'
             else:
                 print 'Interactive namespace is empty.'
             return
 
         # if we have variables, move on...
         count = 0
         for i in varlist:
             print i+'\t',
             count += 1
             if count > 8:
                 count = 0
                 print
         print
 
     @skip_doctest
     def magic_whos(self, parameter_s=''):
         """Like %who, but gives some extra information about each variable.
 
         The same type filtering of %who can be applied here.
 
         For all variables, the type is printed. Additionally it prints:
 
           - For {},[],(): their length.
 
           - For numpy arrays, a summary with shape, number of
           elements, typecode and size in memory.
 
           - Everything else: a string representation, snipping their middle if
           too long.
 
         Examples
         --------
 
         Define two variables and list them with whos::
 
           In [1]: alpha = 123
 
           In [2]: beta = 'test'
 
           In [3]: %whos
           Variable   Type        Data/Info
           --------------------------------
           alpha      int         123
           beta       str         test
         """
 
         varnames = self.magic_who_ls(parameter_s)
         if not varnames:
             if parameter_s:
                 print 'No variables match your requested type.'
             else:
                 print 'Interactive namespace is empty.'
             return
 
         # if we have variables, move on...
 
         # for these types, show len() instead of data:
         seq_types = ['dict', 'list', 'tuple']
 
         # for numpy arrays, display summary info
         ndarray_type = None
         if 'numpy' in sys.modules:
             try:
                 from numpy import ndarray
             except ImportError:
                 pass
             else:
                 ndarray_type = ndarray.__name__
 
         # Find all variable names and types so we can figure out column sizes
         def get_vars(i):
             return self.shell.user_ns[i]
 
         # some types are well known and can be shorter
         abbrevs = {'IPython.core.macro.Macro' : 'Macro'}
         def type_name(v):
             tn = type(v).__name__
             return abbrevs.get(tn,tn)
 
         varlist = map(get_vars,varnames)
 
         typelist = []
         for vv in varlist:
             tt = type_name(vv)
 
             if tt=='instance':
                 typelist.append( abbrevs.get(str(vv.__class__),
                                              str(vv.__class__)))
             else:
                 typelist.append(tt)
 
         # column labels and # of spaces as separator
         varlabel = 'Variable'
         typelabel = 'Type'
         datalabel = 'Data/Info'
         colsep = 3
         # variable format strings
         vformat    = "{0:<{varwidth}}{1:<{typewidth}}"
         aformat    = "%s: %s elems, type `%s`, %s bytes"
         # find the size of the columns to format the output nicely
         varwidth = max(max(map(len,varnames)), len(varlabel)) + colsep
         typewidth = max(max(map(len,typelist)), len(typelabel)) + colsep
         # table header
         print varlabel.ljust(varwidth) + typelabel.ljust(typewidth) + \
               ' '+datalabel+'\n' + '-'*(varwidth+typewidth+len(datalabel)+1)
         # and the table itself
         kb = 1024
         Mb = 1048576  # kb**2
         for vname,var,vtype in zip(varnames,varlist,typelist):
             print vformat.format(vname, vtype, varwidth=varwidth, typewidth=typewidth),
             if vtype in seq_types:
                 print "n="+str(len(var))
             elif vtype == ndarray_type:
                 vshape = str(var.shape).replace(',','').replace(' ','x')[1:-1]
                 if vtype==ndarray_type:
                     # numpy
                     vsize  = var.size
                     vbytes = vsize*var.itemsize
                     vdtype = var.dtype
 
                 if vbytes < 100000:
                     print aformat % (vshape,vsize,vdtype,vbytes)
                 else:
                     print aformat % (vshape,vsize,vdtype,vbytes),
                     if vbytes < Mb:
                         print '(%s kb)' % (vbytes/kb,)
                     else:
                         print '(%s Mb)' % (vbytes/Mb,)
             else:
                 try:
                     vstr = str(var)
                 except UnicodeEncodeError:
-                    vstr = unicode(var).encode(py3compat.getdefaultencoding(),
+                    vstr = unicode(var).encode(getdefaultencoding(),
                                                'backslashreplace')
                 vstr = vstr.replace('\n','\\n')
                 if len(vstr) < 50:
                     print vstr
                 else:
                     print vstr[:25] + "<...>" + vstr[-25:]
 
     def magic_reset(self, parameter_s=''):
         """Resets the namespace by removing all names defined by the user, if
         called without arguments, or by removing some types of objects, such
         as everything currently in IPython's In[] and Out[] containers (see
         the parameters for details).
 
         Parameters
         ----------
         -f : force reset without asking for confirmation.
 
         -s : 'Soft' reset: Only clears your namespace, leaving history intact.
             References to objects may be kept. By default (without this option),
             we do a 'hard' reset, giving you a new session and removing all
             references to objects from the current session.
 
         in : reset input history
        
         out : reset output history
        
         dhist : reset directory history
         
         array : reset only variables that are NumPy arrays
 
         See Also
         --------
         magic_reset_selective : invoked as ``%reset_selective``
 
         Examples
         --------
         ::
 
           In [6]: a = 1
 
           In [7]: a
           Out[7]: 1
 
           In [8]: 'a' in _ip.user_ns
           Out[8]: True
 
           In [9]: %reset -f
 
           In [1]: 'a' in _ip.user_ns
           Out[1]: False
 
           In [2]: %reset -f in
           Flushing input history
 
           In [3]: %reset -f dhist in
           Flushing directory history
           Flushing input history
 
         Notes
         -----
         Calling this magic from clients that do not implement standard input,
         such as the ipython notebook interface, will reset the namespace
         without confirmation.
         """
         opts, args = self.parse_options(parameter_s,'sf', mode='list')
         if 'f' in opts:
             ans = True
         else:
             try:
                 ans = self.shell.ask_yes_no(
                 "Once deleted, variables cannot be recovered. Proceed (y/[n])? ", default='n')
             except StdinNotImplementedError:
                 ans = True
         if not ans:
             print 'Nothing done.'
             return
 
         if 's' in opts:                     # Soft reset
             user_ns = self.shell.user_ns
             for i in self.magic_who_ls():
                 del(user_ns[i])
         elif len(args) == 0:                # Hard reset
             self.shell.reset(new_session = False)
         
         # reset in/out/dhist/array: previously extensinions/clearcmd.py
         ip = self.shell
         user_ns = self.user_ns  # local lookup, heavily used
 
         for target in args:
             target = target.lower() # make matches case insensitive
             if target == 'out':
                 print "Flushing output cache (%d entries)" % len(user_ns['_oh'])
                 self.displayhook.flush()
 
             elif target == 'in':
                 print "Flushing input history"
                 pc = self.displayhook.prompt_count + 1
                 for n in range(1, pc):
                     key = '_i'+repr(n)
                     user_ns.pop(key,None)
                 user_ns.update(dict(_i=u'',_ii=u'',_iii=u''))
                 hm = ip.history_manager
                 # don't delete these, as %save and %macro depending on the length
                 # of these lists to be preserved
                 hm.input_hist_parsed[:] = [''] * pc
                 hm.input_hist_raw[:] = [''] * pc
                 # hm has internal machinery for _i,_ii,_iii, clear it out
                 hm._i = hm._ii = hm._iii = hm._i00 =  u''
 
             elif target == 'array':
                 # Support cleaning up numpy arrays
                 try:
                     from numpy import ndarray
                     # This must be done with items and not iteritems because we're
                     # going to modify the dict in-place.
                     for x,val in user_ns.items():
                         if isinstance(val,ndarray):
                             del user_ns[x]
                 except ImportError:
                     print "reset array only works if Numpy is available."
 
             elif target == 'dhist':
                 print "Flushing directory history"
                 del user_ns['_dh'][:]
 
             else:
                 print "Don't know how to reset ",
                 print target + ", please run `%reset?` for details"
 
         gc.collect()
 
     def magic_reset_selective(self, parameter_s=''):
         """Resets the namespace by removing names defined by the user.
 
         Input/Output history are left around in case you need them.
 
         %reset_selective [-f] regex
 
         No action is taken if regex is not included
 
         Options
           -f : force reset without asking for confirmation.
 
         See Also
         --------
         magic_reset : invoked as ``%reset``
 
         Examples
         --------
 
         We first fully reset the namespace so your output looks identical to
         this example for pedagogical reasons; in practice you do not need a
         full reset::
 
           In [1]: %reset -f
 
         Now, with a clean namespace we can make a few variables and use
         ``%reset_selective`` to only delete names that match our regexp::
 
           In [2]: a=1; b=2; c=3; b1m=4; b2m=5; b3m=6; b4m=7; b2s=8
 
           In [3]: who_ls
           Out[3]: ['a', 'b', 'b1m', 'b2m', 'b2s', 'b3m', 'b4m', 'c']
 
           In [4]: %reset_selective -f b[2-3]m
 
           In [5]: who_ls
           Out[5]: ['a', 'b', 'b1m', 'b2s', 'b4m', 'c']
 
           In [6]: %reset_selective -f d
 
           In [7]: who_ls
           Out[7]: ['a', 'b', 'b1m', 'b2s', 'b4m', 'c']
 
           In [8]: %reset_selective -f c
 
           In [9]: who_ls
           Out[9]: ['a', 'b', 'b1m', 'b2s', 'b4m']
 
           In [10]: %reset_selective -f b
 
           In [11]: who_ls
           Out[11]: ['a']
 
         Notes
         -----
         Calling this magic from clients that do not implement standard input,
         such as the ipython notebook interface, will reset the namespace
         without confirmation.
         """
 
         opts, regex = self.parse_options(parameter_s,'f')
 
         if opts.has_key('f'):
             ans = True
         else:
             try:
                 ans = self.shell.ask_yes_no(
                 "Once deleted, variables cannot be recovered. Proceed (y/[n])? ",
                 default='n')
             except StdinNotImplementedError:
                 ans = True
         if not ans:
             print 'Nothing done.'
             return
         user_ns = self.shell.user_ns
         if not regex:
             print 'No regex pattern specified. Nothing done.'
             return
         else:
             try:
                 m = re.compile(regex)
             except TypeError:
                 raise TypeError('regex must be a string or compiled pattern')
             for i in self.magic_who_ls():
                 if m.search(i):
                     del(user_ns[i])
 
     def magic_xdel(self, parameter_s=''):
         """Delete a variable, trying to clear it from anywhere that
         IPython's machinery has references to it. By default, this uses
         the identity of the named object in the user namespace to remove
         references held under other names. The object is also removed
         from the output history.
 
         Options
           -n : Delete the specified name from all namespaces, without
           checking their identity.
         """
         opts, varname = self.parse_options(parameter_s,'n')
         try:
             self.shell.del_var(varname, ('n' in opts))
         except (NameError, ValueError) as e:
             print type(e).__name__ +": "+ str(e)
 
     def magic_logstart(self,parameter_s=''):
         """Start logging anywhere in a session.
 
         %logstart [-o|-r|-t] [log_name [log_mode]]
 
         If no name is given, it defaults to a file named 'ipython_log.py' in your
         current directory, in 'rotate' mode (see below).
 
         '%logstart name' saves to file 'name' in 'backup' mode.  It saves your
         history up to that point and then continues logging.
 
         %logstart takes a second optional parameter: logging mode. This can be one
         of (note that the modes are given unquoted):\\
           append: well, that says it.\\
           backup: rename (if exists) to name~ and start name.\\
           global: single logfile in your home dir, appended to.\\
           over  : overwrite existing log.\\
           rotate: create rotating logs name.1~, name.2~, etc.
 
         Options:
 
           -o: log also IPython's output.  In this mode, all commands which
           generate an Out[NN] prompt are recorded to the logfile, right after
           their corresponding input line.  The output lines are always
           prepended with a '#[Out]# ' marker, so that the log remains valid
           Python code.
 
           Since this marker is always the same, filtering only the output from
           a log is very easy, using for example a simple awk call::
 
             awk -F'#\\[Out\\]# ' '{if($2) {print $2}}' ipython_log.py
 
           -r: log 'raw' input.  Normally, IPython's logs contain the processed
           input, so that user lines are logged in their final form, converted
           into valid Python.  For example, %Exit is logged as
           _ip.magic("Exit").  If the -r flag is given, all input is logged
           exactly as typed, with no transformations applied.
 
           -t: put timestamps before each input line logged (these are put in
           comments)."""
 
         opts,par = self.parse_options(parameter_s,'ort')
         log_output = 'o' in opts
         log_raw_input = 'r' in opts
         timestamp = 't' in opts
 
         logger = self.shell.logger
 
         # if no args are given, the defaults set in the logger constructor by
         # ipython remain valid
         if par:
             try:
                 logfname,logmode = par.split()
             except:
                 logfname = par
                 logmode = 'backup'
         else:
             logfname = logger.logfname
             logmode = logger.logmode
         # put logfname into rc struct as if it had been called on the command
         # line, so it ends up saved in the log header Save it in case we need
         # to restore it...
         old_logfile = self.shell.logfile
         if logfname:
             logfname = os.path.expanduser(logfname)
         self.shell.logfile = logfname
 
         loghead = '# IPython log file\n\n'
         try:
             started  = logger.logstart(logfname,loghead,logmode,
                                        log_output,timestamp,log_raw_input)
         except:
             self.shell.logfile = old_logfile
             warn("Couldn't start log: %s" % sys.exc_info()[1])
         else:
             # log input history up to this point, optionally interleaving
             # output if requested
 
             if timestamp:
                 # disable timestamping for the previous history, since we've
                 # lost those already (no time machine here).
                 logger.timestamp = False
 
             if log_raw_input:
                 input_hist = self.shell.history_manager.input_hist_raw
             else:
                 input_hist = self.shell.history_manager.input_hist_parsed
 
             if log_output:
                 log_write = logger.log_write
                 output_hist = self.shell.history_manager.output_hist
                 for n in range(1,len(input_hist)-1):
                     log_write(input_hist[n].rstrip() + '\n')
                     if n in output_hist:
                         log_write(repr(output_hist[n]),'output')
             else:
                 logger.log_write('\n'.join(input_hist[1:]))
                 logger.log_write('\n')
             if timestamp:
                 # re-enable timestamping
                 logger.timestamp = True
 
             print ('Activating auto-logging. '
                    'Current session state plus future input saved.')
             logger.logstate()
 
     def magic_logstop(self,parameter_s=''):
         """Fully stop logging and close log file.
 
         In order to start logging again, a new %logstart call needs to be made,
         possibly (though not necessarily) with a new filename, mode and other
         options."""
         self.logger.logstop()
 
     def magic_logoff(self,parameter_s=''):
         """Temporarily stop logging.
 
         You must have previously started logging."""
         self.shell.logger.switch_log(0)
 
     def magic_logon(self,parameter_s=''):
         """Restart logging.
 
         This function is for restarting logging which you've temporarily
         stopped with %logoff. For starting logging for the first time, you
         must use the %logstart function, which allows you to specify an
         optional log filename."""
 
         self.shell.logger.switch_log(1)
 
     def magic_logstate(self,parameter_s=''):
         """Print the status of the logging system."""
 
         self.shell.logger.logstate()
 
     def magic_pdb(self, parameter_s=''):
         """Control the automatic calling of the pdb interactive debugger.
 
         Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
         argument it works as a toggle.
 
         When an exception is triggered, IPython can optionally call the
         interactive pdb debugger after the traceback printout. %pdb toggles
         this feature on and off.
 
         The initial state of this feature is set in your configuration
         file (the option is ``InteractiveShell.pdb``).
 
         If you want to just activate the debugger AFTER an exception has fired,
         without having to type '%pdb on' and rerunning your code, you can use
         the %debug magic."""
 
         par = parameter_s.strip().lower()
 
         if par:
             try:
                 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
             except KeyError:
                 print ('Incorrect argument. Use on/1, off/0, '
                        'or nothing for a toggle.')
                 return
         else:
             # toggle
             new_pdb = not self.shell.call_pdb
 
         # set on the shell
         self.shell.call_pdb = new_pdb
         print 'Automatic pdb calling has been turned',on_off(new_pdb)
 
     def magic_debug(self, parameter_s=''):
         """Activate the interactive debugger in post-mortem mode.
 
         If an exception has just occurred, this lets you inspect its stack
         frames interactively.  Note that this will always work only on the last
         traceback that occurred, so you must call this quickly after an
         exception that you wish to inspect has fired, because if another one
         occurs, it clobbers the previous one.
 
         If you want IPython to automatically do this on every exception, see
         the %pdb magic for more details.
         """
         self.shell.debugger(force=True)
 
     @skip_doctest
     def magic_prun(self, parameter_s ='',user_mode=1,
                    opts=None,arg_lst=None,prog_ns=None):
 
         """Run a statement through the python code profiler.
 
         Usage:
           %prun [options] statement
 
         The given statement (which doesn't require quote marks) is run via the
         python profiler in a manner similar to the profile.run() function.
         Namespaces are internally managed to work correctly; profile.run
         cannot be used in IPython because it makes certain assumptions about
         namespaces which do not hold under IPython.
 
         Options:
 
         -l <limit>: you can place restrictions on what or how much of the
         profile gets printed. The limit value can be:
 
           * A string: only information for function names containing this string
           is printed.
 
           * An integer: only these many lines are printed.
 
           * A float (between 0 and 1): this fraction of the report is printed
           (for example, use a limit of 0.4 to see the topmost 40% only).
 
         You can combine several limits with repeated use of the option. For
         example, '-l __init__ -l 5' will print only the topmost 5 lines of
         information about class constructors.
 
         -r: return the pstats.Stats object generated by the profiling. This
         object has all the information about the profile in it, and you can
         later use it for further analysis or in other functions.
 
        -s <key>: sort profile by given key. You can provide more than one key
         by using the option several times: '-s key1 -s key2 -s key3...'. The
         default sorting key is 'time'.
 
         The following is copied verbatim from the profile documentation
         referenced below:
 
         When more than one key is provided, additional keys are used as
         secondary criteria when the there is equality in all keys selected
         before them.
 
         Abbreviations can be used for any key names, as long as the
         abbreviation is unambiguous.  The following are the keys currently
         defined:
 
                 Valid Arg       Meaning
                   "calls"      call count
                   "cumulative" cumulative time
                   "file"       file name
                   "module"     file name
                   "pcalls"     primitive call count
                   "line"       line number
                   "name"       function name
                   "nfl"        name/file/line
                   "stdname"    standard name
                   "time"       internal time
 
         Note that all sorts on statistics are in descending order (placing
         most time consuming items first), where as name, file, and line number
         searches are in ascending order (i.e., alphabetical). The subtle
         distinction between "nfl" and "stdname" is that the standard name is a
         sort of the name as printed, which means that the embedded line
         numbers get compared in an odd way.  For example, lines 3, 20, and 40
         would (if the file names were the same) appear in the string order
         "20" "3" and "40".  In contrast, "nfl" does a numeric compare of the
         line numbers.  In fact, sort_stats("nfl") is the same as
         sort_stats("name", "file", "line").
 
         -T <filename>: save profile results as shown on screen to a text
         file. The profile is still shown on screen.
 
         -D <filename>: save (via dump_stats) profile statistics to given
         filename. This data is in a format understood by the pstats module, and
         is generated by a call to the dump_stats() method of profile
         objects. The profile is still shown on screen.
 
         -q: suppress output to the pager.  Best used with -T and/or -D above.
 
         If you want to run complete programs under the profiler's control, use
         '%run -p [prof_opts] filename.py [args to program]' where prof_opts
         contains profiler specific options as described here.
 
         You can read the complete documentation for the profile module with::
 
           In [1]: import profile; profile.help()
         """
 
         opts_def = Struct(D=[''],l=[],s=['time'],T=[''])
 
         if user_mode:  # regular user call
             opts,arg_str = self.parse_options(parameter_s,'D:l:rs:T:q',
                                               list_all=1, posix=False)
             namespace = self.shell.user_ns
         else:  # called to run a program by %run -p
             try:
                 filename = get_py_filename(arg_lst[0])
             except IOError as e:
                 try:
                     msg = str(e)
                 except UnicodeError:
                     msg = e.message
                 error(msg)
                 return
 
             arg_str = 'execfile(filename,prog_ns)'
             namespace = {
                 'execfile': self.shell.safe_execfile,
                 'prog_ns': prog_ns,
                 'filename': filename
                 }
 
         opts.merge(opts_def)
 
         prof = profile.Profile()
         try:
             prof = prof.runctx(arg_str,namespace,namespace)
             sys_exit = ''
         except SystemExit:
             sys_exit = """*** SystemExit exception caught in code being profiled."""
 
         stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
 
         lims = opts.l
         if lims:
             lims = []  # rebuild lims with ints/floats/strings
             for lim in opts.l:
                 try:
                     lims.append(int(lim))
                 except ValueError:
                     try:
                         lims.append(float(lim))
                     except ValueError:
                         lims.append(lim)
 
         # Trap output.
         stdout_trap = StringIO()
 
         if hasattr(stats,'stream'):
             # In newer versions of python, the stats object has a 'stream'
             # attribute to write into.
             stats.stream = stdout_trap
             stats.print_stats(*lims)
         else:
             # For older versions, we manually redirect stdout during printing
             sys_stdout = sys.stdout
             try:
                 sys.stdout = stdout_trap
                 stats.print_stats(*lims)
             finally:
                 sys.stdout = sys_stdout
 
         output = stdout_trap.getvalue()
         output = output.rstrip()
 
         if 'q' not in opts:
             page.page(output)
         print sys_exit,
 
         dump_file = opts.D[0]
         text_file = opts.T[0]
         if dump_file:
             dump_file = unquote_filename(dump_file)
             prof.dump_stats(dump_file)
             print '\n*** Profile stats marshalled to file',\
                   `dump_file`+'.',sys_exit
         if text_file:
             text_file = unquote_filename(text_file)
             pfile = open(text_file,'w')
             pfile.write(output)
             pfile.close()
             print '\n*** Profile printout saved to text file',\
                   `text_file`+'.',sys_exit
 
         if opts.has_key('r'):
             return stats
         else:
             return None
 
     @skip_doctest
     def magic_run(self, parameter_s ='', runner=None,
                   file_finder=get_py_filename):
         """Run the named file inside IPython as a program.
 
         Usage:\\
           %run [-n -i -t [-N<N>] -d [-b<N>] -p [profile options]] file [args]
 
         Parameters after the filename are passed as command-line arguments to
         the program (put in sys.argv). Then, control returns to IPython's
         prompt.
 
         This is similar to running at a system prompt:\\
           $ python file args\\
         but with the advantage of giving you IPython's tracebacks, and of
         loading all variables into your interactive namespace for further use
         (unless -p is used, see below).
 
         The file is executed in a namespace initially consisting only of
         __name__=='__main__' and sys.argv constructed as indicated. It thus
         sees its environment as if it were being run as a stand-alone program
         (except for sharing global objects such as previously imported
         modules). But after execution, the IPython interactive namespace gets
         updated with all variables defined in the program (except for __name__
         and sys.argv). This allows for very convenient loading of code for
         interactive work, while giving each program a 'clean sheet' to run in.
 
         Options:
 
         -n: __name__ is NOT set to '__main__', but to the running file's name
         without extension (as python does under import).  This allows running
         scripts and reloading the definitions in them without calling code
         protected by an ' if __name__ == "__main__" ' clause.
 
         -i: run the file in IPython's namespace instead of an empty one. This
         is useful if you are experimenting with code written in a text editor
         which depends on variables defined interactively.
 
         -e: ignore sys.exit() calls or SystemExit exceptions in the script
         being run.  This is particularly useful if IPython is being used to
         run unittests, which always exit with a sys.exit() call.  In such
         cases you are interested in the output of the test results, not in
         seeing a traceback of the unittest module.
 
         -t: print timing information at the end of the run.  IPython will give
         you an estimated CPU time consumption for your script, which under
         Unix uses the resource module to avoid the wraparound problems of
         time.clock().  Under Unix, an estimate of time spent on system tasks
         is also given (for Windows platforms this is reported as 0.0).
 
         If -t is given, an additional -N<N> option can be given, where <N>
         must be an integer indicating how many times you want the script to
         run.  The final timing report will include total and per run results.
 
         For example (testing the script uniq_stable.py)::
 
             In [1]: run -t uniq_stable
 
             IPython CPU timings (estimated):\\
               User  :    0.19597 s.\\
               System:        0.0 s.\\
 
             In [2]: run -t -N5 uniq_stable
 
             IPython CPU timings (estimated):\\
             Total runs performed: 5\\
               Times :      Total       Per run\\
               User  :   0.910862 s,  0.1821724 s.\\
               System:        0.0 s,        0.0 s.
 
         -d: run your program under the control of pdb, the Python debugger.
         This allows you to execute your program step by step, watch variables,
         etc.  Internally, what IPython does is similar to calling:
 
           pdb.run('execfile("YOURFILENAME")')
 
         with a breakpoint set on line 1 of your file.  You can change the line
         number for this automatic breakpoint to be <N> by using the -bN option
         (where N must be an integer).  For example::
 
           %run -d -b40 myscript
 
         will set the first breakpoint at line 40 in myscript.py.  Note that
         the first breakpoint must be set on a line which actually does
         something (not a comment or docstring) for it to stop execution.
 
         When the pdb debugger starts, you will see a (Pdb) prompt.  You must
         first enter 'c' (without quotes) to start execution up to the first
         breakpoint.
 
         Entering 'help' gives information about the use of the debugger.  You
         can easily see pdb's full documentation with "import pdb;pdb.help()"
         at a prompt.
 
         -p: run program under the control of the Python profiler module (which
         prints a detailed report of execution times, function calls, etc).
 
         You can pass other options after -p which affect the behavior of the
         profiler itself. See the docs for %prun for details.
 
         In this mode, the program's variables do NOT propagate back to the
         IPython interactive namespace (because they remain in the namespace
         where the profiler executes them).
 
         Internally this triggers a call to %prun, see its documentation for
         details on the options available specifically for profiling.
 
         There is one special usage for which the text above doesn't apply:
         if the filename ends with .ipy, the file is run as ipython script,
         just as if the commands were written on IPython prompt.
 
         -m: specify module name to load instead of script path. Similar to
         the -m option for the python interpreter. Use this option last if you
         want to combine with other %run options. Unlike the python interpreter
         only source modules are allowed no .pyc or .pyo files.
         For example::
 
             %run -m example
 
         will run the example module.
 
         """
 
         # get arguments and set sys.argv for program to be run.
         opts, arg_lst = self.parse_options(parameter_s, 'nidtN:b:pD:l:rs:T:em:',
                                            mode='list', list_all=1)
         if "m" in opts:
             modulename = opts["m"][0]
             modpath = find_mod(modulename)
             if modpath is None:
                 warn('%r is not a valid modulename on sys.path'%modulename)
                 return
             arg_lst = [modpath] + arg_lst
         try:
             filename = file_finder(arg_lst[0])
         except IndexError:
             warn('you must provide at least a filename.')
             print '\n%run:\n', oinspect.getdoc(self.magic_run)
             return
         except IOError as e:
             try:
                 msg = str(e)
             except UnicodeError:
                 msg = e.message
             error(msg)
             return
 
         if filename.lower().endswith('.ipy'):
             self.shell.safe_execfile_ipy(filename)
             return
 
         # Control the response to exit() calls made by the script being run
         exit_ignore = 'e' in opts
 
         # Make sure that the running script gets a proper sys.argv as if it
         # were run from a system shell.
         save_argv = sys.argv # save it for later restoring
 
         # simulate shell expansion on arguments, at least tilde expansion
         args = [ os.path.expanduser(a) for a in arg_lst[1:] ]
 
         sys.argv = [filename] + args  # put in the proper filename
         # protect sys.argv from potential unicode strings on Python 2:
         if not py3compat.PY3:
             sys.argv = [ py3compat.cast_bytes(a) for a in sys.argv ]
 
         if 'i' in opts:
             # Run in user's interactive namespace
             prog_ns = self.shell.user_ns
             __name__save = self.shell.user_ns['__name__']
             prog_ns['__name__'] = '__main__'
             main_mod = self.shell.new_main_mod(prog_ns)
         else:
             # Run in a fresh, empty namespace
             if 'n' in opts:
                 name = os.path.splitext(os.path.basename(filename))[0]
             else:
                 name = '__main__'
 
             main_mod = self.shell.new_main_mod()
             prog_ns = main_mod.__dict__
             prog_ns['__name__'] = name
 
         # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
         # set the __file__ global in the script's namespace
         prog_ns['__file__'] = filename
 
         # pickle fix.  See interactiveshell for an explanation.  But we need to make sure
         # that, if we overwrite __main__, we replace it at the end
         main_mod_name = prog_ns['__name__']
 
         if main_mod_name == '__main__':
             restore_main = sys.modules['__main__']
         else:
             restore_main = False
 
         # This needs to be undone at the end to prevent holding references to
         # every single object ever created.
         sys.modules[main_mod_name] = main_mod
 
         try:
             stats = None
             with self.readline_no_record:
                 if 'p' in opts:
                     stats = self.magic_prun('', 0, opts, arg_lst, prog_ns)
                 else:
                     if 'd' in opts:
                         deb = debugger.Pdb(self.shell.colors)
                         # reset Breakpoint state, which is moronically kept
                         # in a class
                         bdb.Breakpoint.next = 1
                         bdb.Breakpoint.bplist = {}
                         bdb.Breakpoint.bpbynumber = [None]
                         # Set an initial breakpoint to stop execution
                         maxtries = 10
                         bp = int(opts.get('b', [1])[0])
                         checkline = deb.checkline(filename, bp)
                         if not checkline:
                             for bp in range(bp + 1, bp + maxtries + 1):
                                 if deb.checkline(filename, bp):
                                     break
                             else:
                                 msg = ("\nI failed to find a valid line to set "
                                        "a breakpoint\n"
                                        "after trying up to line: %s.\n"
                                        "Please set a valid breakpoint manually "
                                        "with the -b option." % bp)
                                 error(msg)
                                 return
                         # if we find a good linenumber, set the breakpoint
                         deb.do_break('%s:%s' % (filename, bp))
                         # Start file run
                         print "NOTE: Enter 'c' at the",
                         print "%s prompt to start your script." % deb.prompt
                         ns = {'execfile': py3compat.execfile, 'prog_ns': prog_ns}
                         try:
                             deb.run('execfile("%s", prog_ns)' % filename, ns)
 
                         except:
                             etype, value, tb = sys.exc_info()
                             # Skip three frames in the traceback: the %run one,
                             # one inside bdb.py, and the command-line typed by the
                             # user (run by exec in pdb itself).
                             self.shell.InteractiveTB(etype, value, tb, tb_offset=3)
                     else:
                         if runner is None:
                             runner = self.shell.safe_execfile
                         if 't' in opts:
                             # timed execution
                             try:
                                 nruns = int(opts['N'][0])
                                 if nruns < 1:
                                     error('Number of runs must be >=1')
                                     return
                             except (KeyError):
                                 nruns = 1
                             twall0 = time.time()
                             if nruns == 1:
                                 t0 = clock2()
                                 runner(filename, prog_ns, prog_ns,
                                        exit_ignore=exit_ignore)
                                 t1 = clock2()
                                 t_usr = t1[0] - t0[0]
                                 t_sys = t1[1] - t0[1]
                                 print "\nIPython CPU timings (estimated):"
                                 print "  User   : %10.2f s." % t_usr
                                 print "  System : %10.2f s." % t_sys
                             else:
                                 runs = range(nruns)
                                 t0 = clock2()
                                 for nr in runs:
                                     runner(filename, prog_ns, prog_ns,
                                            exit_ignore=exit_ignore)
                                 t1 = clock2()
                                 t_usr = t1[0] - t0[0]
                                 t_sys = t1[1] - t0[1]
                                 print "\nIPython CPU timings (estimated):"
                                 print "Total runs performed:", nruns
                                 print "  Times  : %10.2f    %10.2f" % ('Total', 'Per run')
                                 print "  User   : %10.2f s, %10.2f s." % (t_usr, t_usr / nruns)
                                 print "  System : %10.2f s, %10.2f s." % (t_sys, t_sys / nruns)
                             twall1 = time.time()
                             print "Wall time: %10.2f s." % (twall1 - twall0)
 
                         else:
                             # regular execution
                             runner(filename, prog_ns, prog_ns, exit_ignore=exit_ignore)
 
                 if 'i' in opts:
                     self.shell.user_ns['__name__'] = __name__save
                 else:
                     # The shell MUST hold a reference to prog_ns so after %run
                     # exits, the python deletion mechanism doesn't zero it out
                     # (leaving dangling references).
                     self.shell.cache_main_mod(prog_ns, filename)
                     # update IPython interactive namespace
 
                     # Some forms of read errors on the file may mean the
                     # __name__ key was never set; using pop we don't have to
                     # worry about a possible KeyError.
                     prog_ns.pop('__name__', None)
 
                     self.shell.user_ns.update(prog_ns)
         finally:
             # It's a bit of a mystery why, but __builtins__ can change from
             # being a module to becoming a dict missing some key data after
             # %run.  As best I can see, this is NOT something IPython is doing
             # at all, and similar problems have been reported before:
             # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
             # Since this seems to be done by the interpreter itself, the best
             # we can do is to at least restore __builtins__ for the user on
             # exit.
             self.shell.user_ns['__builtins__'] = builtin_mod
 
             # Ensure key global structures are restored
             sys.argv = save_argv
             if restore_main:
                 sys.modules['__main__'] = restore_main
             else:
                 # Remove from sys.modules the reference to main_mod we'd
                 # added.  Otherwise it will trap references to objects
                 # contained therein.
                 del sys.modules[main_mod_name]
 
         return stats
 
     @skip_doctest
     def magic_timeit(self, parameter_s =''):
         """Time execution of a Python statement or expression
 
         Usage:\\
           %timeit [-n<N> -r<R> [-t|-c]] statement
 
         Time execution of a Python statement or expression using the timeit
         module.
 
         Options:
         -n<N>: execute the given statement <N> times in a loop. If this value
         is not given, a fitting value is chosen.
 
         -r<R>: repeat the loop iteration <R> times and take the best result.
         Default: 3
 
         -t: use time.time to measure the time, which is the default on Unix.
         This function measures wall time.
 
         -c: use time.clock to measure the time, which is the default on
         Windows and measures wall time. On Unix, resource.getrusage is used
         instead and returns the CPU user time.
 
         -p<P>: use a precision of <P> digits to display the timing result.
         Default: 3
 
 
         Examples
         --------
         ::
 
           In [1]: %timeit pass
           10000000 loops, best of 3: 53.3 ns per loop
 
           In [2]: u = None
 
           In [3]: %timeit u is None
           10000000 loops, best of 3: 184 ns per loop
 
           In [4]: %timeit -r 4 u == None
           1000000 loops, best of 4: 242 ns per loop
 
           In [5]: import time
 
           In [6]: %timeit -n1 time.sleep(2)
           1 loops, best of 3: 2 s per loop
 
 
         The times reported by %timeit will be slightly higher than those
         reported by the timeit.py script when variables are accessed. This is
         due to the fact that %timeit executes the statement in the namespace
         of the shell, compared with timeit.py, which uses a single setup
         statement to import function or create variables. Generally, the bias
         does not matter as long as results from timeit.py are not mixed with
         those from %timeit."""
 
         import timeit
         import math
 
         # XXX: Unfortunately the unicode 'micro' symbol can cause problems in
         # certain terminals.  Until we figure out a robust way of
         # auto-detecting if the terminal can deal with it, use plain 'us' for
         # microseconds.  I am really NOT happy about disabling the proper
         # 'micro' prefix, but crashing is worse... If anyone knows what the
         # right solution for this is, I'm all ears...
         #
         # Note: using
         #
         # s = u'\xb5'
         # s.encode(sys.getdefaultencoding())
         #
         # is not sufficient, as I've seen terminals where that fails but
         # print s
         #
         # succeeds
         #
         # See bug: https://bugs.launchpad.net/ipython/+bug/348466
 
         #units = [u"s", u"ms",u'\xb5',"ns"]
         units = [u"s", u"ms",u'us',"ns"]
 
         scaling = [1, 1e3, 1e6, 1e9]
 
         opts, stmt = self.parse_options(parameter_s,'n:r:tcp:',
                                         posix=False, strict=False)
         if stmt == "":
             return
         timefunc = timeit.default_timer
         number = int(getattr(opts, "n", 0))
         repeat = int(getattr(opts, "r", timeit.default_repeat))
         precision = int(getattr(opts, "p", 3))
         if hasattr(opts, "t"):
             timefunc = time.time
         if hasattr(opts, "c"):
             timefunc = clock
 
         timer = timeit.Timer(timer=timefunc)
         # this code has tight coupling to the inner workings of timeit.Timer,
         # but is there a better way to achieve that the code stmt has access
         # to the shell namespace?
 
         src = timeit.template % {'stmt': timeit.reindent(stmt, 8),
                                  'setup': "pass"}
         # Track compilation time so it can be reported if too long
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
 
         t0 = clock()
         code = compile(src, "<magic-timeit>", "exec")
         tc = clock()-t0
 
         ns = {}
         exec code in self.shell.user_ns, ns
         timer.inner = ns["inner"]
 
         if number == 0:
             # determine number so that 0.2 <= total time < 2.0
             number = 1
             for i in range(1, 10):
                 if timer.timeit(number) >= 0.2:
                     break
                 number *= 10
 
         best = min(timer.repeat(repeat, number)) / number
 
         if best > 0.0 and best < 1000.0:
             order = min(-int(math.floor(math.log10(best)) // 3), 3)
         elif best >= 1000.0:
             order = 0
         else:
             order = 3
         print u"%d loops, best of %d: %.*g %s per loop" % (number, repeat,
                                                           precision,
                                                           best * scaling[order],
                                                           units[order])
         if tc > tc_min:
             print "Compiler time: %.2f s" % tc
 
     @skip_doctest
     @needs_local_scope
     def magic_time(self,parameter_s = ''):
         """Time execution of a Python statement or expression.
 
         The CPU and wall clock times are printed, and the value of the
         expression (if any) is returned.  Note that under Win32, system time
         is always reported as 0, since it can not be measured.
 
         This function provides very basic timing functionality.  In Python
         2.3, the timeit module offers more control and sophistication, so this
         could be rewritten to use it (patches welcome).
 
         Examples
         --------
         ::
 
           In [1]: time 2**128
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
           Out[1]: 340282366920938463463374607431768211456L
 
           In [2]: n = 1000000
 
           In [3]: time sum(range(n))
           CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
           Wall time: 1.37
           Out[3]: 499999500000L
 
           In [4]: time print 'hello world'
           hello world
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
 
           Note that the time needed by Python to compile the given expression
           will be reported if it is more than 0.1s.  In this example, the
           actual exponentiation is done by Python at compilation time, so while
           the expression can take a noticeable amount of time to compute, that
           time is purely due to the compilation:
 
           In [5]: time 3**9999;
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00 s
 
           In [6]: time 3**999999;
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00 s
           Compiler : 0.78 s
           """
 
         # fail immediately if the given expression can't be compiled
 
         expr = self.shell.prefilter(parameter_s,False)
 
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
 
         try:
             mode = 'eval'
             t0 = clock()
             code = compile(expr,'<timed eval>',mode)
             tc = clock()-t0
         except SyntaxError:
             mode = 'exec'
             t0 = clock()
             code = compile(expr,'<timed exec>',mode)
             tc = clock()-t0
         # skew measurement as little as possible
         glob = self.shell.user_ns
         locs = self._magic_locals
         clk = clock2
         wtime = time.time
         # time execution
         wall_st = wtime()
         if mode=='eval':
             st = clk()
             out = eval(code, glob, locs)
             end = clk()
         else:
             st = clk()
             exec code in glob, locs
             end = clk()
             out = None
         wall_end = wtime()
         # Compute actual times and report
         wall_time = wall_end-wall_st
         cpu_user = end[0]-st[0]
         cpu_sys = end[1]-st[1]
         cpu_tot = cpu_user+cpu_sys
         print "CPU times: user %.2f s, sys: %.2f s, total: %.2f s" % \
               (cpu_user,cpu_sys,cpu_tot)
         print "Wall time: %.2f s" % wall_time
         if tc > tc_min:
             print "Compiler : %.2f s" % tc
         return out
 
     @skip_doctest
     def magic_macro(self,parameter_s = ''):
         """Define a macro for future re-execution. It accepts ranges of history,
         filenames or string objects.
 
         Usage:\\
           %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
 
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed as the
           command line is used instead.
 
         This will define a global variable called `name` which is a string
         made of joining the slices and lines you specify (n1,n2,... numbers
         above) from your input history into a single string. This variable
         acts like an automatic function which re-executes those lines as if
         you had typed them. You just type 'name' at the prompt and the code
         executes.
 
         The syntax for indicating input ranges is described in %history.
 
         Note: as a 'hidden' feature, you can also use traditional python slice
         notation, where N:M means numbers N through M-1.
 
         For example, if your history contains (%hist prints it)::
 
           44: x=1
           45: y=3
           46: z=x+y
           47: print x
           48: a=5
           49: print 'x',x,'y',y
 
         you can create a macro with lines 44 through 47 (included) and line 49
         called my_macro with::
 
           In [55]: %macro my_macro 44-47 49
 
         Now, typing `my_macro` (without quotes) will re-execute all this code
         in one pass.
 
         You don't need to give the line-numbers in order, and any given line
         number can appear multiple times. You can assemble macros with any
         lines from your input history in any order.
 
         The macro is a simple object which holds its value in an attribute,
         but IPython's display system checks for macros and executes them as
         code instead of printing them when you type their name.
 
         You can view a macro's contents by explicitly printing it with::
 
           print macro_name
 
         """
         opts,args = self.parse_options(parameter_s,'r',mode='list')
         if not args:   # List existing macros
             return sorted(k for k,v in self.shell.user_ns.iteritems() if\
                                                         isinstance(v, Macro))
         if len(args) == 1:
             raise UsageError(
                 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
         name, codefrom = args[0], " ".join(args[1:])
 
         #print 'rng',ranges  # dbg
         try:
             lines = self.shell.find_user_code(codefrom, 'r' in opts)
         except (ValueError, TypeError) as e:
             print e.args[0]
             return
         macro = Macro(lines)
         self.shell.define_macro(name, macro)
         print 'Macro `%s` created. To execute, type its name (without quotes).' % name
         print '=== Macro contents: ==='
         print macro,
 
     def magic_save(self,parameter_s = ''):
         """Save a set of lines or a macro to a given filename.
 
         Usage:\\
           %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
 
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed as the
           command line is used instead.
 
         This function uses the same syntax as %history for input ranges,
         then saves the lines to the filename you specify.
 
         It adds a '.py' extension to the file if you don't do so yourself, and
         it asks for confirmation before overwriting existing files."""
 
         opts,args = self.parse_options(parameter_s,'r',mode='list')
         fname, codefrom = unquote_filename(args[0]), " ".join(args[1:])
         if not fname.endswith('.py'):
             fname += '.py'
         if os.path.isfile(fname):
             ans = raw_input('File `%s` exists. Overwrite (y/[N])? ' % fname)
             if ans.lower() not in ['y','yes']:
                 print 'Operation cancelled.'
                 return
         try:
             cmds = self.shell.find_user_code(codefrom, 'r' in opts)
         except (TypeError, ValueError) as e:
             print e.args[0]
             return
         with io.open(fname,'w', encoding="utf-8") as f:
             f.write(u"# coding: utf-8\n")
             f.write(py3compat.cast_unicode(cmds))
         print 'The following commands were written to file `%s`:' % fname
         print cmds
 
     def magic_pastebin(self, parameter_s = ''):
         """Upload code to the 'Lodge it' paste bin, returning the URL."""
         try:
             code = self.shell.find_user_code(parameter_s)
         except (ValueError, TypeError) as e:
             print e.args[0]
             return
         pbserver = ServerProxy('http://paste.pocoo.org/xmlrpc/')
         id = pbserver.pastes.newPaste("python", code)
         return "http://paste.pocoo.org/show/" + id
 
     def magic_loadpy(self, arg_s):
         """Load a .py python script into the GUI console.
 
         This magic command can either take a local filename or a url::
 
         %loadpy myscript.py
         %loadpy http://www.example.com/myscript.py
         """
         arg_s = unquote_filename(arg_s)
         remote_url = arg_s.startswith(('http://', 'https://'))
         local_url = not remote_url
         if local_url and not arg_s.endswith('.py'):
             # Local files must be .py; for remote URLs it's possible that the
             # fetch URL doesn't have a .py in it (many servers have an opaque
             # URL, such as scipy-central.org).
             raise ValueError('%%loadpy only works with .py files: %s' % arg_s)
         
         # openpy takes care of finding the source encoding (per PEP 263)
         if remote_url:
             contents = openpy.read_py_url(arg_s, skip_encoding_cookie=True)
         else:
             contents = openpy.read_py_file(arg_s, skip_encoding_cookie=True)
         
         self.set_next_input(contents)
 
     def _find_edit_target(self, args, opts, last_call):
         """Utility method used by magic_edit to find what to edit."""
 
         def make_filename(arg):
             "Make a filename from the given args"
             arg = unquote_filename(arg)
             try:
                 filename = get_py_filename(arg)
             except IOError:
                 # If it ends with .py but doesn't already exist, assume we want
                 # a new file.
                 if arg.endswith('.py'):
                     filename = arg
                 else:
                     filename = None
             return filename
 
         # Set a few locals from the options for convenience:
         opts_prev = 'p' in opts
         opts_raw = 'r' in opts
 
         # custom exceptions
         class DataIsObject(Exception): pass
 
         # Default line number value
         lineno = opts.get('n',None)
 
         if opts_prev:
             args = '_%s' % last_call[0]
             if not self.shell.user_ns.has_key(args):
                 args = last_call[1]
 
         # use last_call to remember the state of the previous call, but don't
         # let it be clobbered by successive '-p' calls.
         try:
             last_call[0] = self.shell.displayhook.prompt_count
             if not opts_prev:
                 last_call[1] = args
         except:
             pass
 
         # by default this is done with temp files, except when the given
         # arg is a filename
         use_temp = True
 
         data = ''
 
         # First, see if the arguments should be a filename.
         filename = make_filename(args)
         if filename:
             use_temp = False
         elif args:
             # Mode where user specifies ranges of lines, like in %macro.
             data = self.extract_input_lines(args, opts_raw)
             if not data:
                 try:
                     # Load the parameter given as a variable. If not a string,
                     # process it as an object instead (below)
 
                     #print '*** args',args,'type',type(args)  # dbg
                     data = eval(args, self.shell.user_ns)
                     if not isinstance(data, basestring):
                         raise DataIsObject
 
                 except (NameError,SyntaxError):
                     # given argument is not a variable, try as a filename
                     filename = make_filename(args)
                     if filename is None:
                         warn("Argument given (%s) can't be found as a variable "
                              "or as a filename." % args)
                         return
                     use_temp = False
 
                 except DataIsObject:
                     # macros have a special edit function
                     if isinstance(data, Macro):
                         raise MacroToEdit(data)
 
                     # For objects, try to edit the file where they are defined
                     try:
                         filename = inspect.getabsfile(data)
                         if 'fakemodule' in filename.lower() and inspect.isclass(data):
                             # class created by %edit? Try to find source
                             # by looking for method definitions instead, the
                             # __module__ in those classes is FakeModule.
                             attrs = [getattr(data, aname) for aname in dir(data)]
                             for attr in attrs:
                                 if not inspect.ismethod(attr):
                                     continue
                                 filename = inspect.getabsfile(attr)
                                 if filename and 'fakemodule' not in filename.lower():
                                     # change the attribute to be the edit target instead
                                     data = attr
                                     break
 
                         datafile = 1
                     except TypeError:
                         filename = make_filename(args)
                         datafile = 1
                         warn('Could not find file where `%s` is defined.\n'
                              'Opening a file named `%s`' % (args,filename))
                     # Now, make sure we can actually read the source (if it was in
                     # a temp file it's gone by now).
                     if datafile:
                         try:
                             if lineno is None:
                                 lineno = inspect.getsourcelines(data)[1]
                         except IOError:
                             filename = make_filename(args)
                             if filename is None:
                                 warn('The file `%s` where `%s` was defined cannot '
                                      'be read.' % (filename,data))
                                 return
                     use_temp = False
 
         if use_temp:
             filename = self.shell.mktempfile(data)
             print 'IPython will make a temporary file named:',filename
 
         return filename, lineno, use_temp
 
     def _edit_macro(self,mname,macro):
         """open an editor with the macro data in a file"""
         filename = self.shell.mktempfile(macro.value)
         self.shell.hooks.editor(filename)
 
         # and make a new macro object, to replace the old one
         mfile = open(filename)
         mvalue = mfile.read()
         mfile.close()
         self.shell.user_ns[mname] = Macro(mvalue)
 
     def magic_ed(self,parameter_s=''):
         """Alias to %edit."""
         return self.magic_edit(parameter_s)
 
     @skip_doctest
     def magic_edit(self,parameter_s='',last_call=['','']):
         """Bring up an editor and execute the resulting code.
 
         Usage:
           %edit [options] [args]
 
         %edit runs IPython's editor hook. The default version of this hook is
         set to call the editor specified by your $EDITOR environment variable.
         If this isn't found, it will default to vi under Linux/Unix and to
         notepad under Windows. See the end of this docstring for how to change
         the editor hook.
 
         You can also set the value of this editor via the
         ``TerminalInteractiveShell.editor`` option in your configuration file.
         This is useful if you wish to use a different editor from your typical
         default with IPython (and for Windows users who typically don't set
         environment variables).
 
         This command allows you to conveniently edit multi-line code right in
         your IPython session.
 
         If called without arguments, %edit opens up an empty editor with a
         temporary file and will execute the contents of this file when you
         close it (don't forget to save it!).
 
 
         Options:
 
         -n <number>: open the editor at a specified line number.  By default,
         the IPython editor hook uses the unix syntax 'editor +N filename', but
         you can configure this by providing your own modified hook if your
         favorite editor supports line-number specifications with a different
         syntax.
 
         -p: this will call the editor with the same data as the previous time
         it was used, regardless of how long ago (in your current session) it
         was.
 
         -r: use 'raw' input.  This option only applies to input taken from the
         user's history.  By default, the 'processed' history is used, so that
         magics are loaded in their transformed version to valid Python.  If
         this option is given, the raw input as typed as the command line is
         used instead.  When you exit the editor, it will be executed by
         IPython's own processor.
 
         -x: do not execute the edited code immediately upon exit. This is
         mainly useful if you are editing programs which need to be called with
         command line arguments, which you can then do using %run.
 
 
         Arguments:
 
         If arguments are given, the following possibilities exist:
 
         - If the argument is a filename, IPython will load that into the
           editor. It will execute its contents with execfile() when you exit,
           loading any code in the file into your interactive namespace.
 
         - The arguments are ranges of input history,  e.g. "7 ~1/4-6".
           The syntax is the same as in the %history magic.
 
         - If the argument is a string variable, its contents are loaded
           into the editor. You can thus edit any string which contains
           python code (including the result of previous edits).
 
         - If the argument is the name of an object (other than a string),
           IPython will try to locate the file where it was defined and open the
           editor at the point where it is defined. You can use `%edit function`
           to load an editor exactly at the point where 'function' is defined,
           edit it and have the file be executed automatically.
 
         - If the object is a macro (see %macro for details), this opens up your
           specified editor with a temporary file containing the macro's data.
           Upon exit, the macro is reloaded with the contents of the file.
 
         Note: opening at an exact line is only supported under Unix, and some
         editors (like kedit and gedit up to Gnome 2.8) do not understand the
         '+NUMBER' parameter necessary for this feature. Good editors like
         (X)Emacs, vi, jed, pico and joe all do.
 
         After executing your code, %edit will return as output the code you
         typed in the editor (except when it was an existing file). This way
         you can reload the code in further invocations of %edit as a variable,
         via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
         the output.
 
         Note that %edit is also available through the alias %ed.
 
         This is an example of creating a simple function inside the editor and
         then modifying it. First, start up the editor::
 
           In [1]: ed
           Editing... done. Executing edited code...
           Out[1]: 'def foo():\\n    print "foo() was defined in an editing
           session"\\n'
 
         We can then call the function foo()::
 
           In [2]: foo()
           foo() was defined in an editing session
 
         Now we edit foo.  IPython automatically loads the editor with the
         (temporary) file where foo() was previously defined::
 
           In [3]: ed foo
           Editing... done. Executing edited code...
 
         And if we call foo() again we get the modified version::
 
           In [4]: foo()
           foo() has now been changed!
 
         Here is an example of how to edit a code snippet successive
         times. First we call the editor::
 
           In [5]: ed
           Editing... done. Executing edited code...
           hello
           Out[5]: "print 'hello'\\n"
 
         Now we call it again with the previous output (stored in _)::
 
           In [6]: ed _
           Editing... done. Executing edited code...
           hello world
           Out[6]: "print 'hello world'\\n"
 
         Now we call it with the output #8 (stored in _8, also as Out[8])::
 
           In [7]: ed _8
           Editing... done. Executing edited code...
           hello again
           Out[7]: "print 'hello again'\\n"
 
 
         Changing the default editor hook:
 
         If you wish to write your own editor hook, you can put it in a
         configuration file which you load at startup time.  The default hook
         is defined in the IPython.core.hooks module, and you can use that as a
         starting example for further modifications.  That file also has
         general instructions on how to set a new hook for use once you've
         defined it."""
         opts,args = self.parse_options(parameter_s,'prxn:')
 
         try:
             filename, lineno, is_temp = self._find_edit_target(args, opts, last_call)
         except MacroToEdit as e:
             self._edit_macro(args, e.args[0])
             return
 
         # do actual editing here
         print 'Editing...',
         sys.stdout.flush()
         try:
             # Quote filenames that may have spaces in them
             if ' ' in filename:
                 filename = "'%s'" % filename
             self.shell.hooks.editor(filename,lineno)
         except TryNext:
             warn('Could not open editor')
             return
 
         # XXX TODO: should this be generalized for all string vars?
         # For now, this is special-cased to blocks created by cpaste
         if args.strip() == 'pasted_block':
             self.shell.user_ns['pasted_block'] = file_read(filename)
 
         if 'x' in opts:  # -x prevents actual execution
             print
         else:
             print 'done. Executing edited code...'
             if 'r' in opts:    # Untranslated IPython code
                 self.shell.run_cell(file_read(filename),
                                                     store_history=False)
             else:
                 self.shell.safe_execfile(filename,self.shell.user_ns,
                                          self.shell.user_ns)
 
         if is_temp:
             try:
                 return open(filename).read()
             except IOError,msg:
                 if msg.filename == filename:
                     warn('File not found. Did you forget to save?')
                     return
                 else:
                     self.shell.showtraceback()
 
     def magic_xmode(self,parameter_s = ''):
         """Switch modes for the exception handlers.
 
         Valid modes: Plain, Context and Verbose.
 
         If called without arguments, acts as a toggle."""
 
         def xmode_switch_err(name):
             warn('Error changing %s exception modes.\n%s' %
                  (name,sys.exc_info()[1]))
 
         shell = self.shell
         new_mode = parameter_s.strip().capitalize()
         try:
             shell.InteractiveTB.set_mode(mode=new_mode)
             print 'Exception reporting mode:',shell.InteractiveTB.mode
         except:
             xmode_switch_err('user')
 
     def magic_colors(self,parameter_s = ''):
         """Switch color scheme for prompts, info system and exception handlers.
 
         Currently implemented schemes: NoColor, Linux, LightBG.
 
         Color scheme names are not case-sensitive.
 
         Examples
         --------
         To get a plain black and white terminal::
 
           %colors nocolor
         """
 
         def color_switch_err(name):
             warn('Error changing %s color schemes.\n%s' %
                  (name,sys.exc_info()[1]))
 
 
         new_scheme = parameter_s.strip()
         if not new_scheme:
             raise UsageError(
                 "%colors: you must specify a color scheme. See '%colors?'")
             return
         # local shortcut
         shell = self.shell
 
         import IPython.utils.rlineimpl as readline
 
         if not shell.colors_force and \
                 not readline.have_readline and sys.platform == "win32":
             msg = """\
 Proper color support under MS Windows requires the pyreadline library.
 You can find it at:
 http://ipython.org/pyreadline.html
 Gary's readline needs the ctypes module, from:
 http://starship.python.net/crew/theller/ctypes
 (Note that ctypes is already part of Python versions 2.5 and newer).
 
 Defaulting color scheme to 'NoColor'"""
             new_scheme = 'NoColor'
             warn(msg)
 
         # readline option is 0
         if not shell.colors_force and not shell.has_readline:
             new_scheme = 'NoColor'
 
         # Set prompt colors
         try:
             shell.prompt_manager.color_scheme = new_scheme
         except:
             color_switch_err('prompt')
         else:
             shell.colors = \
                    shell.prompt_manager.color_scheme_table.active_scheme_name
         # Set exception colors
         try:
             shell.InteractiveTB.set_colors(scheme = new_scheme)
             shell.SyntaxTB.set_colors(scheme = new_scheme)
         except:
             color_switch_err('exception')
 
         # Set info (for 'object?') colors
         if shell.color_info:
             try:
                 shell.inspector.set_active_scheme(new_scheme)
             except:
                 color_switch_err('object inspector')
         else:
             shell.inspector.set_active_scheme('NoColor')
 
     def magic_pprint(self, parameter_s=''):
         """Toggle pretty printing on/off."""
         ptformatter = self.shell.display_formatter.formatters['text/plain']
         ptformatter.pprint = bool(1 - ptformatter.pprint)
         print 'Pretty printing has been turned', \
               ['OFF','ON'][ptformatter.pprint]
 
     #......................................................................
     # Functions to implement unix shell-type things
 
     @skip_doctest
     def magic_alias(self, parameter_s = ''):
         """Define an alias for a system command.
 
         '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
 
         Then, typing 'alias_name params' will execute the system command 'cmd
         params' (from your underlying operating system).
 
         Aliases have lower precedence than magic functions and Python normal
         variables, so if 'foo' is both a Python variable and an alias, the
         alias can not be executed until 'del foo' removes the Python variable.
 
         You can use the %l specifier in an alias definition to represent the
         whole line when the alias is called.  For example::
 
           In [2]: alias bracket echo "Input in brackets: <%l>"
           In [3]: bracket hello world
           Input in brackets: <hello world>
 
         You can also define aliases with parameters using %s specifiers (one
         per parameter)::
 
           In [1]: alias parts echo first %s second %s
           In [2]: %parts A B
           first A second B
           In [3]: %parts A
           Incorrect number of arguments: 2 expected.
           parts is an alias to: 'echo first %s second %s'
 
         Note that %l and %s are mutually exclusive.  You can only use one or
         the other in your aliases.
 
         Aliases expand Python variables just like system calls using ! or !!
         do: all expressions prefixed with '$' get expanded.  For details of
         the semantic rules, see PEP-215:
         http://www.python.org/peps/pep-0215.html.  This is the library used by
         IPython for variable expansion.  If you want to access a true shell
         variable, an extra $ is necessary to prevent its expansion by
         IPython::
 
           In [6]: alias show echo
           In [7]: PATH='A Python string'
           In [8]: show $PATH
           A Python string
           In [9]: show $$PATH
           /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
 
         You can use the alias facility to acess all of $PATH.  See the %rehash
         and %rehashx functions, which automatically create aliases for the
         contents of your $PATH.
 
         If called with no parameters, %alias prints the current alias table."""
 
         par = parameter_s.strip()
         if not par:
             stored = self.db.get('stored_aliases', {} )
             aliases = sorted(self.shell.alias_manager.aliases)
             # for k, v in stored:
             #     atab.append(k, v[0])
 
             print "Total number of aliases:", len(aliases)
             sys.stdout.flush()
             return aliases
 
         # Now try to define a new one
         try:
             alias,cmd = par.split(None, 1)
         except:
             print oinspect.getdoc(self.magic_alias)
         else:
             self.shell.alias_manager.soft_define_alias(alias, cmd)
     # end magic_alias
 
     def magic_unalias(self, parameter_s = ''):
         """Remove an alias"""
 
         aname = parameter_s.strip()
         self.shell.alias_manager.undefine_alias(aname)
         stored = self.db.get('stored_aliases', {} )
         if aname in stored:
             print "Removing %stored alias",aname
             del stored[aname]
             self.db['stored_aliases'] = stored
 
     def magic_rehashx(self, parameter_s = ''):
         """Update the alias table with all executable files in $PATH.
 
         This version explicitly checks that every entry in $PATH is a file
         with execute access (os.X_OK), so it is much slower than %rehash.
 
         Under Windows, it checks executability as a match against a
         '|'-separated string of extensions, stored in the IPython config
         variable win_exec_ext.  This defaults to 'exe|com|bat'.
 
         This function also resets the root module cache of module completer,
         used on slow filesystems.
         """
         from IPython.core.alias import InvalidAliasError
 
         # for the benefit of module completer in ipy_completers.py
         del self.shell.db['rootmodules']
 
         path = [os.path.abspath(os.path.expanduser(p)) for p in
             os.environ.get('PATH','').split(os.pathsep)]
         path = filter(os.path.isdir,path)
 
         syscmdlist = []
         # Now define isexec in a cross platform manner.
         if os.name == 'posix':
             isexec = lambda fname:os.path.isfile(fname) and \
                      os.access(fname,os.X_OK)
         else:
             try:
                 winext = os.environ['pathext'].replace(';','|').replace('.','')
             except KeyError:
                 winext = 'exe|com|bat|py'
             if 'py' not in winext:
                 winext += '|py'
             execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
             isexec = lambda fname:os.path.isfile(fname) and execre.match(fname)
         savedir = os.getcwdu()
 
         # Now walk the paths looking for executables to alias.
         try:
             # write the whole loop for posix/Windows so we don't have an if in
             # the innermost part
             if os.name == 'posix':
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):
                         if isexec(ff):
                             try:
                                 # Removes dots from the name since ipython
                                 # will assume names with dots to be python.
                                 self.shell.alias_manager.define_alias(
                                     ff.replace('.',''), ff)
                             except InvalidAliasError:
                                 pass
                             else:
                                 syscmdlist.append(ff)
             else:
                 no_alias = self.shell.alias_manager.no_alias
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):
                         base, ext = os.path.splitext(ff)
                         if isexec(ff) and base.lower() not in no_alias:
                             if ext.lower() == '.exe':
                                 ff = base
                                 try:
                                     # Removes dots from the name since ipython
                                     # will assume names with dots to be python.
                                     self.shell.alias_manager.define_alias(
                                         base.lower().replace('.',''), ff)
                                 except InvalidAliasError:
                                     pass
                                 syscmdlist.append(ff)
             self.shell.db['syscmdlist'] = syscmdlist
         finally:
             os.chdir(savedir)
 
     @skip_doctest
     def magic_pwd(self, parameter_s = ''):
         """Return the current working directory path.
 
         Examples
         --------
         ::
 
           In [9]: pwd
           Out[9]: '/home/tsuser/sprint/ipython'
         """
         return os.getcwdu()
 
     @skip_doctest
     def magic_cd(self, parameter_s=''):
         """Change the current working directory.
 
         This command automatically maintains an internal list of directories
         you visit during your IPython session, in the variable _dh. The
         command %dhist shows this history nicely formatted. You can also
         do 'cd -<tab>' to see directory history conveniently.
 
         Usage:
 
           cd 'dir': changes to directory 'dir'.
 
           cd -: changes to the last visited directory.
 
           cd -<n>: changes to the n-th directory in the directory history.
 
           cd --foo: change to directory that matches 'foo' in history
 
           cd -b <bookmark_name>: jump to a bookmark set by %bookmark
              (note: cd <bookmark_name> is enough if there is no
               directory <bookmark_name>, but a bookmark with the name exists.)
               'cd -b <tab>' allows you to tab-complete bookmark names.
 
         Options:
 
         -q: quiet.  Do not print the working directory after the cd command is
         executed.  By default IPython's cd command does print this directory,
         since the default prompts do not display path information.
 
         Note that !cd doesn't work for this purpose because the shell where
         !command runs is immediately discarded after executing 'command'.
 
         Examples
         --------
         ::
 
           In [10]: cd parent/child
           /home/tsuser/parent/child
         """
 
         parameter_s = parameter_s.strip()
         #bkms = self.shell.persist.get("bookmarks",{})
 
         oldcwd = os.getcwdu()
         numcd = re.match(r'(-)(\d+)$',parameter_s)
         # jump in directory history by number
         if numcd:
             nn = int(numcd.group(2))
             try:
                 ps = self.shell.user_ns['_dh'][nn]
             except IndexError:
                 print 'The requested directory does not exist in history.'
                 return
             else:
                 opts = {}
         elif parameter_s.startswith('--'):
             ps = None
             fallback = None
             pat = parameter_s[2:]
             dh = self.shell.user_ns['_dh']
             # first search only by basename (last component)
             for ent in reversed(dh):
                 if pat in os.path.basename(ent) and os.path.isdir(ent):
                     ps = ent
                     break
 
                 if fallback is None and pat in ent and os.path.isdir(ent):
                     fallback = ent
 
             # if we have no last part match, pick the first full path match
             if ps is None:
                 ps = fallback
 
             if ps is None:
                 print "No matching entry in directory history"
                 return
             else:
                 opts = {}
 
 
         else:
             #turn all non-space-escaping backslashes to slashes,
             # for c:\windows\directory\names\
             parameter_s = re.sub(r'\\(?! )','/', parameter_s)
             opts,ps = self.parse_options(parameter_s,'qb',mode='string')
         # jump to previous
         if ps == '-':
             try:
                 ps = self.shell.user_ns['_dh'][-2]
             except IndexError:
                 raise UsageError('%cd -: No previous directory to change to.')
         # jump to bookmark if needed
         else:
             if not os.path.isdir(ps) or opts.has_key('b'):
                 bkms = self.db.get('bookmarks', {})
 
                 if bkms.has_key(ps):
                     target = bkms[ps]
                     print '(bookmark:%s) -> %s' % (ps,target)
                     ps = target
                 else:
                     if opts.has_key('b'):
                         raise UsageError("Bookmark '%s' not found.  "
                               "Use '%%bookmark -l' to see your bookmarks." % ps)
 
         # strip extra quotes on Windows, because os.chdir doesn't like them
         ps = unquote_filename(ps)
         # at this point ps should point to the target dir
         if ps:
             try:
                 os.chdir(os.path.expanduser(ps))
                 if hasattr(self.shell, 'term_title') and self.shell.term_title:
                     set_term_title('IPython: ' + abbrev_cwd())
             except OSError:
                 print sys.exc_info()[1]
             else:
                 cwd = os.getcwdu()
                 dhist = self.shell.user_ns['_dh']
                 if oldcwd != cwd:
                     dhist.append(cwd)
                     self.db['dhist'] = compress_dhist(dhist)[-100:]
 
         else:
             os.chdir(self.shell.home_dir)
             if hasattr(self.shell, 'term_title') and self.shell.term_title:
                 set_term_title('IPython: ' + '~')
             cwd = os.getcwdu()
             dhist = self.shell.user_ns['_dh']
 
             if oldcwd != cwd:
                 dhist.append(cwd)
                 self.db['dhist'] = compress_dhist(dhist)[-100:]
         if not 'q' in opts and self.shell.user_ns['_dh']:
             print self.shell.user_ns['_dh'][-1]
 
 
     def magic_env(self, parameter_s=''):
         """List environment variables."""
 
         return dict(os.environ)
 
     def magic_pushd(self, parameter_s=''):
         """Place the current dir on stack and change directory.
 
         Usage:\\
           %pushd ['dirname']
         """
 
         dir_s = self.shell.dir_stack
         tgt = os.path.expanduser(unquote_filename(parameter_s))
         cwd = os.getcwdu().replace(self.home_dir,'~')
         if tgt:
             self.magic_cd(parameter_s)
         dir_s.insert(0,cwd)
         return self.magic_dirs()
 
     def magic_popd(self, parameter_s=''):
         """Change to directory popped off the top of the stack.
         """
         if not self.shell.dir_stack:
             raise UsageError("%popd on empty stack")
         top = self.shell.dir_stack.pop(0)
         self.magic_cd(top)
         print "popd ->",top
 
     def magic_dirs(self, parameter_s=''):
         """Return the current directory stack."""
 
         return self.shell.dir_stack
 
     def magic_dhist(self, parameter_s=''):
         """Print your history of visited directories.
 
         %dhist       -> print full history\\
         %dhist n     -> print last n entries only\\
         %dhist n1 n2 -> print entries between n1 and n2 (n1 not included)\\
 
         This history is automatically maintained by the %cd command, and
         always available as the global list variable _dh. You can use %cd -<n>
         to go to directory number <n>.
 
         Note that most of time, you should view directory history by entering
         cd -<TAB>.
 
         """
 
         dh = self.shell.user_ns['_dh']
         if parameter_s:
             try:
                 args = map(int,parameter_s.split())
             except:
                 self.arg_err(Magic.magic_dhist)
                 return
             if len(args) == 1:
                 ini,fin = max(len(dh)-(args[0]),0),len(dh)
             elif len(args) == 2:
                 ini,fin = args
             else:
                 self.arg_err(Magic.magic_dhist)
                 return
         else:
             ini,fin = 0,len(dh)
         nlprint(dh,
                 header = 'Directory history (kept in _dh)',
                 start=ini,stop=fin)
 
     @skip_doctest
     def magic_sc(self, parameter_s=''):
         """Shell capture - execute a shell command and capture its output.
 
         DEPRECATED. Suboptimal, retained for backwards compatibility.
 
         You should use the form 'var = !command' instead. Example:
 
          "%sc -l myfiles = ls ~" should now be written as
 
          "myfiles = !ls ~"
 
         myfiles.s, myfiles.l and myfiles.n still apply as documented
         below.
 
         --
         %sc [options] varname=command
 
         IPython will run the given command using commands.getoutput(), and
         will then update the user's interactive namespace with a variable
         called varname, containing the value of the call.  Your command can
         contain shell wildcards, pipes, etc.
 
         The '=' sign in the syntax is mandatory, and the variable name you
         supply must follow Python's standard conventions for valid names.
 
         (A special format without variable name exists for internal use)
 
         Options:
 
           -l: list output.  Split the output on newlines into a list before
           assigning it to the given variable.  By default the output is stored
           as a single string.
 
           -v: verbose.  Print the contents of the variable.
 
         In most cases you should not need to split as a list, because the
         returned value is a special type of string which can automatically
         provide its contents either as a list (split on newlines) or as a
         space-separated string.  These are convenient, respectively, either
         for sequential processing or to be passed to a shell command.
 
         For example::
 
             # Capture into variable a
             In [1]: sc a=ls *py
 
             # a is a string with embedded newlines
             In [2]: a
             Out[2]: 'setup.py\\nwin32_manual_post_install.py'
 
             # which can be seen as a list:
             In [3]: a.l
             Out[3]: ['setup.py', 'win32_manual_post_install.py']
 
             # or as a whitespace-separated string:
             In [4]: a.s
             Out[4]: 'setup.py win32_manual_post_install.py'
 
             # a.s is useful to pass as a single command line:
             In [5]: !wc -l $a.s
               146 setup.py
               130 win32_manual_post_install.py
               276 total
 
             # while the list form is useful to loop over:
             In [6]: for f in a.l:
               ...:      !wc -l $f
               ...:
             146 setup.py
             130 win32_manual_post_install.py
 
         Similarly, the lists returned by the -l option are also special, in
         the sense that you can equally invoke the .s attribute on them to
         automatically get a whitespace-separated string from their contents::
 
             In [7]: sc -l b=ls *py
 
             In [8]: b
             Out[8]: ['setup.py', 'win32_manual_post_install.py']
 
             In [9]: b.s
             Out[9]: 'setup.py win32_manual_post_install.py'
 
         In summary, both the lists and strings used for output capture have
         the following special attributes::
 
             .l (or .list) : value as list.
             .n (or .nlstr): value as newline-separated string.
             .s (or .spstr): value as space-separated string.
         """
 
         opts,args = self.parse_options(parameter_s,'lv')
         # Try to get a variable name and command to run
         try:
             # the variable name must be obtained from the parse_options
             # output, which uses shlex.split to strip options out.
             var,_ = args.split('=',1)
             var = var.strip()
             # But the command has to be extracted from the original input
             # parameter_s, not on what parse_options returns, to avoid the
             # quote stripping which shlex.split performs on it.
             _,cmd = parameter_s.split('=',1)
         except ValueError:
             var,cmd = '',''
         # If all looks ok, proceed
         split = 'l' in opts
         out = self.shell.getoutput(cmd, split=split)
         if opts.has_key('v'):
             print '%s ==\n%s' % (var,pformat(out))
         if var:
             self.shell.user_ns.update({var:out})
         else:
             return out
 
     def magic_sx(self, parameter_s=''):
         """Shell execute - run a shell command and capture its output.
 
         %sx command
 
         IPython will run the given command using commands.getoutput(), and
         return the result formatted as a list (split on '\\n').  Since the
         output is _returned_, it will be stored in ipython's regular output
         cache Out[N] and in the '_N' automatic variables.
 
         Notes:
 
         1) If an input line begins with '!!', then %sx is automatically
         invoked.  That is, while::
 
           !ls
 
         causes ipython to simply issue system('ls'), typing::
 
           !!ls
 
         is a shorthand equivalent to::
 
           %sx ls
 
         2) %sx differs from %sc in that %sx automatically splits into a list,
         like '%sc -l'.  The reason for this is to make it as easy as possible
         to process line-oriented shell output via further python commands.
         %sc is meant to provide much finer control, but requires more
         typing.
 
         3) Just like %sc -l, this is a list with special attributes:
         ::
 
           .l (or .list) : value as list.
           .n (or .nlstr): value as newline-separated string.
           .s (or .spstr): value as whitespace-separated string.
 
         This is very useful when trying to use such lists as arguments to
         system commands."""
 
         if parameter_s:
             return self.shell.getoutput(parameter_s)
 
 
     def magic_bookmark(self, parameter_s=''):
         """Manage IPython's bookmark system.
 
         %bookmark <name>       - set bookmark to current dir
         %bookmark <name> <dir> - set bookmark to <dir>
         %bookmark -l           - list all bookmarks
         %bookmark -d <name>    - remove bookmark
         %bookmark -r           - remove all bookmarks
 
         You can later on access a bookmarked folder with::
 
           %cd -b <name>
 
         or simply '%cd <name>' if there is no directory called <name> AND
         there is such a bookmark defined.
 
         Your bookmarks persist through IPython sessions, but they are
         associated with each profile."""
 
         opts,args = self.parse_options(parameter_s,'drl',mode='list')
         if len(args) > 2:
             raise UsageError("%bookmark: too many arguments")
 
         bkms = self.db.get('bookmarks',{})
 
         if opts.has_key('d'):
             try:
                 todel = args[0]
             except IndexError:
                 raise UsageError(
                     "%bookmark -d: must provide a bookmark to delete")
             else:
                 try:
                     del bkms[todel]
                 except KeyError:
                     raise UsageError(
                         "%%bookmark -d: Can't delete bookmark '%s'" % todel)
 
         elif opts.has_key('r'):
             bkms = {}
         elif opts.has_key('l'):
             bks = bkms.keys()
             bks.sort()
             if bks:
                 size = max(map(len,bks))
             else:
                 size = 0
             fmt = '%-'+str(size)+'s -> %s'
             print 'Current bookmarks:'
             for bk in bks:
                 print fmt % (bk,bkms[bk])
         else:
             if not args:
                 raise UsageError("%bookmark: You must specify the bookmark name")
             elif len(args)==1:
                 bkms[args[0]] = os.getcwdu()
             elif len(args)==2:
                 bkms[args[0]] = args[1]
         self.db['bookmarks'] = bkms
 
     def magic_pycat(self, parameter_s=''):
         """Show a syntax-highlighted file through a pager.
 
         This magic is similar to the cat utility, but it will assume the file
         to be Python source and will show it with syntax highlighting. """
 
         try:
             filename = get_py_filename(parameter_s)
             cont = file_read(filename)
         except IOError:
             try:
                 cont = eval(parameter_s,self.user_ns)
             except NameError:
                 cont = None
         if cont is None:
             print "Error: no such file or variable"
             return
 
         page.page(self.shell.pycolorize(cont))
 
     def magic_quickref(self,arg):
         """ Show a quick reference sheet """
         import IPython.core.usage
         qr = IPython.core.usage.quick_reference + self.magic_magic('-brief')
 
         page.page(qr)
 
     def magic_doctest_mode(self,parameter_s=''):
         """Toggle doctest mode on and off.
 
         This mode is intended to make IPython behave as much as possible like a
         plain Python shell, from the perspective of how its prompts, exceptions
         and output look.  This makes it easy to copy and paste parts of a
         session into doctests.  It does so by:
 
         - Changing the prompts to the classic ``>>>`` ones.
         - Changing the exception reporting mode to 'Plain'.
         - Disabling pretty-printing of output.
 
         Note that IPython also supports the pasting of code snippets that have
         leading '>>>' and '...' prompts in them.  This means that you can paste
         doctests from files or docstrings (even if they have leading
         whitespace), and the code will execute correctly.  You can then use
         '%history -t' to see the translated history; this will give you the
         input after removal of all the leading prompts and whitespace, which
         can be pasted back into an editor.
 
         With these features, you can switch into this mode easily whenever you
         need to do testing and changes to doctests, without having to leave
         your existing IPython session.
         """
 
         from IPython.utils.ipstruct import Struct
 
         # Shorthands
         shell = self.shell
         pm = shell.prompt_manager
         meta = shell.meta
         disp_formatter = self.shell.display_formatter
         ptformatter = disp_formatter.formatters['text/plain']
         # dstore is a data store kept in the instance metadata bag to track any
         # changes we make, so we can undo them later.
         dstore = meta.setdefault('doctest_mode',Struct())
         save_dstore = dstore.setdefault
 
         # save a few values we'll need to recover later
         mode = save_dstore('mode',False)
         save_dstore('rc_pprint',ptformatter.pprint)
         save_dstore('xmode',shell.InteractiveTB.mode)
         save_dstore('rc_separate_out',shell.separate_out)
         save_dstore('rc_separate_out2',shell.separate_out2)
         save_dstore('rc_prompts_pad_left',pm.justify)
         save_dstore('rc_separate_in',shell.separate_in)
         save_dstore('rc_plain_text_only',disp_formatter.plain_text_only)
         save_dstore('prompt_templates',(pm.in_template, pm.in2_template, pm.out_template))
 
         if mode == False:
             # turn on
             pm.in_template = '>>> '
             pm.in2_template = '... '
             pm.out_template = ''
 
             # Prompt separators like plain python
             shell.separate_in = ''
             shell.separate_out = ''
             shell.separate_out2 = ''
 
             pm.justify = False
 
             ptformatter.pprint = False
             disp_formatter.plain_text_only = True
 
             shell.magic_xmode('Plain')
         else:
             # turn off
             pm.in_template, pm.in2_template, pm.out_template = dstore.prompt_templates
 
             shell.separate_in = dstore.rc_separate_in
 
             shell.separate_out = dstore.rc_separate_out
             shell.separate_out2 = dstore.rc_separate_out2
 
             pm.justify = dstore.rc_prompts_pad_left
 
             ptformatter.pprint = dstore.rc_pprint
             disp_formatter.plain_text_only = dstore.rc_plain_text_only
 
             shell.magic_xmode(dstore.xmode)
 
         # Store new mode and inform
         dstore.mode = bool(1-int(mode))
         mode_label = ['OFF','ON'][dstore.mode]
         print 'Doctest mode is:', mode_label
 
     def magic_gui(self, parameter_s=''):
         """Enable or disable IPython GUI event loop integration.
 
         %gui [GUINAME]
 
         This magic replaces IPython's threaded shells that were activated
         using the (pylab/wthread/etc.) command line flags.  GUI toolkits
         can now be enabled at runtime and keyboard
         interrupts should work without any problems.  The following toolkits
         are supported:  wxPython, PyQt4, PyGTK, Tk and Cocoa (OSX)::
 
             %gui wx      # enable wxPython event loop integration
             %gui qt4|qt  # enable PyQt4 event loop integration
             %gui gtk     # enable PyGTK event loop integration
             %gui gtk3    # enable Gtk3 event loop integration
             %gui tk      # enable Tk event loop integration
             %gui OSX     # enable Cocoa event loop integration
                          # (requires %matplotlib 1.1) 
             %gui         # disable all event loop integration
 
         WARNING:  after any of these has been called you can simply create
         an application object, but DO NOT start the event loop yourself, as
         we have already handled that.
         """
         opts, arg = self.parse_options(parameter_s, '')
         if arg=='': arg = None
         try:
             return self.enable_gui(arg)
         except Exception as e:
             # print simple error message, rather than traceback if we can't
             # hook up the GUI
             error(str(e))
     
     def magic_install_ext(self, parameter_s):
         """Download and install an extension from a URL, e.g.::
         
             %install_ext https://bitbucket.org/birkenfeld/ipython-physics/raw/d1310a2ab15d/physics.py
         
         The URL should point to an importable Python module - either a .py file
         or a .zip file.
         
         Parameters:
         
           -n filename : Specify a name for the file, rather than taking it from
                         the URL.
         """
         opts, args = self.parse_options(parameter_s, 'n:')
         try:
             filename = self.extension_manager.install_extension(args, opts.get('n'))
         except ValueError as e:
             print e
             return
         
         filename = os.path.basename(filename)
         print "Installed %s. To use it, type:" % filename
         print "  %%load_ext %s" % os.path.splitext(filename)[0]
         
 
     def magic_load_ext(self, module_str):
         """Load an IPython extension by its module name."""
         return self.extension_manager.load_extension(module_str)
 
     def magic_unload_ext(self, module_str):
         """Unload an IPython extension by its module name."""
         self.extension_manager.unload_extension(module_str)
 
     def magic_reload_ext(self, module_str):
         """Reload an IPython extension by its module name."""
         self.extension_manager.reload_extension(module_str)
 
     def magic_install_profiles(self, s):
         """%install_profiles has been deprecated."""
         print '\n'.join([
             "%install_profiles has been deprecated.",
             "Use `ipython profile list` to view available profiles.",
             "Requesting a profile with `ipython profile create <name>`",
             "or `ipython --profile=<name>` will start with the bundled",
             "profile of that name if it exists." 
         ])
 
     def magic_install_default_config(self, s):
         """%install_default_config has been deprecated."""
         print '\n'.join([
             "%install_default_config has been deprecated.",
             "Use `ipython profile create <name>` to initialize a profile",
             "with the default config files.",
             "Add `--reset` to overwrite already existing config files with defaults."
         ])
 
     # Pylab support: simple wrappers that activate pylab, load gui input
     # handling and modify slightly %run
 
     @skip_doctest
     def _pylab_magic_run(self, parameter_s=''):
         Magic.magic_run(self, parameter_s,
                         runner=mpl_runner(self.shell.safe_execfile))
 
     _pylab_magic_run.__doc__ = magic_run.__doc__
 
     @skip_doctest
     def magic_pylab(self, s):
         """Load numpy and matplotlib to work interactively.
 
         %pylab [GUINAME]
 
         This function lets you activate pylab (matplotlib, numpy and
         interactive support) at any point during an IPython session.
 
         It will import at the top level numpy as np, pyplot as plt, matplotlib,
         pylab and mlab, as well as all names from numpy and pylab.
 
         If you are using the inline matplotlib backend for embedded figures,
         you can adjust its behavior via the %config magic::
 
             # enable SVG figures, necessary for SVG+XHTML export in the qtconsole
             In [1]: %config InlineBackend.figure_format = 'svg'
             
             # change the behavior of closing all figures at the end of each
             # execution (cell), or allowing reuse of active figures across
             # cells:
             In [2]: %config InlineBackend.close_figures = False
 
         Parameters
         ----------
         guiname : optional
           One of the valid arguments to the %gui magic ('qt', 'wx', 'gtk',
           'osx' or 'tk').  If given, the corresponding Matplotlib backend is
           used, otherwise matplotlib's default (which you can override in your
           matplotlib config file) is used.
 
         Examples
         --------
         In this case, where the MPL default is TkAgg::
 
             In [2]: %pylab
 
             Welcome to pylab, a matplotlib-based Python environment.
             Backend in use: TkAgg
             For more information, type 'help(pylab)'.
 
         But you can explicitly request a different backend::
 
             In [3]: %pylab qt
 
             Welcome to pylab, a matplotlib-based Python environment.
             Backend in use: Qt4Agg
             For more information, type 'help(pylab)'.
         """
 
         if Application.initialized():
             app = Application.instance()
             try:
                 import_all_status = app.pylab_import_all
             except AttributeError:
                 import_all_status = True
         else:
             import_all_status = True
 
         self.shell.enable_pylab(s, import_all=import_all_status)
 
     def magic_tb(self, s):
         """Print the last traceback with the currently active exception mode.
 
         See %xmode for changing exception reporting modes."""
         self.shell.showtraceback()
 
     @skip_doctest
     def magic_precision(self, s=''):
         """Set floating point precision for pretty printing.
 
         Can set either integer precision or a format string.
 
         If numpy has been imported and precision is an int,
         numpy display precision will also be set, via ``numpy.set_printoptions``.
 
         If no argument is given, defaults will be restored.
 
         Examples
         --------
         ::
 
             In [1]: from math import pi
 
             In [2]: %precision 3
             Out[2]: u'%.3f'
 
             In [3]: pi
             Out[3]: 3.142
 
             In [4]: %precision %i
             Out[4]: u'%i'
 
             In [5]: pi
             Out[5]: 3
 
             In [6]: %precision %e
             Out[6]: u'%e'
 
             In [7]: pi**10
             Out[7]: 9.364805e+04
 
             In [8]: %precision
             Out[8]: u'%r'
 
             In [9]: pi**10
             Out[9]: 93648.047476082982
 
         """
 
         ptformatter = self.shell.display_formatter.formatters['text/plain']
         ptformatter.float_precision = s
         return ptformatter.float_format
 
 
     @magic_arguments.magic_arguments()
     @magic_arguments.argument(
         '-e', '--export', action='store_true', default=False,
         help='Export IPython history as a notebook. The filename argument '
              'is used to specify the notebook name and format. For example '
              'a filename of notebook.ipynb will result in a notebook name '
              'of "notebook" and a format of "xml". Likewise using a ".json" '
              'or ".py" file extension will write the notebook in the json '
              'or py formats.'
     )
     @magic_arguments.argument(
         '-f', '--format',
         help='Convert an existing IPython notebook to a new format. This option '
              'specifies the new format and can have the values: xml, json, py. '
              'The target filename is chosen automatically based on the new '
              'format. The filename argument gives the name of the source file.'
     )
     @magic_arguments.argument(
         'filename', type=unicode,
         help='Notebook name or filename'
     )
     def magic_notebook(self, s):
         """Export and convert IPython notebooks.
 
         This function can export the current IPython history to a notebook file
         or can convert an existing notebook file into a different format. For
         example, to export the history to "foo.ipynb" do "%notebook -e foo.ipynb".
         To export the history to "foo.py" do "%notebook -e foo.py". To convert
         "foo.ipynb" to "foo.json" do "%notebook -f json foo.ipynb". Possible
         formats include (json/ipynb, py).
         """
         args = magic_arguments.parse_argstring(self.magic_notebook, s)
 
         from IPython.nbformat import current
         args.filename = unquote_filename(args.filename)
         if args.export:
             fname, name, format = current.parse_filename(args.filename)
             cells = []
             hist = list(self.history_manager.get_range())
             for session, prompt_number, input in hist[:-1]:
                 cells.append(current.new_code_cell(prompt_number=prompt_number, input=input))
             worksheet = current.new_worksheet(cells=cells)
             nb = current.new_notebook(name=name,worksheets=[worksheet])
             with io.open(fname, 'w', encoding='utf-8') as f:
                 current.write(nb, f, format);
         elif args.format is not None:
             old_fname, old_name, old_format = current.parse_filename(args.filename)
             new_format = args.format
             if new_format == u'xml':
                 raise ValueError('Notebooks cannot be written as xml.')
             elif new_format == u'ipynb' or new_format == u'json':
                 new_fname = old_name + u'.ipynb'
                 new_format = u'json'
             elif new_format == u'py':
                 new_fname = old_name + u'.py'
             else:
                 raise ValueError('Invalid notebook format: %s' % new_format)
             with io.open(old_fname, 'r', encoding='utf-8') as f:
                 nb = current.read(f, old_format)
             with io.open(new_fname, 'w', encoding='utf-8') as f:
                 current.write(nb, f, new_format)
 
     def magic_config(self, s):
         """configure IPython
         
             %config Class[.trait=value]
         
         This magic exposes most of the IPython config system. Any
         Configurable class should be able to be configured with the simple
         line::
         
             %config Class.trait=value
         
         Where `value` will be resolved in the user's namespace, if it is an
         expression or variable name.
         
         Examples
         --------
         
         To see what classes are available for config, pass no arguments::
 
             In [1]: %config
             Available objects for config:
                 TerminalInteractiveShell
                 HistoryManager
                 PrefilterManager
                 AliasManager
                 IPCompleter
                 PromptManager
                 DisplayFormatter
 
         To view what is configurable on a given class, just pass the class
         name::
 
             In [2]: %config IPCompleter
             IPCompleter options
             -----------------
             IPCompleter.omit__names=<Enum>
                 Current: 2
                 Choices: (0, 1, 2)
                 Instruct the completer to omit private method names
                 Specifically, when completing on ``object.<tab>``.
                 When 2 [default]: all names that start with '_' will be excluded.
                 When 1: all 'magic' names (``__foo__``) will be excluded.
                 When 0: nothing will be excluded.
             IPCompleter.merge_completions=<CBool>
                 Current: True
                 Whether to merge completion results into a single list
                 If False, only the completion results from the first non-empty completer
                 will be returned.
             IPCompleter.limit_to__all__=<CBool>
                 Current: False
                 Instruct the completer to use __all__ for the completion
                 Specifically, when completing on ``object.<tab>``.
                 When True: only those names in obj.__all__ will be included.
                 When False [default]: the __all__ attribute is ignored
             IPCompleter.greedy=<CBool>
                 Current: False
                 Activate greedy completion
                 This will enable completion on elements of lists, results of function calls,
                 etc., but can be unsafe because the code is actually evaluated on TAB.
 
         but the real use is in setting values::
 
             In [3]: %config IPCompleter.greedy = True
 
         and these values are read from the user_ns if they are variables::
 
             In [4]: feeling_greedy=False
 
             In [5]: %config IPCompleter.greedy = feeling_greedy
 
         """
         from IPython.config.loader import Config
         # some IPython objects are Configurable, but do not yet have
         # any configurable traits.  Exclude them from the effects of
         # this magic, as their presence is just noise:
         configurables = [ c for c in self.configurables if c.__class__.class_traits(config=True) ]
         classnames = [ c.__class__.__name__ for c in configurables ]
         
         line = s.strip()
         if not line:
             # print available configurable names
             print "Available objects for config:"
             for name in classnames:
                 print "    ", name
             return
         elif line in classnames:
             # `%config TerminalInteractiveShell` will print trait info for
             # TerminalInteractiveShell
             c = configurables[classnames.index(line)]
             cls = c.__class__
             help = cls.class_get_help(c)
             # strip leading '--' from cl-args:
             help = re.sub(re.compile(r'^--', re.MULTILINE), '', help)
             print help
             return
         elif '=' not in line:
             raise UsageError("Invalid config statement: %r, should be Class.trait = value" % line)
             
         
         # otherwise, assume we are setting configurables.
         # leave quotes on args when splitting, because we want
         # unquoted args to eval in user_ns
         cfg = Config()
         exec "cfg."+line in locals(), self.user_ns
         
         for configurable in configurables:
             try:
                 configurable.update_config(cfg)
             except Exception as e:
                 error(e)
 
 # end Magic

IPython/core/splitinput.py

@@ -1,139 +1,140 b''
 # encoding: utf-8
 """
 Simple utility for splitting user input. This is used by both inputsplitter and
 prefilter.
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import re
 import sys
 
 from IPython.utils import py3compat
+from IPython.utils.encoding import get_stream_enc
 
 #-----------------------------------------------------------------------------
 # Main function
 #-----------------------------------------------------------------------------
 
 # RegExp for splitting line contents into pre-char//first word-method//rest.
 # For clarity, each group in on one line.
 
 # WARNING: update the regexp if the escapes in interactiveshell are changed, as
 # they are hardwired in.
 
 # Although it's not solely driven by the regex, note that:
 # ,;/% only trigger if they are the first character on the line
 # ! and !! trigger if they are first char(s) *or* follow an indent
 # ? triggers as first or last char.
 
 line_split = re.compile("""
              ^(\s*)               # any leading space
              ([,;/%]|!!?|\?\??)?  # escape character or characters
              \s*(%?[\w\.\*]*)     # function/method, possibly with leading %
                                   # to correctly treat things like '?%magic'
              (.*?$|$)             # rest of line
              """, re.VERBOSE)
 
 def split_user_input(line, pattern=None):
     """Split user input into initial whitespace, escape character, function part
     and the rest.
     """
     # We need to ensure that the rest of this routine deals only with unicode
-    encoding = py3compat.get_stream_enc(sys.stdin, 'utf-8')
+    encoding = get_stream_enc(sys.stdin, 'utf-8')
     line = py3compat.cast_unicode(line, encoding)
 
     if pattern is None:
         pattern = line_split
     match = pattern.match(line)
     if not match:
         # print "match failed for line '%s'" % line
         try:
             ifun, the_rest = line.split(None,1)
         except ValueError:
             # print "split failed for line '%s'" % line
             ifun, the_rest = line, u''
         pre = re.match('^(\s*)(.*)',line).groups()[0]
         esc = ""
     else:
         pre, esc, ifun, the_rest = match.groups()
 
     #print 'line:<%s>' % line # dbg
     #print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun.strip(),the_rest) # dbg
     return pre, esc or '', ifun.strip(), the_rest.lstrip()
 
 class LineInfo(object):
     """A single line of input and associated info.
 
     Includes the following as properties:
 
     line
       The original, raw line
 
     continue_prompt
       Is this line a continuation in a sequence of multiline input?
 
     pre
       Any leading whitespace.
 
     esc
       The escape character(s) in pre or the empty string if there isn't one.
       Note that '!!' and '??' are possible values for esc. Otherwise it will
       always be a single character.
 
     ifun
       The 'function part', which is basically the maximal initial sequence
       of valid python identifiers and the '.' character. This is what is
       checked for alias and magic transformations, used for auto-calling,
       etc. In contrast to Python identifiers, it may start with "%" and contain
       "*".
 
     the_rest
       Everything else on the line.
     """
     def __init__(self, line, continue_prompt=False):
         self.line            = line
         self.continue_prompt = continue_prompt
         self.pre, self.esc, self.ifun, self.the_rest = split_user_input(line)
 
         self.pre_char       = self.pre.strip()
         if self.pre_char:
             self.pre_whitespace = '' # No whitespace allowd before esc chars
         else:
             self.pre_whitespace = self.pre
 
         self._oinfo = None
 
     def ofind(self, ip):
         """Do a full, attribute-walking lookup of the ifun in the various
         namespaces for the given IPython InteractiveShell instance.
 
         Return a dict with keys: found,obj,ospace,ismagic
 
         Note: can cause state changes because of calling getattr, but should
         only be run if autocall is on and if the line hasn't matched any
         other, less dangerous handlers.
 
         Does cache the results of the call, so can be called multiple times
         without worrying about *further* damaging state.
         """
         if not self._oinfo:
             # ip.shell._ofind is actually on the Magic class!
             self._oinfo = ip.shell._ofind(self.ifun)
         return self._oinfo
 
     def __str__(self):
         return "LineInfo [%s|%s|%s|%s]" %(self.pre, self.esc, self.ifun, self.the_rest)

IPython/core/tests/test_compilerop.py

@@ -1,75 +1,75 b''
 # coding: utf-8
 """Tests for the compilerop module.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011 The IPython Development Team.
 #
 #  Distributed under the terms of the BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Stdlib imports
 import linecache
 import sys
 
 # Third-party imports
 import nose.tools as nt
 
 # Our own imports
 from IPython.core import compilerop
 from IPython.utils import py3compat
 
 #-----------------------------------------------------------------------------
 # Test functions
 #-----------------------------------------------------------------------------
 
 def test_code_name():
     code = 'x=1'
     name = compilerop.code_name(code)
     nt.assert_true(name.startswith('<ipython-input-0'))
 
 
 def test_code_name2():
     code = 'x=1'
     name = compilerop.code_name(code, 9)
     nt.assert_true(name.startswith('<ipython-input-9'))
 
 
 def test_cache():
     """Test the compiler correctly compiles and caches inputs
     """
     cp = compilerop.CachingCompiler()
     ncache = len(linecache.cache)
     cp.cache('x=1')
     nt.assert_true(len(linecache.cache) > ncache)
 
 def setUp():
     # Check we're in a proper Python 2 environment (some imports, such
     # as GTK, can change the default encoding, which can hide bugs.)
-    nt.assert_equal(py3compat.getdefaultencoding(), "utf-8" if py3compat.PY3 else "ascii")
+    nt.assert_equal(sys.getdefaultencoding(), "utf-8" if py3compat.PY3 else "ascii")
 
 def test_cache_unicode():
     cp = compilerop.CachingCompiler()
     ncache = len(linecache.cache)
     cp.cache(u"t = 'žćčšđ'")
     nt.assert_true(len(linecache.cache) > ncache)
 
 def test_compiler_check_cache():
     """Test the compiler properly manages the cache.
     """
     # Rather simple-minded tests that just exercise the API
     cp = compilerop.CachingCompiler()
     cp.cache('x=1', 99)
     # Ensure now that after clearing the cache, our entries survive
     cp.check_cache()
     for k in linecache.cache:
         if k.startswith('<ipython-input-99'):
             break
     else:
         raise AssertionError('Entry for input-99 missing from linecache')

IPython/core/tests/test_history.py

@@ -1,151 +1,151 b''
 # coding: utf-8
 """Tests for the IPython tab-completion machinery.
 """
 #-----------------------------------------------------------------------------
 # Module imports
 #-----------------------------------------------------------------------------
 
 # stdlib
 import os
 import shutil
 import sys
 import tempfile
 import unittest
 from datetime import datetime
 
 # third party
 import nose.tools as nt
 
 # our own packages
 from IPython.config.loader import Config
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.core.history import HistoryManager, extract_hist_ranges
 from IPython.utils import py3compat
 
 def setUp():
-    nt.assert_equal(py3compat.getdefaultencoding(), "utf-8" if py3compat.PY3 else "ascii")
+    nt.assert_equal(sys.getdefaultencoding(), "utf-8" if py3compat.PY3 else "ascii")
 
 def test_history():
     ip = get_ipython()
     with TemporaryDirectory() as tmpdir:
         hist_manager_ori = ip.history_manager
         hist_file = os.path.join(tmpdir, 'history.sqlite')
         try:
             ip.history_manager = HistoryManager(shell=ip, hist_file=hist_file)
             hist = [u'a=1', u'def f():\n    test = 1\n    return test', u"b='€Æ¾÷ß'"]
             for i, h in enumerate(hist, start=1):
                 ip.history_manager.store_inputs(i, h)
 
             ip.history_manager.db_log_output = True
             # Doesn't match the input, but we'll just check it's stored.
             ip.history_manager.output_hist_reprs[3] = "spam"
             ip.history_manager.store_output(3)
 
             nt.assert_equal(ip.history_manager.input_hist_raw, [''] + hist)
             
             # Detailed tests for _get_range_session
             grs = ip.history_manager._get_range_session
             nt.assert_equal(list(grs(start=2,stop=-1)), zip([0], [2], hist[1:-1]))
             nt.assert_equal(list(grs(start=-2)), zip([0,0], [2,3], hist[-2:]))
             nt.assert_equal(list(grs(output=True)), zip([0,0,0], [1,2,3], zip(hist, [None,None,'spam'])))
 
             # Check whether specifying a range beyond the end of the current
             # session results in an error (gh-804)
             ip.magic('%hist 2-500')
             
             # Check that we can write non-ascii characters to a file
             ip.magic("%%hist -f %s" % os.path.join(tmpdir, "test1"))
             ip.magic("%%hist -pf %s" % os.path.join(tmpdir, "test2"))
             ip.magic("%%hist -nf %s" % os.path.join(tmpdir, "test3"))
             ip.magic("%%save %s 1-10" % os.path.join(tmpdir, "test4"))
 
             # New session
             ip.history_manager.reset()
             newcmds = ["z=5","class X(object):\n    pass", "k='p'"]
             for i, cmd in enumerate(newcmds, start=1):
                 ip.history_manager.store_inputs(i, cmd)
             gothist = ip.history_manager.get_range(start=1, stop=4)
             nt.assert_equal(list(gothist), zip([0,0,0],[1,2,3], newcmds))
             # Previous session:
             gothist = ip.history_manager.get_range(-1, 1, 4)
             nt.assert_equal(list(gothist), zip([1,1,1],[1,2,3], hist))
 
             # Check get_hist_tail
             gothist = ip.history_manager.get_tail(4, output=True,
                                                     include_latest=True)
             expected = [(1, 3, (hist[-1], "spam")),
                         (2, 1, (newcmds[0], None)),
                         (2, 2, (newcmds[1], None)),
                         (2, 3, (newcmds[2], None)),]
             nt.assert_equal(list(gothist), expected)
 
             gothist = ip.history_manager.get_tail(2)
             expected = [(2, 1, newcmds[0]),
                         (2, 2, newcmds[1])]
             nt.assert_equal(list(gothist), expected)
 
             # Check get_hist_search
             gothist = ip.history_manager.search("*test*")
             nt.assert_equal(list(gothist), [(1,2,hist[1])] )
             gothist = ip.history_manager.search("b*", output=True)
             nt.assert_equal(list(gothist), [(1,3,(hist[2],"spam"))] )
 
             # Cross testing: check that magic %save can get previous session.
             testfilename = os.path.realpath(os.path.join(tmpdir, "test.py"))
             ip.magic_save(testfilename + " ~1/1-3")
             with py3compat.open(testfilename) as testfile:
                 nt.assert_equal(testfile.read(),
                                         u"# coding: utf-8\n" + u"\n".join(hist))
 
             # Duplicate line numbers - check that it doesn't crash, and
             # gets a new session
             ip.history_manager.store_inputs(1, "rogue")
             ip.history_manager.writeout_cache()
             nt.assert_equal(ip.history_manager.session_number, 3)
         finally:
             # Restore history manager
             ip.history_manager = hist_manager_ori
 
 
 def test_extract_hist_ranges():
     instr = "1 2/3 ~4/5-6 ~4/7-~4/9 ~9/2-~7/5"
     expected = [(0, 1, 2),  # 0 == current session
                 (2, 3, 4),
                 (-4, 5, 7),
                 (-4, 7, 10),
                 (-9, 2, None),  # None == to end
                 (-8, 1, None),
                 (-7, 1, 6)]
     actual = list(extract_hist_ranges(instr))
     nt.assert_equal(actual, expected)
 
 def test_magic_rerun():
     """Simple test for %rerun (no args -> rerun last line)"""
     ip = get_ipython()
     ip.run_cell("a = 10", store_history=True)
     ip.run_cell("a += 1", store_history=True)
     nt.assert_equal(ip.user_ns["a"], 11)
     ip.run_cell("%rerun", store_history=True)
     nt.assert_equal(ip.user_ns["a"], 12)
 
 def test_timestamp_type():
     ip = get_ipython()
     info = ip.history_manager.get_session_info()
     nt.assert_true(isinstance(info[1], datetime))
 
 def test_hist_file_config():
     cfg = Config()
     tfile = tempfile.NamedTemporaryFile(delete=False)
     cfg.HistoryManager.hist_file = tfile.name
     try:
         hm = HistoryManager(shell=get_ipython(), config=cfg)
         nt.assert_equals(hm.hist_file, cfg.HistoryManager.hist_file)
     finally:
         try:
             os.remove(tfile.name)
         except OSError:
             # same catch as in testing.tools.TempFileMixin
             # On Windows, even though we close the file, we still can't
             # delete it.  I have no clue why
             pass
 

IPython/external/pyparsing/_pyparsing.py

@@ -1,3708 +1,3708 b''
 # -*- coding: utf-8 -*-
 # module pyparsing.py
 #
 # Copyright (c) 2003-2009  Paul T. McGuire
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
 # "Software"), to deal in the Software without restriction, including
 # without limitation the rights to use, copy, modify, merge, publish,
 # distribute, sublicense, and/or sell copies of the Software, and to
 # permit persons to whom the Software is furnished to do so, subject to
 # the following conditions:
 #
 # The above copyright notice and this permission notice shall be
 # included in all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #
 #from __future__ import generators
 
 __doc__ = \
 """
 pyparsing module - Classes and methods to define and execute parsing grammars
 
 The pyparsing module is an alternative approach to creating and executing simple grammars,
 vs. the traditional lex/yacc approach, or the use of regular expressions.  With pyparsing, you
 don't need to learn a new syntax for defining grammars or matching expressions - the parsing module
 provides a library of classes that you use to construct the grammar directly in Python.
 
 Here is a program to parse "Hello, World!" (or any greeting of the form "<salutation>, <addressee>!")::
 
     from pyparsing import Word, alphas
 
     # define grammar of a greeting
     greet = Word( alphas ) + "," + Word( alphas ) + "!"
 
     hello = "Hello, World!"
     print hello, "->", greet.parseString( hello )
 
 The program outputs the following::
 
     Hello, World! -> ['Hello', ',', 'World', '!']
 
 The Python representation of the grammar is quite readable, owing to the self-explanatory
 class names, and the use of '+', '|' and '^' operators.
 
 The parsed results returned from parseString() can be accessed as a nested list, a dictionary, or an
 object with named attributes.
 
 The pyparsing module handles some of the problems that are typically vexing when writing text parsers:
  - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello  ,  World  !", etc.)
  - quoted strings
  - embedded comments
 """
 
 __version__ = "1.5.2"
 __versionTime__ = "17 February 2009 19:45"
 __author__ = "Paul McGuire <[e-mail]>"
 
 import string
 from weakref import ref as wkref
 import copy
 import sys
 import warnings
 import re
 import sre_constants
 #~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) )
 
 __all__ = [
 'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty',
 'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal',
 'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or',
 'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException',
 'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException',
 'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', 'Upcase',
 'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore',
 'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col',
 'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString',
 'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'getTokensEndLoc', 'hexnums',
 'htmlComment', 'javaStyleComment', 'keepOriginalText', 'line', 'lineEnd', 'lineStart', 'lineno',
 'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral',
 'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables',
 'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity',
 'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd',
 'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute',
 'indentedBlock', 'originalTextFor',
 ]
 
 
 """
 Detect if we are running version 3.X and make appropriate changes
 Robert A. Clark
 """
 if sys.version_info[0] > 2:
     _PY3K = True
     _MAX_INT = sys.maxsize
     basestring = str
 else:
     _PY3K = False
     _MAX_INT = sys.maxint
 
 if not _PY3K:
     def _ustr(obj):
         """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries
            str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It
            then < returns the unicode object | encodes it with the default encoding | ... >.
         """
         if isinstance(obj,unicode):
             return obj
 
         try:
             # If this works, then _ustr(obj) has the same behaviour as str(obj), so
             # it won't break any existing code.
             return str(obj)
 
         except UnicodeEncodeError:
             # The Python docs (http://docs.python.org/ref/customization.html#l2h-182)
             # state that "The return value must be a string object". However, does a
             # unicode object (being a subclass of basestring) count as a "string
             # object"?
             # If so, then return a unicode object:
             return unicode(obj)
             # Else encode it... but how? There are many choices... :)
             # Replace unprintables with escape codes?
-            #return unicode(obj).encode(py3compat.getdefaultencoding(), 'backslashreplace_errors')
+            #return unicode(obj).encode(sys.getdefaultencoding(), 'backslashreplace_errors')
             # Replace unprintables with question marks?
-            #return unicode(obj).encode(py3compat.getdefaultencoding(), 'replace')
+            #return unicode(obj).encode(sys.getdefaultencoding(), 'replace')
             # ...
 else:
     _ustr = str
     unichr = chr
 
 if not _PY3K:
 	def _str2dict(strg):
 	    return dict( [(c,0) for c in strg] )
 else:
 	_str2dict = set
 
 def _xml_escape(data):
     """Escape &, <, >, ", ', etc. in a string of data."""
 
     # ampersand must be replaced first
     from_symbols = '&><"\''
     to_symbols = ['&'+s+';' for s in "amp gt lt quot apos".split()]
     for from_,to_ in zip(from_symbols, to_symbols):
         data = data.replace(from_, to_)
     return data
 
 class _Constants(object):
     pass
 
 if not _PY3K:
     alphas     = string.lowercase + string.uppercase
 else:
     alphas     = string.ascii_lowercase + string.ascii_uppercase
 nums       = string.digits
 hexnums    = nums + "ABCDEFabcdef"
 alphanums  = alphas + nums
 _bslash = chr(92)
 printables = "".join( [ c for c in string.printable if c not in string.whitespace ] )
 
 class ParseBaseException(Exception):
     """base exception class for all parsing runtime exceptions"""
     # Performance tuning: we construct a *lot* of these, so keep this
     # constructor as small and fast as possible
     def __init__( self, pstr, loc=0, msg=None, elem=None ):
         self.loc = loc
         if msg is None:
             self.msg = pstr
             self.pstr = ""
         else:
             self.msg = msg
             self.pstr = pstr
         self.parserElement = elem
 
     def __getattr__( self, aname ):
         """supported attributes by name are:
             - lineno - returns the line number of the exception text
             - col - returns the column number of the exception text
             - line - returns the line containing the exception text
         """
         if( aname == "lineno" ):
             return lineno( self.loc, self.pstr )
         elif( aname in ("col", "column") ):
             return col( self.loc, self.pstr )
         elif( aname == "line" ):
             return line( self.loc, self.pstr )
         else:
             raise AttributeError(aname)
 
     def __str__( self ):
         return "%s (at char %d), (line:%d, col:%d)" % \
                 ( self.msg, self.loc, self.lineno, self.column )
     def __repr__( self ):
         return _ustr(self)
     def markInputline( self, markerString = ">!<" ):
         """Extracts the exception line from the input string, and marks
            the location of the exception with a special symbol.
         """
         line_str = self.line
         line_column = self.column - 1
         if markerString:
             line_str = "".join( [line_str[:line_column],
                                 markerString, line_str[line_column:]])
         return line_str.strip()
     def __dir__(self):
         return "loc msg pstr parserElement lineno col line " \
                "markInputLine __str__ __repr__".split()
 
 class ParseException(ParseBaseException):
     """exception thrown when parse expressions don't match class;
        supported attributes by name are:
         - lineno - returns the line number of the exception text
         - col - returns the column number of the exception text
         - line - returns the line containing the exception text
     """
     pass
 
 class ParseFatalException(ParseBaseException):
     """user-throwable exception thrown when inconsistent parse content
        is found; stops all parsing immediately"""
     pass
 
 class ParseSyntaxException(ParseFatalException):
     """just like ParseFatalException, but thrown internally when an
        ErrorStop indicates that parsing is to stop immediately because
        an unbacktrackable syntax error has been found"""
     def __init__(self, pe):
         super(ParseSyntaxException, self).__init__(
                                     pe.pstr, pe.loc, pe.msg, pe.parserElement)
 
 #~ class ReparseException(ParseBaseException):
     #~ """Experimental class - parse actions can raise this exception to cause
        #~ pyparsing to reparse the input string:
         #~ - with a modified input string, and/or
         #~ - with a modified start location
        #~ Set the values of the ReparseException in the constructor, and raise the
        #~ exception in a parse action to cause pyparsing to use the new string/location.
        #~ Setting the values as None causes no change to be made.
        #~ """
     #~ def __init_( self, newstring, restartLoc ):
         #~ self.newParseText = newstring
         #~ self.reparseLoc = restartLoc
 
 class RecursiveGrammarException(Exception):
     """exception thrown by validate() if the grammar could be improperly recursive"""
     def __init__( self, parseElementList ):
         self.parseElementTrace = parseElementList
 
     def __str__( self ):
         return "RecursiveGrammarException: %s" % self.parseElementTrace
 
 class _ParseResultsWithOffset(object):
     def __init__(self,p1,p2):
         self.tup = (p1,p2)
     def __getitem__(self,i):
         return self.tup[i]
     def __repr__(self):
         return repr(self.tup)
     def setOffset(self,i):
         self.tup = (self.tup[0],i)
 
 class ParseResults(object):
     """Structured parse results, to provide multiple means of access to the parsed data:
        - as a list (len(results))
        - by list index (results[0], results[1], etc.)
        - by attribute (results.<resultsName>)
        """
     __slots__ = ( "__toklist", "__tokdict", "__doinit", "__name", "__parent", "__accumNames", "__weakref__" )
     def __new__(cls, toklist, name=None, asList=True, modal=True ):
         if isinstance(toklist, cls):
             return toklist
         retobj = object.__new__(cls)
         retobj.__doinit = True
         return retobj
 
     # Performance tuning: we construct a *lot* of these, so keep this
     # constructor as small and fast as possible
     def __init__( self, toklist, name=None, asList=True, modal=True ):
         if self.__doinit:
             self.__doinit = False
             self.__name = None
             self.__parent = None
             self.__accumNames = {}
             if isinstance(toklist, list):
                 self.__toklist = toklist[:]
             else:
                 self.__toklist = [toklist]
             self.__tokdict = dict()
 
         if name:
             if not modal:
                 self.__accumNames[name] = 0
             if isinstance(name,int):
                 name = _ustr(name) # will always return a str, but use _ustr for consistency
             self.__name = name
             if not toklist in (None,'',[]):
                 if isinstance(toklist,basestring):
                     toklist = [ toklist ]
                 if asList:
                     if isinstance(toklist,ParseResults):
                         self[name] = _ParseResultsWithOffset(toklist.copy(),0)
                     else:
                         self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]),0)
                     self[name].__name = name
                 else:
                     try:
                         self[name] = toklist[0]
                     except (KeyError,TypeError,IndexError):
                         self[name] = toklist
 
     def __getitem__( self, i ):
         if isinstance( i, (int,slice) ):
             return self.__toklist[i]
         else:
             if i not in self.__accumNames:
                 return self.__tokdict[i][-1][0]
             else:
                 return ParseResults([ v[0] for v in self.__tokdict[i] ])
 
     def __setitem__( self, k, v ):
         if isinstance(v,_ParseResultsWithOffset):
             self.__tokdict[k] = self.__tokdict.get(k,list()) + [v]
             sub = v[0]
         elif isinstance(k,int):
             self.__toklist[k] = v
             sub = v
         else:
             self.__tokdict[k] = self.__tokdict.get(k,list()) + [_ParseResultsWithOffset(v,0)]
             sub = v
         if isinstance(sub,ParseResults):
             sub.__parent = wkref(self)
 
     def __delitem__( self, i ):
         if isinstance(i,(int,slice)):
             mylen = len( self.__toklist )
             del self.__toklist[i]
 
             # convert int to slice
             if isinstance(i, int):
                 if i < 0:
                     i += mylen
                 i = slice(i, i+1)
             # get removed indices
             removed = list(range(*i.indices(mylen)))
             removed.reverse()
             # fixup indices in token dictionary
             for name in self.__tokdict:
                 occurrences = self.__tokdict[name]
                 for j in removed:
                     for k, (value, position) in enumerate(occurrences):
                         occurrences[k] = _ParseResultsWithOffset(value, position - (position > j))
         else:
             del self.__tokdict[i]
 
     def __contains__( self, k ):
         return k in self.__tokdict
 
     def __len__( self ): return len( self.__toklist )
     def __bool__(self): return len( self.__toklist ) > 0
     __nonzero__ = __bool__
     def __iter__( self ): return iter( self.__toklist )
     def __reversed__( self ): return iter( reversed(self.__toklist) )
     def keys( self ):
         """Returns all named result keys."""
         return self.__tokdict.keys()
 
     def pop( self, index=-1 ):
         """Removes and returns item at specified index (default=last).
            Will work with either numeric indices or dict-key indicies."""
         ret = self[index]
         del self[index]
         return ret
 
     def get(self, key, defaultValue=None):
         """Returns named result matching the given key, or if there is no
            such name, then returns the given defaultValue or None if no
            defaultValue is specified."""
         if key in self:
             return self[key]
         else:
             return defaultValue
 
     def insert( self, index, insStr ):
         self.__toklist.insert(index, insStr)
         # fixup indices in token dictionary
         for name in self.__tokdict:
             occurrences = self.__tokdict[name]
             for k, (value, position) in enumerate(occurrences):
                 occurrences[k] = _ParseResultsWithOffset(value, position + (position > index))
 
     def items( self ):
         """Returns all named result keys and values as a list of tuples."""
         return [(k,self[k]) for k in self.__tokdict]
 
     def values( self ):
         """Returns all named result values."""
         return [ v[-1][0] for v in self.__tokdict.itervalues() ]
 
     def __getattr__( self, name ):
         if name not in self.__slots__:
             if name in self.__tokdict:
                 if name not in self.__accumNames:
                     return self.__tokdict[name][-1][0]
                 else:
                     return ParseResults([ v[0] for v in self.__tokdict[name] ])
             else:
                 return ""
         return None
 
     def __add__( self, other ):
         ret = self.copy()
         ret += other
         return ret
 
     def __iadd__( self, other ):
         if other.__tokdict:
             offset = len(self.__toklist)
             addoffset = ( lambda a: (a<0 and offset) or (a+offset) )
             otheritems = other.__tokdict.iteritems()
             otherdictitems = [(k, _ParseResultsWithOffset(v[0],addoffset(v[1])) )
                                 for (k,vlist) in otheritems for v in vlist]
             for k,v in otherdictitems:
                 self[k] = v
                 if isinstance(v[0],ParseResults):
                     v[0].__parent = wkref(self)
 
         self.__toklist += other.__toklist
         self.__accumNames.update( other.__accumNames )
         del other
         return self
 
     def __repr__( self ):
         return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) )
 
     def __str__( self ):
         out = "["
         sep = ""
         for i in self.__toklist:
             if isinstance(i, ParseResults):
                 out += sep + _ustr(i)
             else:
                 out += sep + repr(i)
             sep = ", "
         out += "]"
         return out
 
     def _asStringList( self, sep='' ):
         out = []
         for item in self.__toklist:
             if out and sep:
                 out.append(sep)
             if isinstance( item, ParseResults ):
                 out += item._asStringList()
             else:
                 out.append( _ustr(item) )
         return out
 
     def asList( self ):
         """Returns the parse results as a nested list of matching tokens, all converted to strings."""
         out = []
         for res in self.__toklist:
             if isinstance(res,ParseResults):
                 out.append( res.asList() )
             else:
                 out.append( res )
         return out
 
     def asDict( self ):
         """Returns the named parse results as dictionary."""
         return dict( self.items() )
 
     def copy( self ):
         """Returns a new copy of a ParseResults object."""
         ret = ParseResults( self.__toklist )
         ret.__tokdict = self.__tokdict.copy()
         ret.__parent = self.__parent
         ret.__accumNames.update( self.__accumNames )
         ret.__name = self.__name
         return ret
 
     def asXML( self, doctag=None, namedItemsOnly=False, indent="", formatted=True ):
         """Returns the parse results as XML. Tags are created for tokens and lists that have defined results names."""
         nl = "\n"
         out = []
         namedItems = dict([(v[1],k) for (k,vlist) in self.__tokdict.iteritems()
                                                             for v in vlist ] )
         nextLevelIndent = indent + "  "
 
         # collapse out indents if formatting is not desired
         if not formatted:
             indent = ""
             nextLevelIndent = ""
             nl = ""
 
         selfTag = None
         if doctag is not None:
             selfTag = doctag
         else:
             if self.__name:
                 selfTag = self.__name
 
         if not selfTag:
             if namedItemsOnly:
                 return ""
             else:
                 selfTag = "ITEM"
 
         out += [ nl, indent, "<", selfTag, ">" ]
 
         worklist = self.__toklist
         for i,res in enumerate(worklist):
             if isinstance(res,ParseResults):
                 if i in namedItems:
                     out += [ res.asXML(namedItems[i],
                                         namedItemsOnly and doctag is None,
                                         nextLevelIndent,
                                         formatted)]
                 else:
                     out += [ res.asXML(None,
                                         namedItemsOnly and doctag is None,
                                         nextLevelIndent,
                                         formatted)]
             else:
                 # individual token, see if there is a name for it
                 resTag = None
                 if i in namedItems:
                     resTag = namedItems[i]
                 if not resTag:
                     if namedItemsOnly:
                         continue
                     else:
                         resTag = "ITEM"
                 xmlBodyText = _xml_escape(_ustr(res))
                 out += [ nl, nextLevelIndent, "<", resTag, ">",
                                                 xmlBodyText,
                                                 "</", resTag, ">" ]
 
         out += [ nl, indent, "</", selfTag, ">" ]
         return "".join(out)
 
     def __lookup(self,sub):
         for k,vlist in self.__tokdict.iteritems():
             for v,loc in vlist:
                 if sub is v:
                     return k
         return None
 
     def getName(self):
         """Returns the results name for this token expression."""
         if self.__name:
             return self.__name
         elif self.__parent:
             par = self.__parent()
             if par:
                 return par.__lookup(self)
             else:
                 return None
         elif (len(self) == 1 and
                len(self.__tokdict) == 1 and
                self.__tokdict.values()[0][0][1] in (0,-1)):
             return self.__tokdict.keys()[0]
         else:
             return None
 
     def dump(self,indent='',depth=0):
         """Diagnostic method for listing out the contents of a ParseResults.
            Accepts an optional indent argument so that this string can be embedded
            in a nested display of other data."""
         out = []
         out.append( indent+_ustr(self.asList()) )
         keys = self.items()
         keys.sort()
         for k,v in keys:
             if out:
                 out.append('\n')
             out.append( "%s%s- %s: " % (indent,('  '*depth), k) )
             if isinstance(v,ParseResults):
                 if v.keys():
                     #~ out.append('\n')
                     out.append( v.dump(indent,depth+1) )
                     #~ out.append('\n')
                 else:
                     out.append(_ustr(v))
             else:
                 out.append(_ustr(v))
         #~ out.append('\n')
         return "".join(out)
 
     # add support for pickle protocol
     def __getstate__(self):
         return ( self.__toklist,
                  ( self.__tokdict.copy(),
                    self.__parent is not None and self.__parent() or None,
                    self.__accumNames,
                    self.__name ) )
 
     def __setstate__(self,state):
         self.__toklist = state[0]
         self.__tokdict, \
         par, \
         inAccumNames, \
         self.__name = state[1]
         self.__accumNames = {}
         self.__accumNames.update(inAccumNames)
         if par is not None:
             self.__parent = wkref(par)
         else:
             self.__parent = None
 
     def __dir__(self):
         return dir(super(ParseResults,self)) + self.keys()
 
 def col (loc,strg):
     """Returns current column within a string, counting newlines as line separators.
    The first column is number 1.
 
    Note: the default parsing behavior is to expand tabs in the input string
    before starting the parsing process.  See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information
    on parsing strings containing <TAB>s, and suggested methods to maintain a
    consistent view of the parsed string, the parse location, and line and column
    positions within the parsed string.
    """
     return (loc<len(strg) and strg[loc] == '\n') and 1 or loc - strg.rfind("\n", 0, loc)
 
 def lineno(loc,strg):
     """Returns current line number within a string, counting newlines as line separators.
    The first line is number 1.
 
    Note: the default parsing behavior is to expand tabs in the input string
    before starting the parsing process.  See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information
    on parsing strings containing <TAB>s, and suggested methods to maintain a
    consistent view of the parsed string, the parse location, and line and column
    positions within the parsed string.
    """
     return strg.count("\n",0,loc) + 1
 
 def line( loc, strg ):
     """Returns the line of text containing loc within a string, counting newlines as line separators.
        """
     lastCR = strg.rfind("\n", 0, loc)
     nextCR = strg.find("\n", loc)
     if nextCR > 0:
         return strg[lastCR+1:nextCR]
     else:
         return strg[lastCR+1:]
 
 def _defaultStartDebugAction( instring, loc, expr ):
     print ("Match " + _ustr(expr) + " at loc " + _ustr(loc) + "(%d,%d)" % ( lineno(loc,instring), col(loc,instring) ))
 
 def _defaultSuccessDebugAction( instring, startloc, endloc, expr, toks ):
     print ("Matched " + _ustr(expr) + " -> " + str(toks.asList()))
 
 def _defaultExceptionDebugAction( instring, loc, expr, exc ):
     print ("Exception raised:" + _ustr(exc))
 
 def nullDebugAction(*args):
     """'Do-nothing' debug action, to suppress debugging output during parsing."""
     pass
 
 class ParserElement(object):
     """Abstract base level parser element class."""
     DEFAULT_WHITE_CHARS = " \n\t\r"
 
     def setDefaultWhitespaceChars( chars ):
         """Overrides the default whitespace chars
         """
         ParserElement.DEFAULT_WHITE_CHARS = chars
     setDefaultWhitespaceChars = staticmethod(setDefaultWhitespaceChars)
 
     def __init__( self, savelist=False ):
         self.parseAction = list()
         self.failAction = None
         #~ self.name = "<unknown>"  # don't define self.name, let subclasses try/except upcall
         self.strRepr = None
         self.resultsName = None
         self.saveAsList = savelist
         self.skipWhitespace = True
         self.whiteChars = ParserElement.DEFAULT_WHITE_CHARS
         self.copyDefaultWhiteChars = True
         self.mayReturnEmpty = False # used when checking for left-recursion
         self.keepTabs = False
         self.ignoreExprs = list()
         self.debug = False
         self.streamlined = False
         self.mayIndexError = True # used to optimize exception handling for subclasses that don't advance parse index
         self.errmsg = ""
         self.modalResults = True # used to mark results names as modal (report only last) or cumulative (list all)
         self.debugActions = ( None, None, None ) #custom debug actions
         self.re = None
         self.callPreparse = True # used to avoid redundant calls to preParse
         self.callDuringTry = False
 
     def copy( self ):
         """Make a copy of this ParserElement.  Useful for defining different parse actions
            for the same parsing pattern, using copies of the original parse element."""
         cpy = copy.copy( self )
         cpy.parseAction = self.parseAction[:]
         cpy.ignoreExprs = self.ignoreExprs[:]
         if self.copyDefaultWhiteChars:
             cpy.whiteChars = ParserElement.DEFAULT_WHITE_CHARS
         return cpy
 
     def setName( self, name ):
         """Define name for this expression, for use in debugging."""
         self.name = name
         self.errmsg = "Expected " + self.name
         if hasattr(self,"exception"):
             self.exception.msg = self.errmsg
         return self
 
     def setResultsName( self, name, listAllMatches=False ):
         """Define name for referencing matching tokens as a nested attribute
            of the returned parse results.
            NOTE: this returns a *copy* of the original ParserElement object;
            this is so that the client can define a basic element, such as an
            integer, and reference it in multiple places with different names.
         """
         newself = self.copy()
         newself.resultsName = name
         newself.modalResults = not listAllMatches
         return newself
 
     def setBreak(self,breakFlag = True):
         """Method to invoke the Python pdb debugger when this element is
            about to be parsed. Set breakFlag to True to enable, False to
            disable.
         """
         if breakFlag:
             _parseMethod = self._parse
             def breaker(instring, loc, doActions=True, callPreParse=True):
                 import pdb
                 pdb.set_trace()
                 return _parseMethod( instring, loc, doActions, callPreParse )
             breaker._originalParseMethod = _parseMethod
             self._parse = breaker
         else:
             if hasattr(self._parse,"_originalParseMethod"):
                 self._parse = self._parse._originalParseMethod
         return self
 
     def _normalizeParseActionArgs( f ):
         """Internal method used to decorate parse actions that take fewer than 3 arguments,
            so that all parse actions can be called as f(s,l,t)."""
         STAR_ARGS = 4
 
         try:
             restore = None
             if isinstance(f,type):
                 restore = f
                 f = f.__init__
             if not _PY3K:
                 codeObj = f.func_code
             else:
                 codeObj = f.code
             if codeObj.co_flags & STAR_ARGS:
                 return f
             numargs = codeObj.co_argcount
             if not _PY3K:
                 if hasattr(f,"im_self"):
                     numargs -= 1
             else:
                 if hasattr(f,"__self__"):
                     numargs -= 1
             if restore:
                 f = restore
         except AttributeError:
             try:
                 if not _PY3K:
                     call_im_func_code = f.__call__.im_func.func_code
                 else:
                     call_im_func_code = f.__code__
 
                 # not a function, must be a callable object, get info from the
                 # im_func binding of its bound __call__ method
                 if call_im_func_code.co_flags & STAR_ARGS:
                     return f
                 numargs = call_im_func_code.co_argcount
                 if not _PY3K:
                     if hasattr(f.__call__,"im_self"):
                         numargs -= 1
                 else:
                     if hasattr(f.__call__,"__self__"):
                         numargs -= 0
             except AttributeError:
                 if not _PY3K:
                     call_func_code = f.__call__.func_code
                 else:
                     call_func_code = f.__call__.__code__
                 # not a bound method, get info directly from __call__ method
                 if call_func_code.co_flags & STAR_ARGS:
                     return f
                 numargs = call_func_code.co_argcount
                 if not _PY3K:
                     if hasattr(f.__call__,"im_self"):
                         numargs -= 1
                 else:
                     if hasattr(f.__call__,"__self__"):
                         numargs -= 1
 
 
         #~ print ("adding function %s with %d args" % (f.func_name,numargs))
         if numargs == 3:
             return f
         else:
             if numargs > 3:
                 def tmp(s,l,t):
                     return f(f.__call__.__self__, s,l,t)
             if numargs == 2:
                 def tmp(s,l,t):
                     return f(l,t)
             elif numargs == 1:
                 def tmp(s,l,t):
                     return f(t)
             else: #~ numargs == 0:
                 def tmp(s,l,t):
                     return f()
             try:
                 tmp.__name__ = f.__name__
             except (AttributeError,TypeError):
                 # no need for special handling if attribute doesnt exist
                 pass
             try:
                 tmp.__doc__ = f.__doc__
             except (AttributeError,TypeError):
                 # no need for special handling if attribute doesnt exist
                 pass
             try:
                 tmp.__dict__.update(f.__dict__)
             except (AttributeError,TypeError):
                 # no need for special handling if attribute doesnt exist
                 pass
             return tmp
     _normalizeParseActionArgs = staticmethod(_normalizeParseActionArgs)
 
     def setParseAction( self, *fns, **kwargs ):
         """Define action to perform when successfully matching parse element definition.
            Parse action fn is a callable method with 0-3 arguments, called as fn(s,loc,toks),
            fn(loc,toks), fn(toks), or just fn(), where:
             - s   = the original string being parsed (see note below)
             - loc = the location of the matching substring
             - toks = a list of the matched tokens, packaged as a ParseResults object
            If the functions in fns modify the tokens, they can return them as the return
            value from fn, and the modified list of tokens will replace the original.
            Otherwise, fn does not need to return any value.
 
            Note: the default parsing behavior is to expand tabs in the input string
            before starting the parsing process.  See L{I{parseString}<parseString>} for more information
            on parsing strings containing <TAB>s, and suggested methods to maintain a
            consistent view of the parsed string, the parse location, and line and column
            positions within the parsed string.
            """
         self.parseAction = list(map(self._normalizeParseActionArgs, list(fns)))
         self.callDuringTry = ("callDuringTry" in kwargs and kwargs["callDuringTry"])
         return self
 
     def addParseAction( self, *fns, **kwargs ):
         """Add parse action to expression's list of parse actions. See L{I{setParseAction}<setParseAction>}."""
         self.parseAction += list(map(self._normalizeParseActionArgs, list(fns)))
         self.callDuringTry = self.callDuringTry or ("callDuringTry" in kwargs and kwargs["callDuringTry"])
         return self
 
     def setFailAction( self, fn ):
         """Define action to perform if parsing fails at this expression.
            Fail acton fn is a callable function that takes the arguments
            fn(s,loc,expr,err) where:
             - s = string being parsed
             - loc = location where expression match was attempted and failed
             - expr = the parse expression that failed
             - err = the exception thrown
            The function returns no value.  It may throw ParseFatalException
            if it is desired to stop parsing immediately."""
         self.failAction = fn
         return self
 
     def _skipIgnorables( self, instring, loc ):
         exprsFound = True
         while exprsFound:
             exprsFound = False
             for e in self.ignoreExprs:
                 try:
                     while 1:
                         loc,dummy = e._parse( instring, loc )
                         exprsFound = True
                 except ParseException:
                     pass
         return loc
 
     def preParse( self, instring, loc ):
         if self.ignoreExprs:
             loc = self._skipIgnorables( instring, loc )
 
         if self.skipWhitespace:
             wt = self.whiteChars
             instrlen = len(instring)
             while loc < instrlen and instring[loc] in wt:
                 loc += 1
 
         return loc
 
     def parseImpl( self, instring, loc, doActions=True ):
         return loc, []
 
     def postParse( self, instring, loc, tokenlist ):
         return tokenlist
 
     #~ @profile
     def _parseNoCache( self, instring, loc, doActions=True, callPreParse=True ):
         debugging = ( self.debug ) #and doActions )
 
         if debugging or self.failAction:
             #~ print ("Match",self,"at loc",loc,"(%d,%d)" % ( lineno(loc,instring), col(loc,instring) ))
             if (self.debugActions[0] ):
                 self.debugActions[0]( instring, loc, self )
             if callPreParse and self.callPreparse:
                 preloc = self.preParse( instring, loc )
             else:
                 preloc = loc
             tokensStart = loc
             try:
                 try:
                     loc,tokens = self.parseImpl( instring, preloc, doActions )
                 except IndexError:
                     raise ParseException( instring, len(instring), self.errmsg, self )
             except ParseBaseException, err:
                 #~ print ("Exception raised:", err)
                 if self.debugActions[2]:
                     self.debugActions[2]( instring, tokensStart, self, err )
                 if self.failAction:
                     self.failAction( instring, tokensStart, self, err )
                 raise
         else:
             if callPreParse and self.callPreparse:
                 preloc = self.preParse( instring, loc )
             else:
                 preloc = loc
             tokensStart = loc
             if self.mayIndexError or loc >= len(instring):
                 try:
                     loc,tokens = self.parseImpl( instring, preloc, doActions )
                 except IndexError:
                     raise ParseException( instring, len(instring), self.errmsg, self )
             else:
                 loc,tokens = self.parseImpl( instring, preloc, doActions )
 
         tokens = self.postParse( instring, loc, tokens )
 
         retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults )
         if self.parseAction and (doActions or self.callDuringTry):
             if debugging:
                 try:
                     for fn in self.parseAction:
                         tokens = fn( instring, tokensStart, retTokens )
                         if tokens is not None:
                             retTokens = ParseResults( tokens,
                                                       self.resultsName,
                                                       asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
                                                       modal=self.modalResults )
                 except ParseBaseException, err:
                     #~ print "Exception raised in user parse action:", err
                     if (self.debugActions[2] ):
                         self.debugActions[2]( instring, tokensStart, self, err )
                     raise
             else:
                 for fn in self.parseAction:
                     tokens = fn( instring, tokensStart, retTokens )
                     if tokens is not None:
                         retTokens = ParseResults( tokens,
                                                   self.resultsName,
                                                   asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
                                                   modal=self.modalResults )
 
         if debugging:
             #~ print ("Matched",self,"->",retTokens.asList())
             if (self.debugActions[1] ):
                 self.debugActions[1]( instring, tokensStart, loc, self, retTokens )
 
         return loc, retTokens
 
     def tryParse( self, instring, loc ):
         try:
             return self._parse( instring, loc, doActions=False )[0]
         except ParseFatalException:
             raise ParseException( instring, loc, self.errmsg, self)
 
     # this method gets repeatedly called during backtracking with the same arguments -
     # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression
     def _parseCache( self, instring, loc, doActions=True, callPreParse=True ):
         lookup = (self,instring,loc,callPreParse,doActions)
         if lookup in ParserElement._exprArgCache:
             value = ParserElement._exprArgCache[ lookup ]
             if isinstance(value,Exception):
                 raise value
             return value
         else:
             try:
                 value = self._parseNoCache( instring, loc, doActions, callPreParse )
                 ParserElement._exprArgCache[ lookup ] = (value[0],value[1].copy())
                 return value
             except ParseBaseException, pe:
                 ParserElement._exprArgCache[ lookup ] = pe
                 raise
 
     _parse = _parseNoCache
 
     # argument cache for optimizing repeated calls when backtracking through recursive expressions
     _exprArgCache = {}
     def resetCache():
         ParserElement._exprArgCache.clear()
     resetCache = staticmethod(resetCache)
 
     _packratEnabled = False
     def enablePackrat():
         """Enables "packrat" parsing, which adds memoizing to the parsing logic.
            Repeated parse attempts at the same string location (which happens
            often in many complex grammars) can immediately return a cached value,
            instead of re-executing parsing/validating code.  Memoizing is done of
            both valid results and parsing exceptions.
 
            This speedup may break existing programs that use parse actions that
            have side-effects.  For this reason, packrat parsing is disabled when
            you first import pyparsing.  To activate the packrat feature, your
            program must call the class method ParserElement.enablePackrat().  If
            your program uses psyco to "compile as you go", you must call
            enablePackrat before calling psyco.full().  If you do not do this,
            Python will crash.  For best results, call enablePackrat() immediately
            after importing pyparsing.
         """
         if not ParserElement._packratEnabled:
             ParserElement._packratEnabled = True
             ParserElement._parse = ParserElement._parseCache
     enablePackrat = staticmethod(enablePackrat)
 
     def parseString( self, instring, parseAll=False ):
         """Execute the parse expression with the given string.
            This is the main interface to the client code, once the complete
            expression has been built.
 
            If you want the grammar to require that the entire input string be
            successfully parsed, then set parseAll to True (equivalent to ending
            the grammar with StringEnd()).
 
            Note: parseString implicitly calls expandtabs() on the input string,
            in order to report proper column numbers in parse actions.
            If the input string contains tabs and
            the grammar uses parse actions that use the loc argument to index into the
            string being parsed, you can ensure you have a consistent view of the input
            string by:
             - calling parseWithTabs on your grammar before calling parseString
               (see L{I{parseWithTabs}<parseWithTabs>})
             - define your parse action using the full (s,loc,toks) signature, and
               reference the input string using the parse action's s argument
             - explictly expand the tabs in your input string before calling
               parseString
         """
         ParserElement.resetCache()
         if not self.streamlined:
             self.streamline()
             #~ self.saveAsList = True
         for e in self.ignoreExprs:
             e.streamline()
         if not self.keepTabs:
             instring = instring.expandtabs()
         try:
             loc, tokens = self._parse( instring, 0 )
             if parseAll:
                 loc = self.preParse( instring, loc )
                 StringEnd()._parse( instring, loc )
         except ParseBaseException, exc:
             # catch and re-raise exception from here, clears out pyparsing internal stack trace
             raise exc
         else:
             return tokens
 
     def scanString( self, instring, maxMatches=_MAX_INT ):
         """Scan the input string for expression matches.  Each match will return the
            matching tokens, start location, and end location.  May be called with optional
            maxMatches argument, to clip scanning after 'n' matches are found.
 
            Note that the start and end locations are reported relative to the string
            being parsed.  See L{I{parseString}<parseString>} for more information on parsing
            strings with embedded tabs."""
         if not self.streamlined:
             self.streamline()
         for e in self.ignoreExprs:
             e.streamline()
 
         if not self.keepTabs:
             instring = _ustr(instring).expandtabs()
         instrlen = len(instring)
         loc = 0
         preparseFn = self.preParse
         parseFn = self._parse
         ParserElement.resetCache()
         matches = 0
         try:
             while loc <= instrlen and matches < maxMatches:
                 try:
                     preloc = preparseFn( instring, loc )
                     nextLoc,tokens = parseFn( instring, preloc, callPreParse=False )
                 except ParseException:
                     loc = preloc+1
                 else:
                     matches += 1
                     yield tokens, preloc, nextLoc
                     loc = nextLoc
         except ParseBaseException, pe:
             raise pe
 
     def transformString( self, instring ):
         """Extension to scanString, to modify matching text with modified tokens that may
            be returned from a parse action.  To use transformString, define a grammar and
            attach a parse action to it that modifies the returned token list.
            Invoking transformString() on a target string will then scan for matches,
            and replace the matched text patterns according to the logic in the parse
            action.  transformString() returns the resulting transformed string."""
         out = []
         lastE = 0
         # force preservation of <TAB>s, to minimize unwanted transformation of string, and to
         # keep string locs straight between transformString and scanString
         self.keepTabs = True
         try:
             for t,s,e in self.scanString( instring ):
                 out.append( instring[lastE:s] )
                 if t:
                     if isinstance(t,ParseResults):
                         out += t.asList()
                     elif isinstance(t,list):
                         out += t
                     else:
                         out.append(t)
                 lastE = e
             out.append(instring[lastE:])
             return "".join(map(_ustr,out))
         except ParseBaseException, pe:
             raise pe
 
     def searchString( self, instring, maxMatches=_MAX_INT ):
         """Another extension to scanString, simplifying the access to the tokens found
            to match the given parse expression.  May be called with optional
            maxMatches argument, to clip searching after 'n' matches are found.
         """
         try:
             return ParseResults([ t for t,s,e in self.scanString( instring, maxMatches ) ])
         except ParseBaseException, pe:
             raise pe
 
     def __add__(self, other ):
         """Implementation of + operator - returns And"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return And( [ self, other ] )
 
     def __radd__(self, other ):
         """Implementation of + operator when left operand is not a ParserElement"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return other + self
 
     def __sub__(self, other):
         """Implementation of - operator, returns And with error stop"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return And( [ self, And._ErrorStop(), other ] )
 
     def __rsub__(self, other ):
         """Implementation of - operator when left operand is not a ParserElement"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return other - self
 
     def __mul__(self,other):
         if isinstance(other,int):
             minElements, optElements = other,0
         elif isinstance(other,tuple):
             other = (other + (None, None))[:2]
             if other[0] is None:
                 other = (0, other[1])
             if isinstance(other[0],int) and other[1] is None:
                 if other[0] == 0:
                     return ZeroOrMore(self)
                 if other[0] == 1:
                     return OneOrMore(self)
                 else:
                     return self*other[0] + ZeroOrMore(self)
             elif isinstance(other[0],int) and isinstance(other[1],int):
                 minElements, optElements = other
                 optElements -= minElements
             else:
                 raise TypeError("cannot multiply 'ParserElement' and ('%s','%s') objects", type(other[0]),type(other[1]))
         else:
             raise TypeError("cannot multiply 'ParserElement' and '%s' objects", type(other))
 
         if minElements < 0:
             raise ValueError("cannot multiply ParserElement by negative value")
         if optElements < 0:
             raise ValueError("second tuple value must be greater or equal to first tuple value")
         if minElements == optElements == 0:
             raise ValueError("cannot multiply ParserElement by 0 or (0,0)")
 
         if (optElements):
             def makeOptionalList(n):
                 if n>1:
                     return Optional(self + makeOptionalList(n-1))
                 else:
                     return Optional(self)
             if minElements:
                 if minElements == 1:
                     ret = self + makeOptionalList(optElements)
                 else:
                     ret = And([self]*minElements) + makeOptionalList(optElements)
             else:
                 ret = makeOptionalList(optElements)
         else:
             if minElements == 1:
                 ret = self
             else:
                 ret = And([self]*minElements)
         return ret
 
     def __rmul__(self, other):
         return self.__mul__(other)
 
     def __or__(self, other ):
         """Implementation of | operator - returns MatchFirst"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return MatchFirst( [ self, other ] )
 
     def __ror__(self, other ):
         """Implementation of | operator when left operand is not a ParserElement"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return other | self
 
     def __xor__(self, other ):
         """Implementation of ^ operator - returns Or"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return Or( [ self, other ] )
 
     def __rxor__(self, other ):
         """Implementation of ^ operator when left operand is not a ParserElement"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return other ^ self
 
     def __and__(self, other ):
         """Implementation of & operator - returns Each"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return Each( [ self, other ] )
 
     def __rand__(self, other ):
         """Implementation of & operator when left operand is not a ParserElement"""
         if isinstance( other, basestring ):
             other = Literal( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
             return None
         return other & self
 
     def __invert__( self ):
         """Implementation of ~ operator - returns NotAny"""
         return NotAny( self )
 
     def __call__(self, name):
         """Shortcut for setResultsName, with listAllMatches=default::
              userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno")
            could be written as::
              userdata = Word(alphas)("name") + Word(nums+"-")("socsecno")
            """
         return self.setResultsName(name)
 
     def suppress( self ):
         """Suppresses the output of this ParserElement; useful to keep punctuation from
            cluttering up returned output.
         """
         return Suppress( self )
 
     def leaveWhitespace( self ):
         """Disables the skipping of whitespace before matching the characters in the
            ParserElement's defined pattern.  This is normally only used internally by
            the pyparsing module, but may be needed in some whitespace-sensitive grammars.
         """
         self.skipWhitespace = False
         return self
 
     def setWhitespaceChars( self, chars ):
         """Overrides the default whitespace chars
         """
         self.skipWhitespace = True
         self.whiteChars = chars
         self.copyDefaultWhiteChars = False
         return self
 
     def parseWithTabs( self ):
         """Overrides default behavior to expand <TAB>s to spaces before parsing the input string.
            Must be called before parseString when the input grammar contains elements that
            match <TAB> characters."""
         self.keepTabs = True
         return self
 
     def ignore( self, other ):
         """Define expression to be ignored (e.g., comments) while doing pattern
            matching; may be called repeatedly, to define multiple comment or other
            ignorable patterns.
         """
         if isinstance( other, Suppress ):
             if other not in self.ignoreExprs:
                 self.ignoreExprs.append( other )
         else:
             self.ignoreExprs.append( Suppress( other ) )
         return self
 
     def setDebugActions( self, startAction, successAction, exceptionAction ):
         """Enable display of debugging messages while doing pattern matching."""
         self.debugActions = (startAction or _defaultStartDebugAction,
                              successAction or _defaultSuccessDebugAction,
                              exceptionAction or _defaultExceptionDebugAction)
         self.debug = True
         return self
 
     def setDebug( self, flag=True ):
         """Enable display of debugging messages while doing pattern matching.
            Set flag to True to enable, False to disable."""
         if flag:
             self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction )
         else:
             self.debug = False
         return self
 
     def __str__( self ):
         return self.name
 
     def __repr__( self ):
         return _ustr(self)
 
     def streamline( self ):
         self.streamlined = True
         self.strRepr = None
         return self
 
     def checkRecursion( self, parseElementList ):
         pass
 
     def validate( self, validateTrace=[] ):
         """Check defined expressions for valid structure, check for infinite recursive definitions."""
         self.checkRecursion( [] )
 
     def parseFile( self, file_or_filename, parseAll=False ):
         """Execute the parse expression on the given file or filename.
            If a filename is specified (instead of a file object),
            the entire file is opened, read, and closed before parsing.
         """
         try:
             file_contents = file_or_filename.read()
         except AttributeError:
             f = open(file_or_filename, "rb")
             file_contents = f.read()
             f.close()
         try:
             return self.parseString(file_contents, parseAll)
         except ParseBaseException, exc:
             # catch and re-raise exception from here, clears out pyparsing internal stack trace
             raise exc
 
     def getException(self):
         return ParseException("",0,self.errmsg,self)
 
     def __getattr__(self,aname):
         if aname == "myException":
             self.myException = ret = self.getException();
             return ret;
         else:
             raise AttributeError("no such attribute " + aname)
 
     def __eq__(self,other):
         if isinstance(other, ParserElement):
             return self is other or self.__dict__ == other.__dict__
         elif isinstance(other, basestring):
             try:
                 self.parseString(_ustr(other), parseAll=True)
                 return True
             except ParseBaseException:
                 return False
         else:
             return super(ParserElement,self)==other
 
     def __ne__(self,other):
         return not (self == other)
 
     def __hash__(self):
         return hash(id(self))
 
     def __req__(self,other):
         return self == other
 
     def __rne__(self,other):
         return not (self == other)
 
 
 class Token(ParserElement):
     """Abstract ParserElement subclass, for defining atomic matching patterns."""
     def __init__( self ):
         super(Token,self).__init__( savelist=False )
         #self.myException = ParseException("",0,"",self)
 
     def setName(self, name):
         s = super(Token,self).setName(name)
         self.errmsg = "Expected " + self.name
         #s.myException.msg = self.errmsg
         return s
 
 
 class Empty(Token):
     """An empty token, will always match."""
     def __init__( self ):
         super(Empty,self).__init__()
         self.name = "Empty"
         self.mayReturnEmpty = True
         self.mayIndexError = False
 
 
 class NoMatch(Token):
     """A token that will never match."""
     def __init__( self ):
         super(NoMatch,self).__init__()
         self.name = "NoMatch"
         self.mayReturnEmpty = True
         self.mayIndexError = False
         self.errmsg = "Unmatchable token"
         #self.myException.msg = self.errmsg
 
     def parseImpl( self, instring, loc, doActions=True ):
         exc = self.myException
         exc.loc = loc
         exc.pstr = instring
         raise exc
 
 
 class Literal(Token):
     """Token to exactly match a specified string."""
     def __init__( self, matchString ):
         super(Literal,self).__init__()
         self.match = matchString
         self.matchLen = len(matchString)
         try:
             self.firstMatchChar = matchString[0]
         except IndexError:
             warnings.warn("null string passed to Literal; use Empty() instead",
                             SyntaxWarning, stacklevel=2)
             self.__class__ = Empty
         self.name = '"%s"' % _ustr(self.match)
         self.errmsg = "Expected " + self.name
         self.mayReturnEmpty = False
         #self.myException.msg = self.errmsg
         self.mayIndexError = False
 
     # Performance tuning: this routine gets called a *lot*
     # if this is a single character match string  and the first character matches,
     # short-circuit as quickly as possible, and avoid calling startswith
     #~ @profile
     def parseImpl( self, instring, loc, doActions=True ):
         if (instring[loc] == self.firstMatchChar and
             (self.matchLen==1 or instring.startswith(self.match,loc)) ):
             return loc+self.matchLen, self.match
         #~ raise ParseException( instring, loc, self.errmsg )
         exc = self.myException
         exc.loc = loc
         exc.pstr = instring
         raise exc
 _L = Literal
 
 class Keyword(Token):
     """Token to exactly match a specified string as a keyword, that is, it must be
        immediately followed by a non-keyword character.  Compare with Literal::
          Literal("if") will match the leading 'if' in 'ifAndOnlyIf'.
          Keyword("if") will not; it will only match the leading 'if in 'if x=1', or 'if(y==2)'
        Accepts two optional constructor arguments in addition to the keyword string:
        identChars is a string of characters that would be valid identifier characters,
        defaulting to all alphanumerics + "_" and "$"; caseless allows case-insensitive
        matching, default is False.
     """
     DEFAULT_KEYWORD_CHARS = alphanums+"_$"
 
     def __init__( self, matchString, identChars=DEFAULT_KEYWORD_CHARS, caseless=False ):
         super(Keyword,self).__init__()
         self.match = matchString
         self.matchLen = len(matchString)
         try:
             self.firstMatchChar = matchString[0]
         except IndexError:
             warnings.warn("null string passed to Keyword; use Empty() instead",
                             SyntaxWarning, stacklevel=2)
         self.name = '"%s"' % self.match
         self.errmsg = "Expected " + self.name
         self.mayReturnEmpty = False
         #self.myException.msg = self.errmsg
         self.mayIndexError = False
         self.caseless = caseless
         if caseless:
             self.caselessmatch = matchString.upper()
             identChars = identChars.upper()
         self.identChars = _str2dict(identChars)
 
     def parseImpl( self, instring, loc, doActions=True ):
         if self.caseless:
             if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
                  (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) and
                  (loc == 0 or instring[loc-1].upper() not in self.identChars) ):
                 return loc+self.matchLen, self.match
         else:
             if (instring[loc] == self.firstMatchChar and
                 (self.matchLen==1 or instring.startswith(self.match,loc)) and
                 (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen] not in self.identChars) and
                 (loc == 0 or instring[loc-1] not in self.identChars) ):
                 return loc+self.matchLen, self.match
         #~ raise ParseException( instring, loc, self.errmsg )
         exc = self.myException
         exc.loc = loc
         exc.pstr = instring
         raise exc
 
     def copy(self):
         c = super(Keyword,self).copy()
         c.identChars = Keyword.DEFAULT_KEYWORD_CHARS
         return c
 
     def setDefaultKeywordChars( chars ):
         """Overrides the default Keyword chars
         """
         Keyword.DEFAULT_KEYWORD_CHARS = chars
     setDefaultKeywordChars = staticmethod(setDefaultKeywordChars)
 
 class CaselessLiteral(Literal):
     """Token to match a specified string, ignoring case of letters.
        Note: the matched results will always be in the case of the given
        match string, NOT the case of the input text.
     """
     def __init__( self, matchString ):
         super(CaselessLiteral,self).__init__( matchString.upper() )
         # Preserve the defining literal.
         self.returnString = matchString
         self.name = "'%s'" % self.returnString
         self.errmsg = "Expected " + self.name
         #self.myException.msg = self.errmsg
 
     def parseImpl( self, instring, loc, doActions=True ):
         if instring[ loc:loc+self.matchLen ].upper() == self.match:
             return loc+self.matchLen, self.returnString
         #~ raise ParseException( instring, loc, self.errmsg )
         exc = self.myException
         exc.loc = loc
         exc.pstr = instring
         raise exc
 
 class CaselessKeyword(Keyword):
     def __init__( self, matchString, identChars=Keyword.DEFAULT_KEYWORD_CHARS ):
         super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True )
 
     def parseImpl( self, instring, loc, doActions=True ):
         if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
              (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ):
             return loc+self.matchLen, self.match
         #~ raise ParseException( instring, loc, self.errmsg )
         exc = self.myException
         exc.loc = loc
         exc.pstr = instring
         raise exc
 
 class Word(Token):
     """Token for matching words composed of allowed character sets.
        Defined with string containing all allowed initial characters,
        an optional string containing allowed body characters (if omitted,
        defaults to the initial character set), and an optional minimum,
        maximum, and/or exact length.  The default value for min is 1 (a
        minimum value < 1 is not valid); the default values for max and exact
        are 0, meaning no maximum or exact length restriction.
     """
     def __init__( self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False ):
         super(Word,self).__init__()
         self.initCharsOrig = initChars
         self.initChars = _str2dict(initChars)
         if bodyChars :
             self.bodyCharsOrig = bodyChars
             self.bodyChars = _str2dict(bodyChars)
         else:
             self.bodyCharsOrig = initChars
             self.bodyChars = _str2dict(initChars)
 
         self.maxSpecified = max > 0
 
         if min < 1:
             raise ValueError("cannot specify a minimum length < 1; use Optional(Word()) if zero-length word is permitted")
 
         self.minLen = min
 
         if max > 0:
             self.maxLen = max
         else:
             self.maxLen = _MAX_INT
 
         if exact > 0:
             self.maxLen = exact
             self.minLen = exact
 
         self.name = _ustr(self)
         self.errmsg = "Expected " + self.name
         #self.myException.msg = self.errmsg
         self.mayIndexError = False
         self.asKeyword = asKeyword
 
         if ' ' not in self.initCharsOrig+self.bodyCharsOrig and (min==1 and max==0 and exact==0):
             if self.bodyCharsOrig == self.initCharsOrig:
                 self.reString = "[%s]+" % _escapeRegexRangeChars(self.initCharsOrig)
             elif len(self.bodyCharsOrig) == 1:
                 self.reString = "%s[%s]*" % \
                                       (re.escape(self.initCharsOrig),
                                       _escapeRegexRangeChars(self.bodyCharsOrig),)
             else:
                 self.reString = "[%s][%s]*" % \
                                       (_escapeRegexRangeChars(self.initCharsOrig),
                                       _escapeRegexRangeChars(self.bodyCharsOrig),)
             if self.asKeyword:
                 self.reString = r"\b"+self.reString+r"\b"
             try:
                 self.re = re.compile( self.reString )
             except:
                 self.re = None
 
     def parseImpl( self, instring, loc, doActions=True ):
         if self.re:
             result = self.re.match(instring,loc)
             if not result:
                 exc = self.myException
                 exc.loc = loc
                 exc.pstr = instring
                 raise exc
 
             loc = result.end()
             return loc,result.group()
 
         if not(instring[ loc ] in self.initChars):
             #~ raise ParseException( instring, loc, self.errmsg )
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
         start = loc
         loc += 1
         instrlen = len(instring)
         bodychars = self.bodyChars
         maxloc = start + self.maxLen
         maxloc = min( maxloc, instrlen )
         while loc < maxloc and instring[loc] in bodychars:
             loc += 1
 
         throwException = False
         if loc - start < self.minLen:
             throwException = True
         if self.maxSpecified and loc < instrlen and instring[loc] in bodychars:
             throwException = True
         if self.asKeyword:
             if (start>0 and instring[start-1] in bodychars) or (loc<instrlen and instring[loc] in bodychars):
                 throwException = True
 
         if throwException:
             #~ raise ParseException( instring, loc, self.errmsg )
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
 
         return loc, instring[start:loc]
 
     def __str__( self ):
         try:
             return super(Word,self).__str__()
         except:
             pass
 
 
         if self.strRepr is None:
 
             def charsAsStr(s):
                 if len(s)>4:
                     return s[:4]+"..."
                 else:
                     return s
 
             if ( self.initCharsOrig != self.bodyCharsOrig ):
                 self.strRepr = "W:(%s,%s)" % ( charsAsStr(self.initCharsOrig), charsAsStr(self.bodyCharsOrig) )
             else:
                 self.strRepr = "W:(%s)" % charsAsStr(self.initCharsOrig)
 
         return self.strRepr
 
 
 class Regex(Token):
     """Token for matching strings that match a given regular expression.
        Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module.
     """
     def __init__( self, pattern, flags=0):
         """The parameters pattern and flags are passed to the re.compile() function as-is. See the Python re module for an explanation of the acceptable patterns and flags."""
         super(Regex,self).__init__()
 
         if len(pattern) == 0:
             warnings.warn("null string passed to Regex; use Empty() instead",
                     SyntaxWarning, stacklevel=2)
 
         self.pattern = pattern
         self.flags = flags
 
         try:
             self.re = re.compile(self.pattern, self.flags)
             self.reString = self.pattern
         except sre_constants.error:
             warnings.warn("invalid pattern (%s) passed to Regex" % pattern,
                 SyntaxWarning, stacklevel=2)
             raise
 
         self.name = _ustr(self)
         self.errmsg = "Expected " + self.name
         #self.myException.msg = self.errmsg
         self.mayIndexError = False
         self.mayReturnEmpty = True
 
     def parseImpl( self, instring, loc, doActions=True ):
         result = self.re.match(instring,loc)
         if not result:
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
 
         loc = result.end()
         d = result.groupdict()
         ret = ParseResults(result.group())
         if d:
             for k in d:
                 ret[k] = d[k]
         return loc,ret
 
     def __str__( self ):
         try:
             return super(Regex,self).__str__()
         except:
             pass
 
         if self.strRepr is None:
             self.strRepr = "Re:(%s)" % repr(self.pattern)
 
         return self.strRepr
 
 
 class QuotedString(Token):
     """Token for matching strings that are delimited by quoting characters.
     """
     def __init__( self, quoteChar, escChar=None, escQuote=None, multiline=False, unquoteResults=True, endQuoteChar=None):
         """
            Defined with the following parameters:
             - quoteChar - string of one or more characters defining the quote delimiting string
             - escChar - character to escape quotes, typically backslash (default=None)
             - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=None)
             - multiline - boolean indicating whether quotes can span multiple lines (default=False)
             - unquoteResults - boolean indicating whether the matched text should be unquoted (default=True)
             - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=None => same as quoteChar)
         """
         super(QuotedString,self).__init__()
 
         # remove white space from quote chars - wont work anyway
         quoteChar = quoteChar.strip()
         if len(quoteChar) == 0:
             warnings.warn("quoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
             raise SyntaxError()
 
         if endQuoteChar is None:
             endQuoteChar = quoteChar
         else:
             endQuoteChar = endQuoteChar.strip()
             if len(endQuoteChar) == 0:
                 warnings.warn("endQuoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
                 raise SyntaxError()
 
         self.quoteChar = quoteChar
         self.quoteCharLen = len(quoteChar)
         self.firstQuoteChar = quoteChar[0]
         self.endQuoteChar = endQuoteChar
         self.endQuoteCharLen = len(endQuoteChar)
         self.escChar = escChar
         self.escQuote = escQuote
         self.unquoteResults = unquoteResults
 
         if multiline:
             self.flags = re.MULTILINE | re.DOTALL
             self.pattern = r'%s(?:[^%s%s]' % \
                 ( re.escape(self.quoteChar),
                   _escapeRegexRangeChars(self.endQuoteChar[0]),
                   (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
         else:
             self.flags = 0
             self.pattern = r'%s(?:[^%s\n\r%s]' % \
                 ( re.escape(self.quoteChar),
                   _escapeRegexRangeChars(self.endQuoteChar[0]),
                   (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
         if len(self.endQuoteChar) > 1:
             self.pattern += (
                 '|(?:' + ')|(?:'.join(["%s[^%s]" % (re.escape(self.endQuoteChar[:i]),
                                                _escapeRegexRangeChars(self.endQuoteChar[i]))
                                     for i in range(len(self.endQuoteChar)-1,0,-1)]) + ')'
                 )
         if escQuote:
             self.pattern += (r'|(?:%s)' % re.escape(escQuote))
         if escChar:
             self.pattern += (r'|(?:%s.)' % re.escape(escChar))
             self.escCharReplacePattern = re.escape(self.escChar)+"(.)"
         self.pattern += (r')*%s' % re.escape(self.endQuoteChar))
 
         try:
             self.re = re.compile(self.pattern, self.flags)
             self.reString = self.pattern
         except sre_constants.error:
             warnings.warn("invalid pattern (%s) passed to Regex" % self.pattern,
                 SyntaxWarning, stacklevel=2)
             raise
 
         self.name = _ustr(self)
         self.errmsg = "Expected " + self.name
         #self.myException.msg = self.errmsg
         self.mayIndexError = False
         self.mayReturnEmpty = True
 
     def parseImpl( self, instring, loc, doActions=True ):
         result = instring[loc] == self.firstQuoteChar and self.re.match(instring,loc) or None
         if not result:
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
 
         loc = result.end()
         ret = result.group()
 
         if self.unquoteResults:
 
             # strip off quotes
             ret = ret[self.quoteCharLen:-self.endQuoteCharLen]
 
             if isinstance(ret,basestring):
                 # replace escaped characters
                 if self.escChar:
                     ret = re.sub(self.escCharReplacePattern,"\g<1>",ret)
 
                 # replace escaped quotes
                 if self.escQuote:
                     ret = ret.replace(self.escQuote, self.endQuoteChar)
 
         return loc, ret
 
     def __str__( self ):
         try:
             return super(QuotedString,self).__str__()
         except:
             pass
 
         if self.strRepr is None:
             self.strRepr = "quoted string, starting with %s ending with %s" % (self.quoteChar, self.endQuoteChar)
 
         return self.strRepr
 
 
 class CharsNotIn(Token):
     """Token for matching words composed of characters *not* in a given set.
        Defined with string containing all disallowed characters, and an optional
        minimum, maximum, and/or exact length.  The default value for min is 1 (a
        minimum value < 1 is not valid); the default values for max and exact
        are 0, meaning no maximum or exact length restriction.
     """
     def __init__( self, notChars, min=1, max=0, exact=0 ):
         super(CharsNotIn,self).__init__()
         self.skipWhitespace = False
         self.notChars = notChars
 
         if min < 1:
             raise ValueError("cannot specify a minimum length < 1; use Optional(CharsNotIn()) if zero-length char group is permitted")
 
         self.minLen = min
 
         if max > 0:
             self.maxLen = max
         else:
             self.maxLen = _MAX_INT
 
         if exact > 0:
             self.maxLen = exact
             self.minLen = exact
 
         self.name = _ustr(self)
         self.errmsg = "Expected " + self.name
         self.mayReturnEmpty = ( self.minLen == 0 )
         #self.myException.msg = self.errmsg
         self.mayIndexError = False
 
     def parseImpl( self, instring, loc, doActions=True ):
         if instring[loc] in self.notChars:
             #~ raise ParseException( instring, loc, self.errmsg )
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
 
         start = loc
         loc += 1
         notchars = self.notChars
         maxlen = min( start+self.maxLen, len(instring) )
         while loc < maxlen and \
               (instring[loc] not in notchars):
             loc += 1
 
         if loc - start < self.minLen:
             #~ raise ParseException( instring, loc, self.errmsg )
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
 
         return loc, instring[start:loc]
 
     def __str__( self ):
         try:
             return super(CharsNotIn, self).__str__()
         except:
             pass
 
         if self.strRepr is None:
             if len(self.notChars) > 4:
                 self.strRepr = "!W:(%s...)" % self.notChars[:4]
             else:
                 self.strRepr = "!W:(%s)" % self.notChars
 
         return self.strRepr
 
 class White(Token):
     """Special matching class for matching whitespace.  Normally, whitespace is ignored
        by pyparsing grammars.  This class is included when some whitespace structures
        are significant.  Define with a string containing the whitespace characters to be
        matched; default is " \\t\\r\\n".  Also takes optional min, max, and exact arguments,
        as defined for the Word class."""
     whiteStrs = {
         " " : "<SPC>",
         "\t": "<TAB>",
         "\n": "<LF>",
         "\r": "<CR>",
         "\f": "<FF>",
         }
     def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0):
         super(White,self).__init__()
         self.matchWhite = ws
         self.setWhitespaceChars( "".join([c for c in self.whiteChars if c not in self.matchWhite]) )
         #~ self.leaveWhitespace()
         self.name = ("".join([White.whiteStrs[c] for c in self.matchWhite]))
         self.mayReturnEmpty = True
         self.errmsg = "Expected " + self.name
         #self.myException.msg = self.errmsg
 
         self.minLen = min
 
         if max > 0:
             self.maxLen = max
         else:
             self.maxLen = _MAX_INT
 
         if exact > 0:
             self.maxLen = exact
             self.minLen = exact
 
     def parseImpl( self, instring, loc, doActions=True ):
         if not(instring[ loc ] in self.matchWhite):
             #~ raise ParseException( instring, loc, self.errmsg )
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
         start = loc
         loc += 1
         maxloc = start + self.maxLen
         maxloc = min( maxloc, len(instring) )
         while loc < maxloc and instring[loc] in self.matchWhite:
             loc += 1
 
         if loc - start < self.minLen:
             #~ raise ParseException( instring, loc, self.errmsg )
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
 
         return loc, instring[start:loc]
 
 
 class _PositionToken(Token):
     def __init__( self ):
         super(_PositionToken,self).__init__()
         self.name=self.__class__.__name__
         self.mayReturnEmpty = True
         self.mayIndexError = False
 
 class GoToColumn(_PositionToken):
     """Token to advance to a specific column of input text; useful for tabular report scraping."""
     def __init__( self, colno ):
         super(GoToColumn,self).__init__()
         self.col = colno
 
     def preParse( self, instring, loc ):
         if col(loc,instring) != self.col:
             instrlen = len(instring)
             if self.ignoreExprs:
                 loc = self._skipIgnorables( instring, loc )
             while loc < instrlen and instring[loc].isspace() and col( loc, instring ) != self.col :
                 loc += 1
         return loc
 
     def parseImpl( self, instring, loc, doActions=True ):
         thiscol = col( loc, instring )
         if thiscol > self.col:
             raise ParseException( instring, loc, "Text not in expected column", self )
         newloc = loc + self.col - thiscol
         ret = instring[ loc: newloc ]
         return newloc, ret
 
 class LineStart(_PositionToken):
     """Matches if current position is at the beginning of a line within the parse string"""
     def __init__( self ):
         super(LineStart,self).__init__()
         self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") )
         self.errmsg = "Expected start of line"
         #self.myException.msg = self.errmsg
 
     def preParse( self, instring, loc ):
         preloc = super(LineStart,self).preParse(instring,loc)
         if instring[preloc] == "\n":
             loc += 1
         return loc
 
     def parseImpl( self, instring, loc, doActions=True ):
         if not( loc==0 or
             (loc == self.preParse( instring, 0 )) or
             (instring[loc-1] == "\n") ): #col(loc, instring) != 1:
             #~ raise ParseException( instring, loc, "Expected start of line" )
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
         return loc, []
 
 class LineEnd(_PositionToken):
     """Matches if current position is at the end of a line within the parse string"""
     def __init__( self ):
         super(LineEnd,self).__init__()
         self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") )
         self.errmsg = "Expected end of line"
         #self.myException.msg = self.errmsg
 
     def parseImpl( self, instring, loc, doActions=True ):
         if loc<len(instring):
             if instring[loc] == "\n":
                 return loc+1, "\n"
             else:
                 #~ raise ParseException( instring, loc, "Expected end of line" )
                 exc = self.myException
                 exc.loc = loc
                 exc.pstr = instring
                 raise exc
         elif loc == len(instring):
             return loc+1, []
         else:
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
 
 class StringStart(_PositionToken):
     """Matches if current position is at the beginning of the parse string"""
     def __init__( self ):
         super(StringStart,self).__init__()
         self.errmsg = "Expected start of text"
         #self.myException.msg = self.errmsg
 
     def parseImpl( self, instring, loc, doActions=True ):
         if loc != 0:
             # see if entire string up to here is just whitespace and ignoreables
             if loc != self.preParse( instring, 0 ):
                 #~ raise ParseException( instring, loc, "Expected start of text" )
                 exc = self.myException
                 exc.loc = loc
                 exc.pstr = instring
                 raise exc
         return loc, []
 
 class StringEnd(_PositionToken):
     """Matches if current position is at the end of the parse string"""
     def __init__( self ):
         super(StringEnd,self).__init__()
         self.errmsg = "Expected end of text"
         #self.myException.msg = self.errmsg
 
     def parseImpl( self, instring, loc, doActions=True ):
         if loc < len(instring):
             #~ raise ParseException( instring, loc, "Expected end of text" )
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
         elif loc == len(instring):
             return loc+1, []
         elif loc > len(instring):
             return loc, []
         else:
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
 
 class WordStart(_PositionToken):
     """Matches if the current position is at the beginning of a Word, and
        is not preceded by any character in a given set of wordChars
        (default=printables). To emulate the \b behavior of regular expressions,
        use WordStart(alphanums). WordStart will also match at the beginning of
        the string being parsed, or at the beginning of a line.
     """
     def __init__(self, wordChars = printables):
         super(WordStart,self).__init__()
         self.wordChars = _str2dict(wordChars)
         self.errmsg = "Not at the start of a word"
 
     def parseImpl(self, instring, loc, doActions=True ):
         if loc != 0:
             if (instring[loc-1] in self.wordChars or
                 instring[loc] not in self.wordChars):
                 exc = self.myException
                 exc.loc = loc
                 exc.pstr = instring
                 raise exc
         return loc, []
 
 class WordEnd(_PositionToken):
     """Matches if the current position is at the end of a Word, and
        is not followed by any character in a given set of wordChars
        (default=printables). To emulate the \b behavior of regular expressions,
        use WordEnd(alphanums). WordEnd will also match at the end of
        the string being parsed, or at the end of a line.
     """
     def __init__(self, wordChars = printables):
         super(WordEnd,self).__init__()
         self.wordChars = _str2dict(wordChars)
         self.skipWhitespace = False
         self.errmsg = "Not at the end of a word"
 
     def parseImpl(self, instring, loc, doActions=True ):
         instrlen = len(instring)
         if instrlen>0 and loc<instrlen:
             if (instring[loc] in self.wordChars or
                 instring[loc-1] not in self.wordChars):
                 #~ raise ParseException( instring, loc, "Expected end of word" )
                 exc = self.myException
                 exc.loc = loc
                 exc.pstr = instring
                 raise exc
         return loc, []
 
 
 class ParseExpression(ParserElement):
     """Abstract subclass of ParserElement, for combining and post-processing parsed tokens."""
     def __init__( self, exprs, savelist = False ):
         super(ParseExpression,self).__init__(savelist)
         if isinstance( exprs, list ):
             self.exprs = exprs
         elif isinstance( exprs, basestring ):
             self.exprs = [ Literal( exprs ) ]
         else:
             try:
                 self.exprs = list( exprs )
             except TypeError:
                 self.exprs = [ exprs ]
         self.callPreparse = False
 
     def __getitem__( self, i ):
         return self.exprs[i]
 
     def append( self, other ):
         self.exprs.append( other )
         self.strRepr = None
         return self
 
     def leaveWhitespace( self ):
         """Extends leaveWhitespace defined in base class, and also invokes leaveWhitespace on
            all contained expressions."""
         self.skipWhitespace = False
         self.exprs = [ e.copy() for e in self.exprs ]
         for e in self.exprs:
             e.leaveWhitespace()
         return self
 
     def ignore( self, other ):
         if isinstance( other, Suppress ):
             if other not in self.ignoreExprs:
                 super( ParseExpression, self).ignore( other )
                 for e in self.exprs:
                     e.ignore( self.ignoreExprs[-1] )
         else:
             super( ParseExpression, self).ignore( other )
             for e in self.exprs:
                 e.ignore( self.ignoreExprs[-1] )
         return self
 
     def __str__( self ):
         try:
             return super(ParseExpression,self).__str__()
         except:
             pass
 
         if self.strRepr is None:
             self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.exprs) )
         return self.strRepr
 
     def streamline( self ):
         super(ParseExpression,self).streamline()
 
         for e in self.exprs:
             e.streamline()
 
         # collapse nested And's of the form And( And( And( a,b), c), d) to And( a,b,c,d )
         # but only if there are no parse actions or resultsNames on the nested And's
         # (likewise for Or's and MatchFirst's)
         if ( len(self.exprs) == 2 ):
             other = self.exprs[0]
             if ( isinstance( other, self.__class__ ) and
                   not(other.parseAction) and
                   other.resultsName is None and
                   not other.debug ):
                 self.exprs = other.exprs[:] + [ self.exprs[1] ]
                 self.strRepr = None
                 self.mayReturnEmpty |= other.mayReturnEmpty
                 self.mayIndexError  |= other.mayIndexError
 
             other = self.exprs[-1]
             if ( isinstance( other, self.__class__ ) and
                   not(other.parseAction) and
                   other.resultsName is None and
                   not other.debug ):
                 self.exprs = self.exprs[:-1] + other.exprs[:]
                 self.strRepr = None
                 self.mayReturnEmpty |= other.mayReturnEmpty
                 self.mayIndexError  |= other.mayIndexError
 
         return self
 
     def setResultsName( self, name, listAllMatches=False ):
         ret = super(ParseExpression,self).setResultsName(name,listAllMatches)
         return ret
 
     def validate( self, validateTrace=[] ):
         tmp = validateTrace[:]+[self]
         for e in self.exprs:
             e.validate(tmp)
         self.checkRecursion( [] )
 
 class And(ParseExpression):
     """Requires all given ParseExpressions to be found in the given order.
        Expressions may be separated by whitespace.
        May be constructed using the '+' operator.
     """
 
     class _ErrorStop(Empty):
         def __init__(self, *args, **kwargs):
             super(Empty,self).__init__(*args, **kwargs)
             self.leaveWhitespace()
 
     def __init__( self, exprs, savelist = True ):
         super(And,self).__init__(exprs, savelist)
         self.mayReturnEmpty = True
         for e in self.exprs:
             if not e.mayReturnEmpty:
                 self.mayReturnEmpty = False
                 break
         self.setWhitespaceChars( exprs[0].whiteChars )
         self.skipWhitespace = exprs[0].skipWhitespace
         self.callPreparse = True
 
     def parseImpl( self, instring, loc, doActions=True ):
         # pass False as last arg to _parse for first element, since we already
         # pre-parsed the string as part of our And pre-parsing
         loc, resultlist = self.exprs[0]._parse( instring, loc, doActions, callPreParse=False )
         errorStop = False
         for e in self.exprs[1:]:
             if isinstance(e, And._ErrorStop):
                 errorStop = True
                 continue
             if errorStop:
                 try:
                     loc, exprtokens = e._parse( instring, loc, doActions )
                 except ParseSyntaxException:
                     raise
                 except ParseBaseException, pe:
                     raise ParseSyntaxException(pe)
                 except IndexError, ie:
                     raise ParseSyntaxException( ParseException(instring, len(instring), self.errmsg, self) )
             else:
                 loc, exprtokens = e._parse( instring, loc, doActions )
             if exprtokens or exprtokens.keys():
                 resultlist += exprtokens
         return loc, resultlist
 
     def __iadd__(self, other ):
         if isinstance( other, basestring ):
             other = Literal( other )
         return self.append( other ) #And( [ self, other ] )
 
     def checkRecursion( self, parseElementList ):
         subRecCheckList = parseElementList[:] + [ self ]
         for e in self.exprs:
             e.checkRecursion( subRecCheckList )
             if not e.mayReturnEmpty:
                 break
 
     def __str__( self ):
         if hasattr(self,"name"):
             return self.name
 
         if self.strRepr is None:
             self.strRepr = "{" + " ".join( [ _ustr(e) for e in self.exprs ] ) + "}"
 
         return self.strRepr
 
 
 class Or(ParseExpression):
     """Requires that at least one ParseExpression is found.
        If two expressions match, the expression that matches the longest string will be used.
        May be constructed using the '^' operator.
     """
     def __init__( self, exprs, savelist = False ):
         super(Or,self).__init__(exprs, savelist)
         self.mayReturnEmpty = False
         for e in self.exprs:
             if e.mayReturnEmpty:
                 self.mayReturnEmpty = True
                 break
 
     def parseImpl( self, instring, loc, doActions=True ):
         maxExcLoc = -1
         maxMatchLoc = -1
         maxException = None
         for e in self.exprs:
             try:
                 loc2 = e.tryParse( instring, loc )
             except ParseException, err:
                 if err.loc > maxExcLoc:
                     maxException = err
                     maxExcLoc = err.loc
             except IndexError:
                 if len(instring) > maxExcLoc:
                     maxException = ParseException(instring,len(instring),e.errmsg,self)
                     maxExcLoc = len(instring)
             else:
                 if loc2 > maxMatchLoc:
                     maxMatchLoc = loc2
                     maxMatchExp = e
 
         if maxMatchLoc < 0:
             if maxException is not None:
                 raise maxException
             else:
                 raise ParseException(instring, loc, "no defined alternatives to match", self)
 
         return maxMatchExp._parse( instring, loc, doActions )
 
     def __ixor__(self, other ):
         if isinstance( other, basestring ):
             other = Literal( other )
         return self.append( other ) #Or( [ self, other ] )
 
     def __str__( self ):
         if hasattr(self,"name"):
             return self.name
 
         if self.strRepr is None:
             self.strRepr = "{" + " ^ ".join( [ _ustr(e) for e in self.exprs ] ) + "}"
 
         return self.strRepr
 
     def checkRecursion( self, parseElementList ):
         subRecCheckList = parseElementList[:] + [ self ]
         for e in self.exprs:
             e.checkRecursion( subRecCheckList )
 
 
 class MatchFirst(ParseExpression):
     """Requires that at least one ParseExpression is found.
        If two expressions match, the first one listed is the one that will match.
        May be constructed using the '|' operator.
     """
     def __init__( self, exprs, savelist = False ):
         super(MatchFirst,self).__init__(exprs, savelist)
         if exprs:
             self.mayReturnEmpty = False
             for e in self.exprs:
                 if e.mayReturnEmpty:
                     self.mayReturnEmpty = True
                     break
         else:
             self.mayReturnEmpty = True
 
     def parseImpl( self, instring, loc, doActions=True ):
         maxExcLoc = -1
         maxException = None
         for e in self.exprs:
             try:
                 ret = e._parse( instring, loc, doActions )
                 return ret
             except ParseException, err:
                 if err.loc > maxExcLoc:
                     maxException = err
                     maxExcLoc = err.loc
             except IndexError:
                 if len(instring) > maxExcLoc:
                     maxException = ParseException(instring,len(instring),e.errmsg,self)
                     maxExcLoc = len(instring)
 
         # only got here if no expression matched, raise exception for match that made it the furthest
         else:
             if maxException is not None:
                 raise maxException
             else:
                 raise ParseException(instring, loc, "no defined alternatives to match", self)
 
     def __ior__(self, other ):
         if isinstance( other, basestring ):
             other = Literal( other )
         return self.append( other ) #MatchFirst( [ self, other ] )
 
     def __str__( self ):
         if hasattr(self,"name"):
             return self.name
 
         if self.strRepr is None:
             self.strRepr = "{" + " | ".join( [ _ustr(e) for e in self.exprs ] ) + "}"
 
         return self.strRepr
 
     def checkRecursion( self, parseElementList ):
         subRecCheckList = parseElementList[:] + [ self ]
         for e in self.exprs:
             e.checkRecursion( subRecCheckList )
 
 
 class Each(ParseExpression):
     """Requires all given ParseExpressions to be found, but in any order.
        Expressions may be separated by whitespace.
        May be constructed using the '&' operator.
     """
     def __init__( self, exprs, savelist = True ):
         super(Each,self).__init__(exprs, savelist)
         self.mayReturnEmpty = True
         for e in self.exprs:
             if not e.mayReturnEmpty:
                 self.mayReturnEmpty = False
                 break
         self.skipWhitespace = True
         self.initExprGroups = True
 
     def parseImpl( self, instring, loc, doActions=True ):
         if self.initExprGroups:
             self.optionals = [ e.expr for e in self.exprs if isinstance(e,Optional) ]
             self.multioptionals = [ e.expr for e in self.exprs if isinstance(e,ZeroOrMore) ]
             self.multirequired = [ e.expr for e in self.exprs if isinstance(e,OneOrMore) ]
             self.required = [ e for e in self.exprs if not isinstance(e,(Optional,ZeroOrMore,OneOrMore)) ]
             self.required += self.multirequired
             self.initExprGroups = False
         tmpLoc = loc
         tmpReqd = self.required[:]
         tmpOpt  = self.optionals[:]
         matchOrder = []
 
         keepMatching = True
         while keepMatching:
             tmpExprs = tmpReqd + tmpOpt + self.multioptionals + self.multirequired
             failed = []
             for e in tmpExprs:
                 try:
                     tmpLoc = e.tryParse( instring, tmpLoc )
                 except ParseException:
                     failed.append(e)
                 else:
                     matchOrder.append(e)
                     if e in tmpReqd:
                         tmpReqd.remove(e)
                     elif e in tmpOpt:
                         tmpOpt.remove(e)
             if len(failed) == len(tmpExprs):
                 keepMatching = False
 
         if tmpReqd:
             missing = ", ".join( [ _ustr(e) for e in tmpReqd ] )
             raise ParseException(instring,loc,"Missing one or more required elements (%s)" % missing )
 
         # add any unmatched Optionals, in case they have default values defined
         matchOrder += list(e for e in self.exprs if isinstance(e,Optional) and e.expr in tmpOpt)
 
         resultlist = []
         for e in matchOrder:
             loc,results = e._parse(instring,loc,doActions)
             resultlist.append(results)
 
         finalResults = ParseResults([])
         for r in resultlist:
             dups = {}
             for k in r.keys():
                 if k in finalResults.keys():
                     tmp = ParseResults(finalResults[k])
                     tmp += ParseResults(r[k])
                     dups[k] = tmp
             finalResults += ParseResults(r)
             for k,v in dups.iteritems():
                 finalResults[k] = v
         return loc, finalResults
 
     def __str__( self ):
         if hasattr(self,"name"):
             return self.name
 
         if self.strRepr is None:
             self.strRepr = "{" + " & ".join( [ _ustr(e) for e in self.exprs ] ) + "}"
 
         return self.strRepr
 
     def checkRecursion( self, parseElementList ):
         subRecCheckList = parseElementList[:] + [ self ]
         for e in self.exprs:
             e.checkRecursion( subRecCheckList )
 
 
 class ParseElementEnhance(ParserElement):
     """Abstract subclass of ParserElement, for combining and post-processing parsed tokens."""
     def __init__( self, expr, savelist=False ):
         super(ParseElementEnhance,self).__init__(savelist)
         if isinstance( expr, basestring ):
             expr = Literal(expr)
         self.expr = expr
         self.strRepr = None
         if expr is not None:
             self.mayIndexError = expr.mayIndexError
             self.mayReturnEmpty = expr.mayReturnEmpty
             self.setWhitespaceChars( expr.whiteChars )
             self.skipWhitespace = expr.skipWhitespace
             self.saveAsList = expr.saveAsList
             self.callPreparse = expr.callPreparse
             self.ignoreExprs.extend(expr.ignoreExprs)
 
     def parseImpl( self, instring, loc, doActions=True ):
         if self.expr is not None:
             return self.expr._parse( instring, loc, doActions, callPreParse=False )
         else:
             raise ParseException("",loc,self.errmsg,self)
 
     def leaveWhitespace( self ):
         self.skipWhitespace = False
         self.expr = self.expr.copy()
         if self.expr is not None:
             self.expr.leaveWhitespace()
         return self
 
     def ignore( self, other ):
         if isinstance( other, Suppress ):
             if other not in self.ignoreExprs:
                 super( ParseElementEnhance, self).ignore( other )
                 if self.expr is not None:
                     self.expr.ignore( self.ignoreExprs[-1] )
         else:
             super( ParseElementEnhance, self).ignore( other )
             if self.expr is not None:
                 self.expr.ignore( self.ignoreExprs[-1] )
         return self
 
     def streamline( self ):
         super(ParseElementEnhance,self).streamline()
         if self.expr is not None:
             self.expr.streamline()
         return self
 
     def checkRecursion( self, parseElementList ):
         if self in parseElementList:
             raise RecursiveGrammarException( parseElementList+[self] )
         subRecCheckList = parseElementList[:] + [ self ]
         if self.expr is not None:
             self.expr.checkRecursion( subRecCheckList )
 
     def validate( self, validateTrace=[] ):
         tmp = validateTrace[:]+[self]
         if self.expr is not None:
             self.expr.validate(tmp)
         self.checkRecursion( [] )
 
     def __str__( self ):
         try:
             return super(ParseElementEnhance,self).__str__()
         except:
             pass
 
         if self.strRepr is None and self.expr is not None:
             self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.expr) )
         return self.strRepr
 
 
 class FollowedBy(ParseElementEnhance):
     """Lookahead matching of the given parse expression.  FollowedBy
     does *not* advance the parsing position within the input string, it only
     verifies that the specified parse expression matches at the current
     position.  FollowedBy always returns a null token list."""
     def __init__( self, expr ):
         super(FollowedBy,self).__init__(expr)
         self.mayReturnEmpty = True
 
     def parseImpl( self, instring, loc, doActions=True ):
         self.expr.tryParse( instring, loc )
         return loc, []
 
 
 class NotAny(ParseElementEnhance):
     """Lookahead to disallow matching with the given parse expression.  NotAny
     does *not* advance the parsing position within the input string, it only
     verifies that the specified parse expression does *not* match at the current
     position.  Also, NotAny does *not* skip over leading whitespace. NotAny
     always returns a null token list.  May be constructed using the '~' operator."""
     def __init__( self, expr ):
         super(NotAny,self).__init__(expr)
         #~ self.leaveWhitespace()
         self.skipWhitespace = False  # do NOT use self.leaveWhitespace(), don't want to propagate to exprs
         self.mayReturnEmpty = True
         self.errmsg = "Found unwanted token, "+_ustr(self.expr)
         #self.myException = ParseException("",0,self.errmsg,self)
 
     def parseImpl( self, instring, loc, doActions=True ):
         try:
             self.expr.tryParse( instring, loc )
         except (ParseException,IndexError):
             pass
         else:
             #~ raise ParseException(instring, loc, self.errmsg )
             exc = self.myException
             exc.loc = loc
             exc.pstr = instring
             raise exc
         return loc, []
 
     def __str__( self ):
         if hasattr(self,"name"):
             return self.name
 
         if self.strRepr is None:
             self.strRepr = "~{" + _ustr(self.expr) + "}"
 
         return self.strRepr
 
 
 class ZeroOrMore(ParseElementEnhance):
     """Optional repetition of zero or more of the given expression."""
     def __init__( self, expr ):
         super(ZeroOrMore,self).__init__(expr)
         self.mayReturnEmpty = True
 
     def parseImpl( self, instring, loc, doActions=True ):
         tokens = []
         try:
             loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
             hasIgnoreExprs = ( len(self.ignoreExprs) > 0 )
             while 1:
                 if hasIgnoreExprs:
                     preloc = self._skipIgnorables( instring, loc )
                 else:
                     preloc = loc
                 loc, tmptokens = self.expr._parse( instring, preloc, doActions )
                 if tmptokens or tmptokens.keys():
                     tokens += tmptokens
         except (ParseException,IndexError):
             pass
 
         return loc, tokens
 
     def __str__( self ):
         if hasattr(self,"name"):
             return self.name
 
         if self.strRepr is None:
             self.strRepr = "[" + _ustr(self.expr) + "]..."
 
         return self.strRepr
 
     def setResultsName( self, name, listAllMatches=False ):
         ret = super(ZeroOrMore,self).setResultsName(name,listAllMatches)
         ret.saveAsList = True
         return ret
 
 
 class OneOrMore(ParseElementEnhance):
     """Repetition of one or more of the given expression."""
     def parseImpl( self, instring, loc, doActions=True ):
         # must be at least one
         loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
         try:
             hasIgnoreExprs = ( len(self.ignoreExprs) > 0 )
             while 1:
                 if hasIgnoreExprs:
                     preloc = self._skipIgnorables( instring, loc )
                 else:
                     preloc = loc
                 loc, tmptokens = self.expr._parse( instring, preloc, doActions )
                 if tmptokens or tmptokens.keys():
                     tokens += tmptokens
         except (ParseException,IndexError):
             pass
 
         return loc, tokens
 
     def __str__( self ):
         if hasattr(self,"name"):
             return self.name
 
         if self.strRepr is None:
             self.strRepr = "{" + _ustr(self.expr) + "}..."
 
         return self.strRepr
 
     def setResultsName( self, name, listAllMatches=False ):
         ret = super(OneOrMore,self).setResultsName(name,listAllMatches)
         ret.saveAsList = True
         return ret
 
 class _NullToken(object):
     def __bool__(self):
         return False
     __nonzero__ = __bool__
     def __str__(self):
         return ""
 
 _optionalNotMatched = _NullToken()
 class Optional(ParseElementEnhance):
     """Optional matching of the given expression.
        A default return string can also be specified, if the optional expression
        is not found.
     """
     def __init__( self, exprs, default=_optionalNotMatched ):
         super(Optional,self).__init__( exprs, savelist=False )
         self.defaultValue = default
         self.mayReturnEmpty = True
 
     def parseImpl( self, instring, loc, doActions=True ):
         try:
             loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
         except (ParseException,IndexError):
             if self.defaultValue is not _optionalNotMatched:
                 if self.expr.resultsName:
                     tokens = ParseResults([ self.defaultValue ])
                     tokens[self.expr.resultsName] = self.defaultValue
                 else:
                     tokens = [ self.defaultValue ]
             else:
                 tokens = []
         return loc, tokens
 
     def __str__( self ):
         if hasattr(self,"name"):
             return self.name
 
         if self.strRepr is None:
             self.strRepr = "[" + _ustr(self.expr) + "]"
 
         return self.strRepr
 
 
 class SkipTo(ParseElementEnhance):
     """Token for skipping over all undefined text until the matched expression is found.
        If include is set to true, the matched expression is also parsed (the skipped text
        and matched expression are returned as a 2-element list).  The ignore
        argument is used to define grammars (typically quoted strings and comments) that
        might contain false matches.
     """
     def __init__( self, other, include=False, ignore=None, failOn=None ):
         super( SkipTo, self ).__init__( other )
         self.ignoreExpr = ignore
         self.mayReturnEmpty = True
         self.mayIndexError = False
         self.includeMatch = include
         self.asList = False
         if failOn is not None and isinstance(failOn, basestring):
             self.failOn = Literal(failOn)
         else:
             self.failOn = failOn
         self.errmsg = "No match found for "+_ustr(self.expr)
         #self.myException = ParseException("",0,self.errmsg,self)
 
     def parseImpl( self, instring, loc, doActions=True ):
         startLoc = loc
         instrlen = len(instring)
         expr = self.expr
         failParse = False
         while loc <= instrlen:
             try:
                 if self.failOn:
                     try:
                         self.failOn.tryParse(instring, loc)
                     except ParseBaseException:
                         pass
                     else:
                         failParse = True
                         raise ParseException(instring, loc, "Found expression " + str(self.failOn))
                     failParse = False
                 if self.ignoreExpr is not None:
                     while 1:
                         try:
                             loc = self.ignoreExpr.tryParse(instring,loc)
                             print "found ignoreExpr, advance to", loc
                         except ParseBaseException:
                             break
                 expr._parse( instring, loc, doActions=False, callPreParse=False )
                 skipText = instring[startLoc:loc]
                 if self.includeMatch:
                     loc,mat = expr._parse(instring,loc,doActions,callPreParse=False)
                     if mat:
                         skipRes = ParseResults( skipText )
                         skipRes += mat
                         return loc, [ skipRes ]
                     else:
                         return loc, [ skipText ]
                 else:
                     return loc, [ skipText ]
             except (ParseException,IndexError):
                 if failParse:
                     raise
                 else:
                     loc += 1
         exc = self.myException
         exc.loc = loc
         exc.pstr = instring
         raise exc
 
 class Forward(ParseElementEnhance):
     """Forward declaration of an expression to be defined later -
        used for recursive grammars, such as algebraic infix notation.
        When the expression is known, it is assigned to the Forward variable using the '<<' operator.
 
        Note: take care when assigning to Forward not to overlook precedence of operators.
        Specifically, '|' has a lower precedence than '<<', so that::
           fwdExpr << a | b | c
        will actually be evaluated as::
           (fwdExpr << a) | b | c
        thereby leaving b and c out as parseable alternatives.  It is recommended that you
        explicitly group the values inserted into the Forward::
           fwdExpr << (a | b | c)
     """
     def __init__( self, other=None ):
         super(Forward,self).__init__( other, savelist=False )
 
     def __lshift__( self, other ):
         if isinstance( other, basestring ):
             other = Literal(other)
         self.expr = other
         self.mayReturnEmpty = other.mayReturnEmpty
         self.strRepr = None
         self.mayIndexError = self.expr.mayIndexError
         self.mayReturnEmpty = self.expr.mayReturnEmpty
         self.setWhitespaceChars( self.expr.whiteChars )
         self.skipWhitespace = self.expr.skipWhitespace
         self.saveAsList = self.expr.saveAsList
         self.ignoreExprs.extend(self.expr.ignoreExprs)
         return None
 
     def leaveWhitespace( self ):
         self.skipWhitespace = False
         return self
 
     def streamline( self ):
         if not self.streamlined:
             self.streamlined = True
             if self.expr is not None:
                 self.expr.streamline()
         return self
 
     def validate( self, validateTrace=[] ):
         if self not in validateTrace:
             tmp = validateTrace[:]+[self]
             if self.expr is not None:
                 self.expr.validate(tmp)
         self.checkRecursion([])
 
     def __str__( self ):
         if hasattr(self,"name"):
             return self.name
 
         self._revertClass = self.__class__
         self.__class__ = _ForwardNoRecurse
         try:
             if self.expr is not None:
                 retString = _ustr(self.expr)
             else:
                 retString = "None"
         finally:
             self.__class__ = self._revertClass
         return self.__class__.__name__ + ": " + retString
 
     def copy(self):
         if self.expr is not None:
             return super(Forward,self).copy()
         else:
             ret = Forward()
             ret << self
             return ret
 
 class _ForwardNoRecurse(Forward):
     def __str__( self ):
         return "..."
 
 class TokenConverter(ParseElementEnhance):
     """Abstract subclass of ParseExpression, for converting parsed results."""
     def __init__( self, expr, savelist=False ):
         super(TokenConverter,self).__init__( expr )#, savelist )
         self.saveAsList = False
 
 class Upcase(TokenConverter):
     """Converter to upper case all matching tokens."""
     def __init__(self, *args):
         super(Upcase,self).__init__(*args)
         warnings.warn("Upcase class is deprecated, use upcaseTokens parse action instead",
                        DeprecationWarning,stacklevel=2)
 
     def postParse( self, instring, loc, tokenlist ):
         return list(map( string.upper, tokenlist ))
 
 
 class Combine(TokenConverter):
     """Converter to concatenate all matching tokens to a single string.
        By default, the matching patterns must also be contiguous in the input string;
        this can be disabled by specifying 'adjacent=False' in the constructor.
     """
     def __init__( self, expr, joinString="", adjacent=True ):
         super(Combine,self).__init__( expr )
         # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself
         if adjacent:
             self.leaveWhitespace()
         self.adjacent = adjacent
         self.skipWhitespace = True
         self.joinString = joinString
 
     def ignore( self, other ):
         if self.adjacent:
             ParserElement.ignore(self, other)
         else:
             super( Combine, self).ignore( other )
         return self
 
     def postParse( self, instring, loc, tokenlist ):
         retToks = tokenlist.copy()
         del retToks[:]
         retToks += ParseResults([ "".join(tokenlist._asStringList(self.joinString)) ], modal=self.modalResults)
 
         if self.resultsName and len(retToks.keys())>0:
             return [ retToks ]
         else:
             return retToks
 
 class Group(TokenConverter):
     """Converter to return the matched tokens as a list - useful for returning tokens of ZeroOrMore and OneOrMore expressions."""
     def __init__( self, expr ):
         super(Group,self).__init__( expr )
         self.saveAsList = True
 
     def postParse( self, instring, loc, tokenlist ):
         return [ tokenlist ]
 
 class Dict(TokenConverter):
     """Converter to return a repetitive expression as a list, but also as a dictionary.
        Each element can also be referenced using the first token in the expression as its key.
        Useful for tabular report scraping when the first column can be used as a item key.
     """
     def __init__( self, exprs ):
         super(Dict,self).__init__( exprs )
         self.saveAsList = True
 
     def postParse( self, instring, loc, tokenlist ):
         for i,tok in enumerate(tokenlist):
             if len(tok) == 0:
                 continue
             ikey = tok[0]
             if isinstance(ikey,int):
                 ikey = _ustr(tok[0]).strip()
             if len(tok)==1:
                 tokenlist[ikey] = _ParseResultsWithOffset("",i)
             elif len(tok)==2 and not isinstance(tok[1],ParseResults):
                 tokenlist[ikey] = _ParseResultsWithOffset(tok[1],i)
             else:
                 dictvalue = tok.copy() #ParseResults(i)
                 del dictvalue[0]
                 if len(dictvalue)!= 1 or (isinstance(dictvalue,ParseResults) and dictvalue.keys()):
                     tokenlist[ikey] = _ParseResultsWithOffset(dictvalue,i)
                 else:
                     tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0],i)
 
         if self.resultsName:
             return [ tokenlist ]
         else:
             return tokenlist
 
 
 class Suppress(TokenConverter):
     """Converter for ignoring the results of a parsed expression."""
     def postParse( self, instring, loc, tokenlist ):
         return []
 
     def suppress( self ):
         return self
 
 
 class OnlyOnce(object):
     """Wrapper for parse actions, to ensure they are only called once."""
     def __init__(self, methodCall):
         self.callable = ParserElement._normalizeParseActionArgs(methodCall)
         self.called = False
     def __call__(self,s,l,t):
         if not self.called:
             results = self.callable(s,l,t)
             self.called = True
             return results
         raise ParseException(s,l,"")
     def reset(self):
         self.called = False
 
 def traceParseAction(f):
     """Decorator for debugging parse actions."""
     f = ParserElement._normalizeParseActionArgs(f)
     def z(*paArgs):
         thisFunc = f.func_name
         s,l,t = paArgs[-3:]
         if len(paArgs)>3:
             thisFunc = paArgs[0].__class__.__name__ + '.' + thisFunc
         sys.stderr.write( ">>entering %s(line: '%s', %d, %s)\n" % (thisFunc,line(l,s),l,t) )
         try:
             ret = f(*paArgs)
         except Exception, exc:
             sys.stderr.write( "<<leaving %s (exception: %s)\n" % (thisFunc,exc) )
             raise
         sys.stderr.write( "<<leaving %s (ret: %s)\n" % (thisFunc,ret) )
         return ret
     try:
         z.__name__ = f.__name__
     except AttributeError:
         pass
     return z
 
 #
 # global helpers
 #
 def delimitedList( expr, delim=",", combine=False ):
     """Helper to define a delimited list of expressions - the delimiter defaults to ','.
        By default, the list elements and delimiters can have intervening whitespace, and
        comments, but this can be overridden by passing 'combine=True' in the constructor.
        If combine is set to True, the matching tokens are returned as a single token
        string, with the delimiters included; otherwise, the matching tokens are returned
        as a list of tokens, with the delimiters suppressed.
     """
     dlName = _ustr(expr)+" ["+_ustr(delim)+" "+_ustr(expr)+"]..."
     if combine:
         return Combine( expr + ZeroOrMore( delim + expr ) ).setName(dlName)
     else:
         return ( expr + ZeroOrMore( Suppress( delim ) + expr ) ).setName(dlName)
 
 def countedArray( expr ):
     """Helper to define a counted list of expressions.
        This helper defines a pattern of the form::
            integer expr expr expr...
        where the leading integer tells how many expr expressions follow.
        The matched tokens returns the array of expr tokens as a list - the leading count token is suppressed.
     """
     arrayExpr = Forward()
     def countFieldParseAction(s,l,t):
         n = int(t[0])
         arrayExpr << (n and Group(And([expr]*n)) or Group(empty))
         return []
     return ( Word(nums).setName("arrayLen").setParseAction(countFieldParseAction, callDuringTry=True) + arrayExpr )
 
 def _flatten(L):
     if type(L) is not list: return [L]
     if L == []: return L
     return _flatten(L[0]) + _flatten(L[1:])
 
 def matchPreviousLiteral(expr):
     """Helper to define an expression that is indirectly defined from
        the tokens matched in a previous expression, that is, it looks
        for a 'repeat' of a previous expression.  For example::
            first = Word(nums)
            second = matchPreviousLiteral(first)
            matchExpr = first + ":" + second
        will match "1:1", but not "1:2".  Because this matches a
        previous literal, will also match the leading "1:1" in "1:10".
        If this is not desired, use matchPreviousExpr.
        Do *not* use with packrat parsing enabled.
     """
     rep = Forward()
     def copyTokenToRepeater(s,l,t):
         if t:
             if len(t) == 1:
                 rep << t[0]
             else:
                 # flatten t tokens
                 tflat = _flatten(t.asList())
                 rep << And( [ Literal(tt) for tt in tflat ] )
         else:
             rep << Empty()
     expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
     return rep
 
 def matchPreviousExpr(expr):
     """Helper to define an expression that is indirectly defined from
        the tokens matched in a previous expression, that is, it looks
        for a 'repeat' of a previous expression.  For example::
            first = Word(nums)
            second = matchPreviousExpr(first)
            matchExpr = first + ":" + second
        will match "1:1", but not "1:2".  Because this matches by
        expressions, will *not* match the leading "1:1" in "1:10";
        the expressions are evaluated first, and then compared, so
        "1" is compared with "10".
        Do *not* use with packrat parsing enabled.
     """
     rep = Forward()
     e2 = expr.copy()
     rep << e2
     def copyTokenToRepeater(s,l,t):
         matchTokens = _flatten(t.asList())
         def mustMatchTheseTokens(s,l,t):
             theseTokens = _flatten(t.asList())
             if  theseTokens != matchTokens:
                 raise ParseException("",0,"")
         rep.setParseAction( mustMatchTheseTokens, callDuringTry=True )
     expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
     return rep
 
 def _escapeRegexRangeChars(s):
     #~  escape these chars: ^-]
     for c in r"\^-]":
         s = s.replace(c,_bslash+c)
     s = s.replace("\n",r"\n")
     s = s.replace("\t",r"\t")
     return _ustr(s)
 
 def oneOf( strs, caseless=False, useRegex=True ):
     """Helper to quickly define a set of alternative Literals, and makes sure to do
        longest-first testing when there is a conflict, regardless of the input order,
        but returns a MatchFirst for best performance.
 
        Parameters:
         - strs - a string of space-delimited literals, or a list of string literals
         - caseless - (default=False) - treat all literals as caseless
         - useRegex - (default=True) - as an optimization, will generate a Regex
           object; otherwise, will generate a MatchFirst object (if caseless=True, or
           if creating a Regex raises an exception)
     """
     if caseless:
         isequal = ( lambda a,b: a.upper() == b.upper() )
         masks = ( lambda a,b: b.upper().startswith(a.upper()) )
         parseElementClass = CaselessLiteral
     else:
         isequal = ( lambda a,b: a == b )
         masks = ( lambda a,b: b.startswith(a) )
         parseElementClass = Literal
 
     if isinstance(strs,(list,tuple)):
         symbols = list(strs[:])
     elif isinstance(strs,basestring):
         symbols = strs.split()
     else:
         warnings.warn("Invalid argument to oneOf, expected string or list",
                 SyntaxWarning, stacklevel=2)
 
     i = 0
     while i < len(symbols)-1:
         cur = symbols[i]
         for j,other in enumerate(symbols[i+1:]):
             if ( isequal(other, cur) ):
                 del symbols[i+j+1]
                 break
             elif ( masks(cur, other) ):
                 del symbols[i+j+1]
                 symbols.insert(i,other)
                 cur = other
                 break
         else:
             i += 1
 
     if not caseless and useRegex:
         #~ print (strs,"->", "|".join( [ _escapeRegexChars(sym) for sym in symbols] ))
         try:
             if len(symbols)==len("".join(symbols)):
                 return Regex( "[%s]" % "".join( [ _escapeRegexRangeChars(sym) for sym in symbols] ) )
             else:
                 return Regex( "|".join( [ re.escape(sym) for sym in symbols] ) )
         except:
             warnings.warn("Exception creating Regex for oneOf, building MatchFirst",
                     SyntaxWarning, stacklevel=2)
 
 
     # last resort, just use MatchFirst
     return MatchFirst( [ parseElementClass(sym) for sym in symbols ] )
 
 def dictOf( key, value ):
     """Helper to easily and clearly define a dictionary by specifying the respective patterns
        for the key and value.  Takes care of defining the Dict, ZeroOrMore, and Group tokens
        in the proper order.  The key pattern can include delimiting markers or punctuation,
        as long as they are suppressed, thereby leaving the significant key text.  The value
        pattern can include named results, so that the Dict results can include named token
        fields.
     """
     return Dict( ZeroOrMore( Group ( key + value ) ) )
 
 def originalTextFor(expr, asString=True):
     """Helper to return the original, untokenized text for a given expression.  Useful to
        restore the parsed fields of an HTML start tag into the raw tag text itself, or to
        revert separate tokens with intervening whitespace back to the original matching
        input text. Simpler to use than the parse action keepOriginalText, and does not
        require the inspect module to chase up the call stack.  By default, returns a
        string containing the original parsed text.
 
        If the optional asString argument is passed as False, then the return value is a
        ParseResults containing any results names that were originally matched, and a
        single token containing the original matched text from the input string.  So if
        the expression passed to originalTextFor contains expressions with defined
        results names, you must set asString to False if you want to preserve those
        results name values."""
     locMarker = Empty().setParseAction(lambda s,loc,t: loc)
     matchExpr = locMarker("_original_start") + expr + locMarker("_original_end")
     if asString:
         extractText = lambda s,l,t: s[t._original_start:t._original_end]
     else:
         def extractText(s,l,t):
             del t[:]
             t.insert(0, s[t._original_start:t._original_end])
             del t["_original_start"]
             del t["_original_end"]
     matchExpr.setParseAction(extractText)
     return matchExpr
 
 # convenience constants for positional expressions
 empty       = Empty().setName("empty")
 lineStart   = LineStart().setName("lineStart")
 lineEnd     = LineEnd().setName("lineEnd")
 stringStart = StringStart().setName("stringStart")
 stringEnd   = StringEnd().setName("stringEnd")
 
 _escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1])
 _printables_less_backslash = "".join([ c for c in printables if c not in  r"\]" ])
 _escapedHexChar = Combine( Suppress(_bslash + "0x") + Word(hexnums) ).setParseAction(lambda s,l,t:unichr(int(t[0],16)))
 _escapedOctChar = Combine( Suppress(_bslash) + Word("0","01234567") ).setParseAction(lambda s,l,t:unichr(int(t[0],8)))
 _singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | Word(_printables_less_backslash,exact=1)
 _charRange = Group(_singleChar + Suppress("-") + _singleChar)
 _reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]"
 
 _expanded = lambda p: (isinstance(p,ParseResults) and ''.join([ unichr(c) for c in range(ord(p[0]),ord(p[1])+1) ]) or p)
 
 def srange(s):
     r"""Helper to easily define string ranges for use in Word construction.  Borrows
        syntax from regexp '[]' string range definitions::
           srange("[0-9]")   -> "0123456789"
           srange("[a-z]")   -> "abcdefghijklmnopqrstuvwxyz"
           srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_"
        The input string must be enclosed in []'s, and the returned string is the expanded
        character set joined into a single string.
        The values enclosed in the []'s may be::
           a single character
           an escaped character with a leading backslash (such as \- or \])
           an escaped hex character with a leading '\0x' (\0x21, which is a '!' character)
           an escaped octal character with a leading '\0' (\041, which is a '!' character)
           a range of any of the above, separated by a dash ('a-z', etc.)
           any combination of the above ('aeiouy', 'a-zA-Z0-9_$', etc.)
     """
     try:
         return "".join([_expanded(part) for part in _reBracketExpr.parseString(s).body])
     except:
         return ""
 
 def matchOnlyAtCol(n):
     """Helper method for defining parse actions that require matching at a specific
        column in the input text.
     """
     def verifyCol(strg,locn,toks):
         if col(locn,strg) != n:
             raise ParseException(strg,locn,"matched token not at column %d" % n)
     return verifyCol
 
 def replaceWith(replStr):
     """Helper method for common parse actions that simply return a literal value.  Especially
        useful when used with transformString().
     """
     def _replFunc(*args):
         return [replStr]
     return _replFunc
 
 def removeQuotes(s,l,t):
     """Helper parse action for removing quotation marks from parsed quoted strings.
        To use, add this parse action to quoted string using::
          quotedString.setParseAction( removeQuotes )
     """
     return t[0][1:-1]
 
 def upcaseTokens(s,l,t):
     """Helper parse action to convert tokens to upper case."""
     return [ tt.upper() for tt in map(_ustr,t) ]
 
 def downcaseTokens(s,l,t):
     """Helper parse action to convert tokens to lower case."""
     return [ tt.lower() for tt in map(_ustr,t) ]
 
 def keepOriginalText(s,startLoc,t):
     """Helper parse action to preserve original parsed text,
        overriding any nested parse actions."""
     try:
         endloc = getTokensEndLoc()
     except ParseException:
         raise ParseFatalException("incorrect usage of keepOriginalText - may only be called as a parse action")
     del t[:]
     t += ParseResults(s[startLoc:endloc])
     return t
 
 def getTokensEndLoc():
     """Method to be called from within a parse action to determine the end
        location of the parsed tokens."""
     import inspect
     fstack = inspect.stack()
     try:
         # search up the stack (through intervening argument normalizers) for correct calling routine
         for f in fstack[2:]:
             if f[3] == "_parseNoCache":
                 endloc = f[0].f_locals["loc"]
                 return endloc
         else:
             raise ParseFatalException("incorrect usage of getTokensEndLoc - may only be called from within a parse action")
     finally:
         del fstack
 
 def _makeTags(tagStr, xml):
     """Internal helper to construct opening and closing tag expressions, given a tag name"""
     if isinstance(tagStr,basestring):
         resname = tagStr
         tagStr = Keyword(tagStr, caseless=not xml)
     else:
         resname = tagStr.name
 
     tagAttrName = Word(alphas,alphanums+"_-:")
     if (xml):
         tagAttrValue = dblQuotedString.copy().setParseAction( removeQuotes )
         openTag = Suppress("<") + tagStr + \
                 Dict(ZeroOrMore(Group( tagAttrName + Suppress("=") + tagAttrValue ))) + \
                 Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
     else:
         printablesLessRAbrack = "".join( [ c for c in printables if c not in ">" ] )
         tagAttrValue = quotedString.copy().setParseAction( removeQuotes ) | Word(printablesLessRAbrack)
         openTag = Suppress("<") + tagStr + \
                 Dict(ZeroOrMore(Group( tagAttrName.setParseAction(downcaseTokens) + \
                 Optional( Suppress("=") + tagAttrValue ) ))) + \
                 Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
     closeTag = Combine(_L("</") + tagStr + ">")
 
     openTag = openTag.setResultsName("start"+"".join(resname.replace(":"," ").title().split())).setName("<%s>" % tagStr)
     closeTag = closeTag.setResultsName("end"+"".join(resname.replace(":"," ").title().split())).setName("</%s>" % tagStr)
 
     return openTag, closeTag
 
 def makeHTMLTags(tagStr):
     """Helper to construct opening and closing tag expressions for HTML, given a tag name"""
     return _makeTags( tagStr, False )
 
 def makeXMLTags(tagStr):
     """Helper to construct opening and closing tag expressions for XML, given a tag name"""
     return _makeTags( tagStr, True )
 
 def withAttribute(*args,**attrDict):
     """Helper to create a validating parse action to be used with start tags created
        with makeXMLTags or makeHTMLTags. Use withAttribute to qualify a starting tag
        with a required attribute value, to avoid false matches on common tags such as
        <TD> or <DIV>.
 
        Call withAttribute with a series of attribute names and values. Specify the list
        of filter attributes names and values as:
         - keyword arguments, as in (class="Customer",align="right"), or
         - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") )
        For attribute names with a namespace prefix, you must use the second form.  Attribute
        names are matched insensitive to upper/lower case.
 
        To verify that the attribute exists, but without specifying a value, pass
        withAttribute.ANY_VALUE as the value.
        """
     if args:
         attrs = args[:]
     else:
         attrs = attrDict.iteritems()
     attrs = [(k,v) for k,v in attrs]
     def pa(s,l,tokens):
         for attrName,attrValue in attrs:
             if attrName not in tokens:
                 raise ParseException(s,l,"no matching attribute " + attrName)
             if attrValue != withAttribute.ANY_VALUE and tokens[attrName] != attrValue:
                 raise ParseException(s,l,"attribute '%s' has value '%s', must be '%s'" %
                                             (attrName, tokens[attrName], attrValue))
     return pa
 withAttribute.ANY_VALUE = object()
 
 opAssoc = _Constants()
 opAssoc.LEFT = object()
 opAssoc.RIGHT = object()
 
 def operatorPrecedence( baseExpr, opList ):
     """Helper method for constructing grammars of expressions made up of
        operators working in a precedence hierarchy.  Operators may be unary or
        binary, left- or right-associative.  Parse actions can also be attached
        to operator expressions.
 
        Parameters:
         - baseExpr - expression representing the most basic element for the nested
         - opList - list of tuples, one for each operator precedence level in the
           expression grammar; each tuple is of the form
           (opExpr, numTerms, rightLeftAssoc, parseAction), where:
            - opExpr is the pyparsing expression for the operator;
               may also be a string, which will be converted to a Literal;
               if numTerms is 3, opExpr is a tuple of two expressions, for the
               two operators separating the 3 terms
            - numTerms is the number of terms for this operator (must
               be 1, 2, or 3)
            - rightLeftAssoc is the indicator whether the operator is
               right or left associative, using the pyparsing-defined
               constants opAssoc.RIGHT and opAssoc.LEFT.
            - parseAction is the parse action to be associated with
               expressions matching this operator expression (the
               parse action tuple member may be omitted)
     """
     ret = Forward()
     lastExpr = baseExpr | ( Suppress('(') + ret + Suppress(')') )
     for i,operDef in enumerate(opList):
         opExpr,arity,rightLeftAssoc,pa = (operDef + (None,))[:4]
         if arity == 3:
             if opExpr is None or len(opExpr) != 2:
                 raise ValueError("if numterms=3, opExpr must be a tuple or list of two expressions")
             opExpr1, opExpr2 = opExpr
         thisExpr = Forward()#.setName("expr%d" % i)
         if rightLeftAssoc == opAssoc.LEFT:
             if arity == 1:
                 matchExpr = FollowedBy(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) )
             elif arity == 2:
                 if opExpr is not None:
                     matchExpr = FollowedBy(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) )
                 else:
                     matchExpr = FollowedBy(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) )
             elif arity == 3:
                 matchExpr = FollowedBy(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \
                             Group( lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr )
             else:
                 raise ValueError("operator must be unary (1), binary (2), or ternary (3)")
         elif rightLeftAssoc == opAssoc.RIGHT:
             if arity == 1:
                 # try to avoid LR with this extra test
                 if not isinstance(opExpr, Optional):
                     opExpr = Optional(opExpr)
                 matchExpr = FollowedBy(opExpr.expr + thisExpr) + Group( opExpr + thisExpr )
             elif arity == 2:
                 if opExpr is not None:
                     matchExpr = FollowedBy(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) )
                 else:
                     matchExpr = FollowedBy(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) )
             elif arity == 3:
                 matchExpr = FollowedBy(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \
                             Group( lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr )
             else:
                 raise ValueError("operator must be unary (1), binary (2), or ternary (3)")
         else:
             raise ValueError("operator must indicate right or left associativity")
         if pa:
             matchExpr.setParseAction( pa )
         thisExpr << ( matchExpr | lastExpr )
         lastExpr = thisExpr
     ret << lastExpr
     return ret
 
 dblQuotedString = Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\x[0-9a-fA-F]+)|(?:\\.))*"').setName("string enclosed in double quotes")
 sglQuotedString = Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\x[0-9a-fA-F]+)|(?:\\.))*'").setName("string enclosed in single quotes")
 quotedString = Regex(r'''(?:"(?:[^"\n\r\\]|(?:"")|(?:\\x[0-9a-fA-F]+)|(?:\\.))*")|(?:'(?:[^'\n\r\\]|(?:'')|(?:\\x[0-9a-fA-F]+)|(?:\\.))*')''').setName("quotedString using single or double quotes")
 unicodeString = Combine(_L('u') + quotedString.copy())
 
 def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString):
     """Helper method for defining nested lists enclosed in opening and closing
        delimiters ("(" and ")" are the default).
 
        Parameters:
         - opener - opening character for a nested list (default="("); can also be a pyparsing expression
         - closer - closing character for a nested list (default=")"); can also be a pyparsing expression
         - content - expression for items within the nested lists (default=None)
         - ignoreExpr - expression for ignoring opening and closing delimiters (default=quotedString)
 
        If an expression is not provided for the content argument, the nested
        expression will capture all whitespace-delimited content between delimiters
        as a list of separate values.
 
        Use the ignoreExpr argument to define expressions that may contain
        opening or closing characters that should not be treated as opening
        or closing characters for nesting, such as quotedString or a comment
        expression.  Specify multiple expressions using an Or or MatchFirst.
        The default is quotedString, but if no expressions are to be ignored,
        then pass None for this argument.
     """
     if opener == closer:
         raise ValueError("opening and closing strings cannot be the same")
     if content is None:
         if isinstance(opener,basestring) and isinstance(closer,basestring):
             if len(opener) == 1 and len(closer)==1:
                 if ignoreExpr is not None:
                     content = (Combine(OneOrMore(~ignoreExpr +
                                     CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS,exact=1))
                                 ).setParseAction(lambda t:t[0].strip()))
                 else:
                     content = (empty+CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS
                                 ).setParseAction(lambda t:t[0].strip()))
             else:
                 if ignoreExpr is not None:
                     content = (Combine(OneOrMore(~ignoreExpr +
                                     ~Literal(opener) + ~Literal(closer) +
                                     CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1))
                                 ).setParseAction(lambda t:t[0].strip()))
                 else:
                     content = (Combine(OneOrMore(~Literal(opener) + ~Literal(closer) +
                                     CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1))
                                 ).setParseAction(lambda t:t[0].strip()))
         else:
             raise ValueError("opening and closing arguments must be strings if no content expression is given")
     ret = Forward()
     if ignoreExpr is not None:
         ret << Group( Suppress(opener) + ZeroOrMore( ignoreExpr | ret | content ) + Suppress(closer) )
     else:
         ret << Group( Suppress(opener) + ZeroOrMore( ret | content )  + Suppress(closer) )
     return ret
 
 def indentedBlock(blockStatementExpr, indentStack, indent=True):
     """Helper method for defining space-delimited indentation blocks, such as
        those used to define block statements in Python source code.
 
        Parameters:
         - blockStatementExpr - expression defining syntax of statement that
             is repeated within the indented block
         - indentStack - list created by caller to manage indentation stack
             (multiple statementWithIndentedBlock expressions within a single grammar
             should share a common indentStack)
         - indent - boolean indicating whether block must be indented beyond the
             the current level; set to False for block of left-most statements
             (default=True)
 
        A valid block must contain at least one blockStatement.
     """
     def checkPeerIndent(s,l,t):
         if l >= len(s): return
         curCol = col(l,s)
         if curCol != indentStack[-1]:
             if curCol > indentStack[-1]:
                 raise ParseFatalException(s,l,"illegal nesting")
             raise ParseException(s,l,"not a peer entry")
 
     def checkSubIndent(s,l,t):
         curCol = col(l,s)
         if curCol > indentStack[-1]:
             indentStack.append( curCol )
         else:
             raise ParseException(s,l,"not a subentry")
 
     def checkUnindent(s,l,t):
         if l >= len(s): return
         curCol = col(l,s)
         if not(indentStack and curCol < indentStack[-1] and curCol <= indentStack[-2]):
             raise ParseException(s,l,"not an unindent")
         indentStack.pop()
 
     NL = OneOrMore(LineEnd().setWhitespaceChars("\t ").suppress())
     INDENT = Empty() + Empty().setParseAction(checkSubIndent)
     PEER   = Empty().setParseAction(checkPeerIndent)
     UNDENT = Empty().setParseAction(checkUnindent)
     if indent:
         smExpr = Group( Optional(NL) +
             FollowedBy(blockStatementExpr) +
             INDENT + (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) + UNDENT)
     else:
         smExpr = Group( Optional(NL) +
             (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) )
     blockStatementExpr.ignore(_bslash + LineEnd())
     return smExpr
 
 alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]")
 punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]")
 
 anyOpenTag,anyCloseTag = makeHTMLTags(Word(alphas,alphanums+"_:"))
 commonHTMLEntity = Combine(_L("&") + oneOf("gt lt amp nbsp quot").setResultsName("entity") +";").streamline()
 _htmlEntityMap = dict(zip("gt lt amp nbsp quot".split(),'><& "'))
 replaceHTMLEntity = lambda t : t.entity in _htmlEntityMap and _htmlEntityMap[t.entity] or None
 
 # it's easy to get these comment structures wrong - they're very common, so may as well make them available
 cStyleComment = Regex(r"/\*(?:[^*]*\*+)+?/").setName("C style comment")
 
 htmlComment = Regex(r"<!--[\s\S]*?-->")
 restOfLine = Regex(r".*").leaveWhitespace()
 dblSlashComment = Regex(r"\/\/(\\\n|.)*").setName("// comment")
 cppStyleComment = Regex(r"/(?:\*(?:[^*]*\*+)+?/|/[^\n]*(?:\n[^\n]*)*?(?:(?<!\\)|\Z))").setName("C++ style comment")
 
 javaStyleComment = cppStyleComment
 pythonStyleComment = Regex(r"#.*").setName("Python style comment")
 _noncomma = "".join( [ c for c in printables if c != "," ] )
 _commasepitem = Combine(OneOrMore(Word(_noncomma) +
                                   Optional( Word(" \t") +
                                             ~Literal(",") + ~LineEnd() ) ) ).streamline().setName("commaItem")
 commaSeparatedList = delimitedList( Optional( quotedString | _commasepitem, default="") ).setName("commaSeparatedList")
 
 
 if __name__ == "__main__":
 
     def test( teststring ):
         try:
             tokens = simpleSQL.parseString( teststring )
             tokenlist = tokens.asList()
             print (teststring + "->"   + str(tokenlist))
             print ("tokens = "         + str(tokens))
             print ("tokens.columns = " + str(tokens.columns))
             print ("tokens.tables = "  + str(tokens.tables))
             print (tokens.asXML("SQL",True))
         except ParseBaseException,err:
             print (teststring + "->")
             print (err.line)
             print (" "*(err.column-1) + "^")
             print (err)
         print()
 
     selectToken    = CaselessLiteral( "select" )
     fromToken      = CaselessLiteral( "from" )
 
     ident          = Word( alphas, alphanums + "_$" )
     columnName     = delimitedList( ident, ".", combine=True ).setParseAction( upcaseTokens )
     columnNameList = Group( delimitedList( columnName ) )#.setName("columns")
     tableName      = delimitedList( ident, ".", combine=True ).setParseAction( upcaseTokens )
     tableNameList  = Group( delimitedList( tableName ) )#.setName("tables")
     simpleSQL      = ( selectToken + \
                      ( '*' | columnNameList ).setResultsName( "columns" ) + \
                      fromToken + \
                      tableNameList.setResultsName( "tables" ) )
 
     test( "SELECT * from XYZZY, ABC" )
     test( "select * from SYS.XYZZY" )
     test( "Select A from Sys.dual" )
     test( "Select AA,BB,CC from Sys.dual" )
     test( "Select A, B, C from Sys.dual" )
     test( "Select A, B, C from Sys.dual" )
     test( "Xelect A, B, C from Sys.dual" )
     test( "Select A, B, C frox Sys.dual" )
     test( "Select" )
     test( "Select ^^^ frox Sys.dual" )
     test( "Select A, B, C from Sys.dual, Table2   " )

IPython/frontend/terminal/interactiveshell.py

@@ -1,668 +1,669 b''
 # -*- coding: utf-8 -*-
 """Subclass of InteractiveShell for terminal based frontends."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001 Janko Hauser <[e-mail]>
 #  Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import __builtin__
 import bdb
 import os
 import re
 import sys
 import textwrap
 
 try:
     from contextlib import nested
 except:
     from IPython.utils.nested_context import nested
 
 from IPython.core.error import TryNext, UsageError
 from IPython.core.usage import interactive_usage, default_banner
 from IPython.core.interactiveshell import InteractiveShell, InteractiveShellABC
 from IPython.core.pylabtools import pylab_activate
 from IPython.testing.skipdoctest import skip_doctest
+from IPython.utils.encoding import get_stream_enc
 from IPython.utils import py3compat
 from IPython.utils.terminal import toggle_set_term_title, set_term_title
 from IPython.utils.process import abbrev_cwd
 from IPython.utils.warn import warn, error
 from IPython.utils.text import num_ini_spaces, SList
 from IPython.utils.traitlets import Integer, CBool, Unicode
 
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
 def get_default_editor():
     try:
         ed = os.environ['EDITOR']
     except KeyError:
         if os.name == 'posix':
             ed = 'vi'  # the only one guaranteed to be there!
         else:
             ed = 'notepad' # same in Windows!
     return ed
 
 
 def get_pasted_lines(sentinel, l_input=py3compat.input):
     """ Yield pasted lines until the user enters the given sentinel value.
     """
     print "Pasting code; enter '%s' alone on the line to stop or use Ctrl-D." \
           % sentinel
     while True:
         try:
             l = l_input(':')
             if l == sentinel:
                 return
             else:
                 yield l
         except EOFError:
             print '<EOF>'
             return
 
 
 def strip_email_quotes(raw_lines):
     """ Strip email quotation marks at the beginning of each line.
 
     We don't do any more input transofrmations here because the main shell's
     prefiltering handles other cases.
     """
     lines = [re.sub(r'^\s*(\s?>)+', '', l) for l in raw_lines]
     return '\n'.join(lines) + '\n'
 
 
 # These two functions are needed by the %paste/%cpaste magics.  In practice
 # they are basically methods (they take the shell as their first argument), but
 # we leave them as standalone functions because eventually the magics
 # themselves will become separate objects altogether.  At that point, the
 # magics will have access to the shell object, and these functions can be made
 # methods of the magic object, but not of the shell.
 
 def store_or_execute(shell, block, name):
     """ Execute a block, or store it in a variable, per the user's request.
     """
     # Dedent and prefilter so what we store matches what is executed by
     # run_cell.
     b = shell.prefilter(textwrap.dedent(block))
 
     if name:
         # If storing it for further editing, run the prefilter on it
         shell.user_ns[name] = SList(b.splitlines())
         print "Block assigned to '%s'" % name
     else:
         shell.user_ns['pasted_block'] = b
         shell.run_cell(b)
 
 
 def rerun_pasted(shell, name='pasted_block'):
     """ Rerun a previously pasted command.
     """
     b = shell.user_ns.get(name)
 
     # Sanity checks
     if b is None:
         raise UsageError('No previous pasted block available')
     if not isinstance(b, basestring):
         raise UsageError(
             "Variable 'pasted_block' is not a string, can't execute")
 
     print "Re-executing '%s...' (%d chars)"% (b.split('\n',1)[0], len(b))
     shell.run_cell(b)
 
 
 #-----------------------------------------------------------------------------
 # Main class
 #-----------------------------------------------------------------------------
 
 class TerminalInteractiveShell(InteractiveShell):
 
     autoedit_syntax = CBool(False, config=True,
         help="auto editing of files with syntax errors.")
     banner = Unicode('')
     banner1 = Unicode(default_banner, config=True,
         help="""The part of the banner to be printed before the profile"""
     )
     banner2 = Unicode('', config=True,
         help="""The part of the banner to be printed after the profile"""
     )
     confirm_exit = CBool(True, config=True,
         help="""
         Set to confirm when you try to exit IPython with an EOF (Control-D
         in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
         you can force a direct exit without any confirmation.""",
     )
     # This display_banner only controls whether or not self.show_banner()
     # is called when mainloop/interact are called.  The default is False
     # because for the terminal based application, the banner behavior
     # is controlled by Global.display_banner, which IPythonApp looks at
     # to determine if *it* should call show_banner() by hand or not.
     display_banner = CBool(False) # This isn't configurable!
     embedded = CBool(False)
     embedded_active = CBool(False)
     editor = Unicode(get_default_editor(), config=True,
         help="Set the editor used by IPython (default to $EDITOR/vi/notepad)."
     )
     pager = Unicode('less', config=True,
         help="The shell program to be used for paging.")
 
     screen_length = Integer(0, config=True,
         help=
         """Number of lines of your screen, used to control printing of very
         long strings.  Strings longer than this number of lines will be sent
         through a pager instead of directly printed.  The default value for
         this is 0, which means IPython will auto-detect your screen size every
         time it needs to print certain potentially long strings (this doesn't
         change the behavior of the 'print' keyword, it's only triggered
         internally). If for some reason this isn't working well (it needs
         curses support), specify it yourself. Otherwise don't change the
         default.""",
     )
     term_title = CBool(False, config=True,
         help="Enable auto setting the terminal title."
     )
 
     # In the terminal, GUI control is done via PyOS_InputHook
     from IPython.lib.inputhook import enable_gui
     enable_gui = staticmethod(enable_gui)
     
     def __init__(self, config=None, ipython_dir=None, profile_dir=None,
                  user_ns=None, user_module=None, custom_exceptions=((),None),
                  usage=None, banner1=None, banner2=None, display_banner=None):
 
         super(TerminalInteractiveShell, self).__init__(
             config=config, profile_dir=profile_dir, user_ns=user_ns,
             user_module=user_module, custom_exceptions=custom_exceptions
         )
         # use os.system instead of utils.process.system by default,
         # because piped system doesn't make sense in the Terminal:
         self.system = self.system_raw
 
         self.init_term_title()
         self.init_usage(usage)
         self.init_banner(banner1, banner2, display_banner)
 
     #-------------------------------------------------------------------------
     # Things related to the terminal
     #-------------------------------------------------------------------------
 
     @property
     def usable_screen_length(self):
         if self.screen_length == 0:
             return 0
         else:
             num_lines_bot = self.separate_in.count('\n')+1
             return self.screen_length - num_lines_bot
 
     def init_term_title(self):
         # Enable or disable the terminal title.
         if self.term_title:
             toggle_set_term_title(True)
             set_term_title('IPython: ' + abbrev_cwd())
         else:
             toggle_set_term_title(False)
 
     #-------------------------------------------------------------------------
     # Things related to aliases
     #-------------------------------------------------------------------------
 
     def init_alias(self):
         # The parent class defines aliases that can be safely used with any
         # frontend.
         super(TerminalInteractiveShell, self).init_alias()
 
         # Now define aliases that only make sense on the terminal, because they
         # need direct access to the console in a way that we can't emulate in
         # GUI or web frontend
         if os.name == 'posix':
             aliases = [('clear', 'clear'), ('more', 'more'), ('less', 'less'),
                        ('man', 'man')]
         elif os.name == 'nt':
             aliases = [('cls', 'cls')]
 
 
         for name, cmd in aliases:
             self.alias_manager.define_alias(name, cmd)
 
     #-------------------------------------------------------------------------
     # Things related to the banner and usage
     #-------------------------------------------------------------------------
 
     def _banner1_changed(self):
         self.compute_banner()
 
     def _banner2_changed(self):
         self.compute_banner()
 
     def _term_title_changed(self, name, new_value):
         self.init_term_title()
 
     def init_banner(self, banner1, banner2, display_banner):
         if banner1 is not None:
             self.banner1 = banner1
         if banner2 is not None:
             self.banner2 = banner2
         if display_banner is not None:
             self.display_banner = display_banner
         self.compute_banner()
 
     def show_banner(self, banner=None):
         if banner is None:
             banner = self.banner
         self.write(banner)
 
     def compute_banner(self):
         self.banner = self.banner1
         if self.profile and self.profile != 'default':
             self.banner += '\nIPython profile: %s\n' % self.profile
         if self.banner2:
             self.banner += '\n' + self.banner2
 
     def init_usage(self, usage=None):
         if usage is None:
             self.usage = interactive_usage
         else:
             self.usage = usage
 
     #-------------------------------------------------------------------------
     # Mainloop and code execution logic
     #-------------------------------------------------------------------------
 
     def mainloop(self, display_banner=None):
         """Start the mainloop.
 
         If an optional banner argument is given, it will override the
         internally created default banner.
         """
 
         with nested(self.builtin_trap, self.display_trap):
 
             while 1:
                 try:
                     self.interact(display_banner=display_banner)
                     #self.interact_with_readline()
                     # XXX for testing of a readline-decoupled repl loop, call
                     # interact_with_readline above
                     break
                 except KeyboardInterrupt:
                     # this should not be necessary, but KeyboardInterrupt
                     # handling seems rather unpredictable...
                     self.write("\nKeyboardInterrupt in interact()\n")
 
     def _replace_rlhist_multiline(self, source_raw, hlen_before_cell):
         """Store multiple lines as a single entry in history"""
 
         # do nothing without readline or disabled multiline
         if not self.has_readline or not self.multiline_history:
             return hlen_before_cell
 
         # windows rl has no remove_history_item
         if not hasattr(self.readline, "remove_history_item"):
             return hlen_before_cell
 
         # skip empty cells
         if not source_raw.rstrip():
             return hlen_before_cell
 
         # nothing changed do nothing, e.g. when rl removes consecutive dups
         hlen = self.readline.get_current_history_length()
         if hlen == hlen_before_cell:
             return hlen_before_cell
 
         for i in range(hlen - hlen_before_cell):
             self.readline.remove_history_item(hlen - i - 1)
-        stdin_encoding = py3compat.get_stream_enc(sys.stdin, 'utf-8')
+        stdin_encoding = get_stream_enc(sys.stdin, 'utf-8')
         self.readline.add_history(py3compat.unicode_to_str(source_raw.rstrip(),
                                     stdin_encoding))
         return self.readline.get_current_history_length()
 
     def interact(self, display_banner=None):
         """Closely emulate the interactive Python console."""
 
         # batch run -> do not interact
         if self.exit_now:
             return
 
         if display_banner is None:
             display_banner = self.display_banner
 
         if isinstance(display_banner, basestring):
             self.show_banner(display_banner)
         elif display_banner:
             self.show_banner()
 
         more = False
 
         if self.has_readline:
             self.readline_startup_hook(self.pre_readline)
             hlen_b4_cell = self.readline.get_current_history_length()
         else:
             hlen_b4_cell = 0
         # exit_now is set by a call to %Exit or %Quit, through the
         # ask_exit callback.
 
         while not self.exit_now:
             self.hooks.pre_prompt_hook()
             if more:
                 try:
                     prompt = self.prompt_manager.render('in2')
                 except:
                     self.showtraceback()
                 if self.autoindent:
                     self.rl_do_indent = True
 
             else:
                 try:
                     prompt = self.separate_in + self.prompt_manager.render('in')
                 except:
                     self.showtraceback()
             try:
                 line = self.raw_input(prompt)
                 if self.exit_now:
                     # quick exit on sys.std[in|out] close
                     break
                 if self.autoindent:
                     self.rl_do_indent = False
 
             except KeyboardInterrupt:
                 #double-guard against keyboardinterrupts during kbdint handling
                 try:
                     self.write('\nKeyboardInterrupt\n')
                     source_raw = self.input_splitter.source_raw_reset()[1]
                     hlen_b4_cell = \
                         self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
                     more = False
                 except KeyboardInterrupt:
                     pass
             except EOFError:
                 if self.autoindent:
                     self.rl_do_indent = False
                     if self.has_readline:
                         self.readline_startup_hook(None)
                 self.write('\n')
                 self.exit()
             except bdb.BdbQuit:
                 warn('The Python debugger has exited with a BdbQuit exception.\n'
                      'Because of how pdb handles the stack, it is impossible\n'
                      'for IPython to properly format this particular exception.\n'
                      'IPython will resume normal operation.')
             except:
                 # exceptions here are VERY RARE, but they can be triggered
                 # asynchronously by signal handlers, for example.
                 self.showtraceback()
             else:
                 self.input_splitter.push(line)
                 more = self.input_splitter.push_accepts_more()
                 if (self.SyntaxTB.last_syntax_error and
                     self.autoedit_syntax):
                     self.edit_syntax_error()
                 if not more:
                     source_raw = self.input_splitter.source_raw_reset()[1]
                     self.run_cell(source_raw, store_history=True)
                     hlen_b4_cell = \
                         self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
 
         # Turn off the exit flag, so the mainloop can be restarted if desired
         self.exit_now = False
 
     def raw_input(self, prompt=''):
         """Write a prompt and read a line.
 
         The returned line does not include the trailing newline.
         When the user enters the EOF key sequence, EOFError is raised.
 
         Optional inputs:
 
           - prompt(''): a string to be printed to prompt the user.
 
           - continue_prompt(False): whether this line is the first one or a
           continuation in a sequence of inputs.
         """
         # Code run by the user may have modified the readline completer state.
         # We must ensure that our completer is back in place.
 
         if self.has_readline:
             self.set_readline_completer()
 
         try:
             line = py3compat.str_to_unicode(self.raw_input_original(prompt))
         except ValueError:
             warn("\n********\nYou or a %run:ed script called sys.stdin.close()"
                  " or sys.stdout.close()!\nExiting IPython!")
             self.ask_exit()
             return ""
 
         # Try to be reasonably smart about not re-indenting pasted input more
         # than necessary.  We do this by trimming out the auto-indent initial
         # spaces, if the user's actual input started itself with whitespace.
         if self.autoindent:
             if num_ini_spaces(line) > self.indent_current_nsp:
                 line = line[self.indent_current_nsp:]
                 self.indent_current_nsp = 0
 
         return line
 
     #-------------------------------------------------------------------------
     # Methods to support auto-editing of SyntaxErrors.
     #-------------------------------------------------------------------------
 
     def edit_syntax_error(self):
         """The bottom half of the syntax error handler called in the main loop.
 
         Loop until syntax error is fixed or user cancels.
         """
 
         while self.SyntaxTB.last_syntax_error:
             # copy and clear last_syntax_error
             err = self.SyntaxTB.clear_err_state()
             if not self._should_recompile(err):
                 return
             try:
                 # may set last_syntax_error again if a SyntaxError is raised
                 self.safe_execfile(err.filename,self.user_ns)
             except:
                 self.showtraceback()
             else:
                 try:
                     f = open(err.filename)
                     try:
                         # This should be inside a display_trap block and I
                         # think it is.
                         sys.displayhook(f.read())
                     finally:
                         f.close()
                 except:
                     self.showtraceback()
 
     def _should_recompile(self,e):
         """Utility routine for edit_syntax_error"""
 
         if e.filename in ('<ipython console>','<input>','<string>',
                           '<console>','<BackgroundJob compilation>',
                           None):
 
             return False
         try:
             if (self.autoedit_syntax and
                 not self.ask_yes_no('Return to editor to correct syntax error? '
                               '[Y/n] ','y')):
                 return False
         except EOFError:
             return False
 
         def int0(x):
             try:
                 return int(x)
             except TypeError:
                 return 0
         # always pass integer line and offset values to editor hook
         try:
             self.hooks.fix_error_editor(e.filename,
                 int0(e.lineno),int0(e.offset),e.msg)
         except TryNext:
             warn('Could not open editor')
             return False
         return True
 
     #-------------------------------------------------------------------------
     # Things related to exiting
     #-------------------------------------------------------------------------
 
     def ask_exit(self):
         """ Ask the shell to exit. Can be overiden and used as a callback. """
         self.exit_now = True
 
     def exit(self):
         """Handle interactive exit.
 
         This method calls the ask_exit callback."""
         if self.confirm_exit:
             if self.ask_yes_no('Do you really want to exit ([y]/n)?','y'):
                 self.ask_exit()
         else:
             self.ask_exit()
 
     #------------------------------------------------------------------------
     # Magic overrides
     #------------------------------------------------------------------------
     # Once the base class stops inheriting from magic, this code needs to be
     # moved into a separate machinery as well.  For now, at least isolate here
     # the magics which this class needs to implement differently from the base
     # class, or that are unique to it.
 
     def magic_autoindent(self, parameter_s = ''):
         """Toggle autoindent on/off (if available)."""
 
         self.shell.set_autoindent()
         print "Automatic indentation is:",['OFF','ON'][self.shell.autoindent]
 
     @skip_doctest
     def magic_cpaste(self, parameter_s=''):
         """Paste & execute a pre-formatted code block from clipboard.
 
         You must terminate the block with '--' (two minus-signs) or Ctrl-D
         alone on the line. You can also provide your own sentinel with '%paste
         -s %%' ('%%' is the new sentinel for this operation)
 
         The block is dedented prior to execution to enable execution of method
         definitions. '>' and '+' characters at the beginning of a line are
         ignored, to allow pasting directly from e-mails, diff files and
         doctests (the '...' continuation prompt is also stripped).  The
         executed block is also assigned to variable named 'pasted_block' for
         later editing with '%edit pasted_block'.
 
         You can also pass a variable name as an argument, e.g. '%cpaste foo'.
         This assigns the pasted block to variable 'foo' as string, without
         dedenting or executing it (preceding >>> and + is still stripped)
 
         '%cpaste -r' re-executes the block previously entered by cpaste.
 
         Do not be alarmed by garbled output on Windows (it's a readline bug).
         Just press enter and type -- (and press enter again) and the block
         will be what was just pasted.
 
         IPython statements (magics, shell escapes) are not supported (yet).
 
         See also
         --------
         paste: automatically pull code from clipboard.
 
         Examples
         --------
         ::
 
           In [8]: %cpaste
           Pasting code; enter '--' alone on the line to stop.
           :>>> a = ["world!", "Hello"]
           :>>> print " ".join(sorted(a))
           :--
           Hello world!
         """
 
         opts, name = self.parse_options(parameter_s, 'rs:', mode='string')
         if 'r' in opts:
             rerun_pasted(self.shell)
             return
 
         sentinel = opts.get('s', '--')
         block = strip_email_quotes(get_pasted_lines(sentinel))
         store_or_execute(self.shell, block, name)
 
     def magic_paste(self, parameter_s=''):
         """Paste & execute a pre-formatted code block from clipboard.
 
         The text is pulled directly from the clipboard without user
         intervention and printed back on the screen before execution (unless
         the -q flag is given to force quiet mode).
 
         The block is dedented prior to execution to enable execution of method
         definitions. '>' and '+' characters at the beginning of a line are
         ignored, to allow pasting directly from e-mails, diff files and
         doctests (the '...' continuation prompt is also stripped).  The
         executed block is also assigned to variable named 'pasted_block' for
         later editing with '%edit pasted_block'.
 
         You can also pass a variable name as an argument, e.g. '%paste foo'.
         This assigns the pasted block to variable 'foo' as string, without
         dedenting or executing it (preceding >>> and + is still stripped)
 
         Options
         -------
 
           -r: re-executes the block previously entered by cpaste.
 
           -q: quiet mode: do not echo the pasted text back to the terminal.
 
         IPython statements (magics, shell escapes) are not supported (yet).
 
         See also
         --------
         cpaste: manually paste code into terminal until you mark its end.
         """
         opts, name = self.parse_options(parameter_s, 'rq', mode='string')
         if 'r' in opts:
             rerun_pasted(self.shell)
             return
         try:
             text = self.shell.hooks.clipboard_get()
             block = strip_email_quotes(text.splitlines())
         except TryNext as clipboard_exc:
             message = getattr(clipboard_exc, 'args')
             if message:
                 error(message[0])
             else:
                 error('Could not get text from the clipboard.')
             return
 
         # By default, echo back to terminal unless quiet mode is requested
         if 'q' not in opts:
             write = self.shell.write
             write(self.shell.pycolorize(block))
             if not block.endswith('\n'):
                 write('\n')
             write("## -- End pasted text --\n")
 
         store_or_execute(self.shell, block, name)
 
     # Class-level: add a '%cls' magic only on Windows
     if sys.platform == 'win32':
         def magic_cls(self, s):
             """Clear screen.
             """
             os.system("cls")
 
     def showindentationerror(self):
         super(TerminalInteractiveShell, self).showindentationerror()
         print("If you want to paste code into IPython, try the "
               "%paste and %cpaste magic functions.")
 
 
 InteractiveShellABC.register(TerminalInteractiveShell)

IPython/testing/tools.py

@@ -1,391 +1,392 b''
 """Generic testing tools.
 
 In particular, this module exposes a set of top-level assert* functions that
 can be used in place of nose.tools.assert* in method generators (the ones in
 nose can not, at least as of nose 0.10.4).
 
 
 Authors
 -------
 - Fernando Perez <[e-mail]>
 """
 
 from __future__ import absolute_import
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2009-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import re
 import sys
 import tempfile
 
 from contextlib import contextmanager
 from io import StringIO
 
 try:
     # These tools are used by parts of the runtime, so we make the nose
     # dependency optional at this point.  Nose is a hard dependency to run the
     # test suite, but NOT to use ipython itself.
     import nose.tools as nt
     has_nose = True
 except ImportError:
     has_nose = False
 
 from IPython.config.loader import Config
 from IPython.utils.process import find_cmd, getoutputerror
 from IPython.utils.text import list_strings
 from IPython.utils.io import temp_pyfile, Tee
 from IPython.utils import py3compat
+from IPython.utils.encoding import getdefaultencoding
 
 from . import decorators as dec
 from . import skipdoctest
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # Make a bunch of nose.tools assert wrappers that can be used in test
 # generators.  This will expose an assert* function for each one in nose.tools.
 
 _tpl = """
 def %(name)s(*a,**kw):
     return nt.%(name)s(*a,**kw)
 """
 
 if has_nose:
     for _x in [a for a in dir(nt) if a.startswith('assert')]:
         exec _tpl % dict(name=_x)
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 # The docstring for full_path doctests differently on win32 (different path
 # separator) so just skip the doctest there.  The example remains informative.
 doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
 
 @doctest_deco
 def full_path(startPath,files):
     """Make full paths for all the listed files, based on startPath.
 
     Only the base part of startPath is kept, since this routine is typically
     used with a script's __file__ variable as startPath.  The base of startPath
     is then prepended to all the listed files, forming the output list.
 
     Parameters
     ----------
       startPath : string
         Initial path to use as the base for the results.  This path is split
       using os.path.split() and only its first component is kept.
 
       files : string or list
         One or more files.
 
     Examples
     --------
 
     >>> full_path('/foo/bar.py',['a.txt','b.txt'])
     ['/foo/a.txt', '/foo/b.txt']
 
     >>> full_path('/foo',['a.txt','b.txt'])
     ['/a.txt', '/b.txt']
 
     If a single file is given, the output is still a list:
     >>> full_path('/foo','a.txt')
     ['/a.txt']
     """
 
     files = list_strings(files)
     base = os.path.split(startPath)[0]
     return [ os.path.join(base,f) for f in files ]
 
 
 def parse_test_output(txt):
     """Parse the output of a test run and return errors, failures.
 
     Parameters
     ----------
     txt : str
       Text output of a test run, assumed to contain a line of one of the
       following forms::
         'FAILED (errors=1)'
         'FAILED (failures=1)'
         'FAILED (errors=1, failures=1)'
 
     Returns
     -------
     nerr, nfail: number of errors and failures.
     """
 
     err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
     if err_m:
         nerr = int(err_m.group(1))
         nfail = 0
         return  nerr, nfail
 
     fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
     if fail_m:
         nerr = 0
         nfail = int(fail_m.group(1))
         return  nerr, nfail
 
     both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
                        re.MULTILINE)
     if both_m:
         nerr = int(both_m.group(1))
         nfail = int(both_m.group(2))
         return  nerr, nfail
 
     # If the input didn't match any of these forms, assume no error/failures
     return 0, 0
 
 
 # So nose doesn't think this is a test
 parse_test_output.__test__ = False
 
 
 def default_argv():
     """Return a valid default argv for creating testing instances of ipython"""
 
     return ['--quick', # so no config file is loaded
             # Other defaults to minimize side effects on stdout
             '--colors=NoColor', '--no-term-title','--no-banner',
             '--autocall=0']
 
 
 def default_config():
     """Return a config object with good defaults for testing."""
     config = Config()
     config.TerminalInteractiveShell.colors = 'NoColor'
     config.TerminalTerminalInteractiveShell.term_title = False,
     config.TerminalInteractiveShell.autocall = 0
     config.HistoryManager.hist_file = tempfile.mktemp(u'test_hist.sqlite')
     config.HistoryManager.db_cache_size = 10000
     return config
 
 
 def ipexec(fname, options=None):
     """Utility to call 'ipython filename'.
 
     Starts IPython witha minimal and safe configuration to make startup as fast
     as possible.
 
     Note that this starts IPython in a subprocess!
 
     Parameters
     ----------
     fname : str
       Name of file to be executed (should have .py or .ipy extension).
 
     options : optional, list
       Extra command-line flags to be passed to IPython.
 
     Returns
     -------
     (stdout, stderr) of ipython subprocess.
     """
     if options is None: options = []
 
     # For these subprocess calls, eliminate all prompt printing so we only see
     # output from script execution
     prompt_opts = [ '--PromptManager.in_template=""',
                     '--PromptManager.in2_template=""',
                     '--PromptManager.out_template=""'
     ]
     cmdargs = ' '.join(default_argv() + prompt_opts + options)
 
     _ip = get_ipython()
     test_dir = os.path.dirname(__file__)
 
     ipython_cmd = find_cmd('ipython3' if py3compat.PY3 else 'ipython')
     # Absolute path for filename
     full_fname = os.path.join(test_dir, fname)
     full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname)
     #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg
     out, err = getoutputerror(full_cmd)
     # `import readline` causes 'ESC[?1034h' to be output sometimes,
     # so strip that out before doing comparisons
     if out:
         out = re.sub(r'\x1b\[[^h]+h', '', out)
     return out, err
 
 
 def ipexec_validate(fname, expected_out, expected_err='',
                     options=None):
     """Utility to call 'ipython filename' and validate output/error.
 
     This function raises an AssertionError if the validation fails.
 
     Note that this starts IPython in a subprocess!
 
     Parameters
     ----------
     fname : str
       Name of the file to be executed (should have .py or .ipy extension).
 
     expected_out : str
       Expected stdout of the process.
 
     expected_err : optional, str
       Expected stderr of the process.
 
     options : optional, list
       Extra command-line flags to be passed to IPython.
 
     Returns
     -------
     None
     """
 
     import nose.tools as nt
 
     out, err = ipexec(fname, options)
     #print 'OUT', out  # dbg
     #print 'ERR', err  # dbg
     # If there are any errors, we must check those befor stdout, as they may be
     # more informative than simply having an empty stdout.
     if err:
         if expected_err:
             nt.assert_equals(err.strip(), expected_err.strip())
         else:
             raise ValueError('Running file %r produced error: %r' %
                              (fname, err))
     # If no errors or output on stderr was expected, match stdout
     nt.assert_equals(out.strip(), expected_out.strip())
 
 
 class TempFileMixin(object):
     """Utility class to create temporary Python/IPython files.
 
     Meant as a mixin class for test cases."""
 
     def mktmp(self, src, ext='.py'):
         """Make a valid python temp file."""
         fname, f = temp_pyfile(src, ext)
         self.tmpfile = f
         self.fname = fname
 
     def tearDown(self):
         if hasattr(self, 'tmpfile'):
             # If the tmpfile wasn't made because of skipped tests, like in
             # win32, there's nothing to cleanup.
             self.tmpfile.close()
             try:
                 os.unlink(self.fname)
             except:
                 # On Windows, even though we close the file, we still can't
                 # delete it.  I have no clue why
                 pass
 
 pair_fail_msg = ("Testing {0}\n\n"
                 "In:\n"
                 "  {1!r}\n"
                 "Expected:\n"
                 "  {2!r}\n"
                 "Got:\n"
                 "  {3!r}\n")
 def check_pairs(func, pairs):
     """Utility function for the common case of checking a function with a
     sequence of input/output pairs.
 
     Parameters
     ----------
     func : callable
       The function to be tested. Should accept a single argument.
     pairs : iterable
       A list of (input, expected_output) tuples.
 
     Returns
     -------
     None. Raises an AssertionError if any output does not match the expected
     value.
     """
     name = getattr(func, "func_name", getattr(func, "__name__", "<unknown>"))
     for inp, expected in pairs:
         out = func(inp)
         assert out == expected, pair_fail_msg.format(name, inp, expected, out)
 
 
 if py3compat.PY3:
     MyStringIO = StringIO
 else:
     # In Python 2, stdout/stderr can have either bytes or unicode written to them,
     # so we need a class that can handle both.
     class MyStringIO(StringIO):
         def write(self, s):
-            s = py3compat.cast_unicode(s, encoding=py3compat.getdefaultencoding())
+            s = py3compat.cast_unicode(s, encoding=getdefaultencoding())
             super(MyStringIO, self).write(s)
 
 notprinted_msg = """Did not find {0!r} in printed output (on {1}):
 {2!r}"""
 
 class AssertPrints(object):
     """Context manager for testing that code prints certain text.
     
     Examples
     --------
     >>> with AssertPrints("abc", suppress=False):
     ...     print "abcd"
     ...     print "def"
     ... 
     abcd
     def
     """
     def __init__(self, s, channel='stdout', suppress=True):
         self.s = s
         self.channel = channel
         self.suppress = suppress
     
     def __enter__(self):
         self.orig_stream = getattr(sys, self.channel)
         self.buffer = MyStringIO()
         self.tee = Tee(self.buffer, channel=self.channel)
         setattr(sys, self.channel, self.buffer if self.suppress else self.tee)
     
     def __exit__(self, etype, value, traceback):
         self.tee.flush()
         setattr(sys, self.channel, self.orig_stream)
         printed = self.buffer.getvalue()
         assert self.s in printed, notprinted_msg.format(self.s, self.channel, printed)
         return False
     
 class AssertNotPrints(AssertPrints):
     """Context manager for checking that certain output *isn't* produced.
     
     Counterpart of AssertPrints"""
     def __exit__(self, etype, value, traceback):
         self.tee.flush()
         setattr(sys, self.channel, self.orig_stream)
         printed = self.buffer.getvalue()
         assert self.s not in printed, notprinted_msg.format(self.s, self.channel, printed)
         return False
 
 @contextmanager
 def mute_warn():
     from IPython.utils import warn
     save_warn = warn.warn
     warn.warn = lambda *a, **kw: None
     try:
         yield
     finally:
         warn.warn = save_warn
 
 @contextmanager
 def make_tempfile(name):
     """ Create an empty, named, temporary file for the duration of the context.
     """
     f = open(name, 'w')
     f.close()
     try:
         yield
     finally:
         os.unlink(name)

IPython/utils/_process_posix.py

@@ -1,197 +1,198 b''
 """Posix-specific implementation of process utilities.
 
 This file is only meant to be imported by process.py, not by end-users.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Stdlib
 import subprocess as sp
 import sys
 
 from IPython.external import pexpect
 
 # Our own
 from .autoattr import auto_attr
 from ._process_common import getoutput, arg_split
 from IPython.utils import text
 from IPython.utils import py3compat
+from IPython.utils.encoding import getdefaultencoding
 
 #-----------------------------------------------------------------------------
 # Function definitions
 #-----------------------------------------------------------------------------
 
 def _find_cmd(cmd):
     """Find the full path to a command using which."""
 
     path = sp.Popen(['/usr/bin/env', 'which', cmd],
                     stdout=sp.PIPE).communicate()[0]
     return py3compat.bytes_to_str(path)
 
 
 class ProcessHandler(object):
     """Execute subprocesses under the control of pexpect.
     """
     # Timeout in seconds to wait on each reading of the subprocess' output.
     # This should not be set too low to avoid cpu overusage from our side,
     # since we read in a loop whose period is controlled by this timeout.
     read_timeout = 0.05
 
     # Timeout to give a process if we receive SIGINT, between sending the
     # SIGINT to the process and forcefully terminating it.
     terminate_timeout = 0.2
 
     # File object where stdout and stderr of the subprocess will be written
     logfile = None
 
     # Shell to call for subprocesses to execute
     sh = None
 
     @auto_attr
     def sh(self):
         sh = pexpect.which('sh')
         if sh is None:
             raise OSError('"sh" shell not found')
         return sh
 
     def __init__(self, logfile=None, read_timeout=None, terminate_timeout=None):
         """Arguments are used for pexpect calls."""
         self.read_timeout = (ProcessHandler.read_timeout if read_timeout is
                              None else read_timeout)
         self.terminate_timeout = (ProcessHandler.terminate_timeout if
                                   terminate_timeout is None else
                                   terminate_timeout)
         self.logfile = sys.stdout if logfile is None else logfile
 
     def getoutput(self, cmd):
         """Run a command and return its stdout/stderr as a string.
 
         Parameters
         ----------
         cmd : str
           A command to be executed in the system shell.
 
         Returns
         -------
         output : str
           A string containing the combination of stdout and stderr from the
         subprocess, in whatever order the subprocess originally wrote to its
         file descriptors (so the order of the information in this string is the
         correct order as would be seen if running the command in a terminal).
         """
         try:
             return pexpect.run(self.sh, args=['-c', cmd]).replace('\r\n', '\n')
         except KeyboardInterrupt:
             print('^C', file=sys.stderr, end='')
 
     def getoutput_pexpect(self, cmd):
         """Run a command and return its stdout/stderr as a string.
 
         Parameters
         ----------
         cmd : str
           A command to be executed in the system shell.
 
         Returns
         -------
         output : str
           A string containing the combination of stdout and stderr from the
         subprocess, in whatever order the subprocess originally wrote to its
         file descriptors (so the order of the information in this string is the
         correct order as would be seen if running the command in a terminal).
         """
         try:
             return pexpect.run(self.sh, args=['-c', cmd]).replace('\r\n', '\n')
         except KeyboardInterrupt:
             print('^C', file=sys.stderr, end='')
 
     def system(self, cmd):
         """Execute a command in a subshell.
 
         Parameters
         ----------
         cmd : str
           A command to be executed in the system shell.
 
         Returns
         -------
         int : child's exitstatus
         """
         # Get likely encoding for the output.
-        enc = py3compat.getdefaultencoding()
+        enc = getdefaultencoding()
         
         # Patterns to match on the output, for pexpect.  We read input and
         # allow either a short timeout or EOF
         patterns = [pexpect.TIMEOUT, pexpect.EOF]
         # the index of the EOF pattern in the list.
         # even though we know it's 1, this call means we don't have to worry if
         # we change the above list, and forget to change this value:
         EOF_index = patterns.index(pexpect.EOF)
         # The size of the output stored so far in the process output buffer.
         # Since pexpect only appends to this buffer, each time we print we
         # record how far we've printed, so that next time we only print *new*
         # content from the buffer.
         out_size = 0
         try:
             # Since we're not really searching the buffer for text patterns, we
             # can set pexpect's search window to be tiny and it won't matter.
             # We only search for the 'patterns' timeout or EOF, which aren't in
             # the text itself.
             #child = pexpect.spawn(pcmd, searchwindowsize=1)
             if hasattr(pexpect, 'spawnb'):
                 child = pexpect.spawnb(self.sh, args=['-c', cmd]) # Pexpect-U
             else:
                 child = pexpect.spawn(self.sh, args=['-c', cmd])  # Vanilla Pexpect
             flush = sys.stdout.flush
             while True:
                 # res is the index of the pattern that caused the match, so we
                 # know whether we've finished (if we matched EOF) or not
                 res_idx = child.expect_list(patterns, self.read_timeout)
                 print(child.before[out_size:].decode(enc, 'replace'), end='')
                 flush()
                 if res_idx==EOF_index:
                     break
                 # Update the pointer to what we've already printed
                 out_size = len(child.before)
         except KeyboardInterrupt:
             # We need to send ^C to the process.  The ascii code for '^C' is 3
             # (the character is known as ETX for 'End of Text', see
             # curses.ascii.ETX).
             child.sendline(chr(3))
             # Read and print any more output the program might produce on its
             # way out.
             try:
                 out_size = len(child.before)
                 child.expect_list(patterns, self.terminate_timeout)
                 print(child.before[out_size:].decode(enc, 'replace'), end='')
                 sys.stdout.flush()
             except KeyboardInterrupt:
                 # Impatient users tend to type it multiple times
                 pass
             finally:
                 # Ensure the subprocess really is terminated
                 child.terminate(force=True)
         # add isalive check, to ensure exitstatus is set:
         child.isalive()
         return child.exitstatus
 
 
 # Make system() with a functional interface for outside use.  Note that we use
 # getoutput() from the _common utils, which is built on top of popen(). Using
 # pexpect to get subprocess output produces difficult to parse output, since
 # programs think they are talking to a tty and produce highly formatted output
 # (ls is a good example) that makes them hard.
 system = ProcessHandler().system
 
 
 

IPython/utils/jsonutil.py

@@ -1,165 +1,166 b''
 """Utilities to manipulate JSON objects.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING.txt, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 # stdlib
 import re
 import sys
 import types
 from datetime import datetime
 
 from IPython.utils import py3compat
+from IPython.utils.encoding import getdefaultencoding
 from IPython.utils import text
 next_attr_name = '__next__' if py3compat.PY3 else 'next'
 
 #-----------------------------------------------------------------------------
 # Globals and constants
 #-----------------------------------------------------------------------------
 
 # timestamp formats
 ISO8601="%Y-%m-%dT%H:%M:%S.%f"
 ISO8601_PAT=re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+$")
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 def rekey(dikt):
     """Rekey a dict that has been forced to use str keys where there should be
     ints by json."""
     for k in dikt.iterkeys():
         if isinstance(k, basestring):
             ik=fk=None
             try:
                 ik = int(k)
             except ValueError:
                 try:
                     fk = float(k)
                 except ValueError:
                     continue
             if ik is not None:
                 nk = ik
             else:
                 nk = fk
             if nk in dikt:
                 raise KeyError("already have key %r"%nk)
             dikt[nk] = dikt.pop(k)
     return dikt
 
 
 def extract_dates(obj):
     """extract ISO8601 dates from unpacked JSON"""
     if isinstance(obj, dict):
         obj = dict(obj) # don't clobber
         for k,v in obj.iteritems():
             obj[k] = extract_dates(v)
     elif isinstance(obj, (list, tuple)):
         obj = [ extract_dates(o) for o in obj ]
     elif isinstance(obj, basestring):
         if ISO8601_PAT.match(obj):
             obj = datetime.strptime(obj, ISO8601)
     return obj
 
 def squash_dates(obj):
     """squash datetime objects into ISO8601 strings"""
     if isinstance(obj, dict):
         obj = dict(obj) # don't clobber
         for k,v in obj.iteritems():
             obj[k] = squash_dates(v)
     elif isinstance(obj, (list, tuple)):
         obj = [ squash_dates(o) for o in obj ]
     elif isinstance(obj, datetime):
         obj = obj.strftime(ISO8601)
     return obj
     
 def date_default(obj):
     """default function for packing datetime objects in JSON."""
     if isinstance(obj, datetime):
         return obj.strftime(ISO8601)
     else:
         raise TypeError("%r is not JSON serializable"%obj)
 
 
 
 def json_clean(obj):
     """Clean an object to ensure it's safe to encode in JSON.
     
     Atomic, immutable objects are returned unmodified.  Sets and tuples are
     converted to lists, lists are copied and dicts are also copied.
 
     Note: dicts whose keys could cause collisions upon encoding (such as a dict
     with both the number 1 and the string '1' as keys) will cause a ValueError
     to be raised.
 
     Parameters
     ----------
     obj : any python object
 
     Returns
     -------
     out : object
     
       A version of the input which will not cause an encoding error when
       encoded as JSON.  Note that this function does not *encode* its inputs,
       it simply sanitizes it so that there will be no encoding errors later.
 
     Examples
     --------
     >>> json_clean(4)
     4
     >>> json_clean(range(10))
     [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
     >>> json_clean(dict(x=1, y=2))
     {'y': 2, 'x': 1}
     >>> json_clean(dict(x=1, y=2, z=[1,2,3]))
     {'y': 2, 'x': 1, 'z': [1, 2, 3]}
     >>> json_clean(True)
     True
     """
     # types that are 'atomic' and ok in json as-is.  bool doesn't need to be
     # listed explicitly because bools pass as int instances
     atomic_ok = (unicode, int, float, types.NoneType)
     
     # containers that we need to convert into lists
     container_to_list = (tuple, set, types.GeneratorType)
     
     if isinstance(obj, atomic_ok):
         return obj
     
     if isinstance(obj, bytes):
-        return obj.decode(py3compat.getdefaultencoding(), 'replace')
+        return obj.decode(getdefaultencoding(), 'replace')
     
     if isinstance(obj, container_to_list) or (
         hasattr(obj, '__iter__') and hasattr(obj, next_attr_name)):
         obj = list(obj)
         
     if isinstance(obj, list):
         return [json_clean(x) for x in obj]
 
     if isinstance(obj, dict):
         # First, validate that the dict won't lose data in conversion due to
         # key collisions after stringification.  This can happen with keys like
         # True and 'true' or 1 and '1', which collide in JSON.
         nkeys = len(obj)
         nkeys_collapsed = len(set(map(str, obj)))
         if nkeys != nkeys_collapsed:
             raise ValueError('dict can not be safely converted to JSON: '
                              'key collision would lead to dropped values')
         # If all OK, proceed by making the new dict that will be json-safe
         out = {}
         for k,v in obj.iteritems():
             out[str(k)] = json_clean(v)
         return out
 
     # If we get here, we don't know how to handle the object, so we just get
     # its repr and return that.  This will catch lambdas, open sockets, class
     # objects, and any other complicated contraption that json can't encode
     return repr(obj)

IPython/utils/py3compat.py

@@ -1,207 +1,178 b''
 # coding: utf-8
 """Compatibility tricks for Python 3. Mainly to do with unicode."""
 import __builtin__
 import functools
 import sys
 import re
 import types
-import locale
+
+from IPython.utils.encoding import getdefaultencoding
 
 orig_open = open
 
 def no_code(x, encoding=None):
     return x
 
-# to deal with the possibility of sys.std* not being a stream at all
-def get_stream_enc(stream, default=None):
-    if not hasattr(stream, 'encoding') or not stream.encoding:
-        return default
-    else:
-        return stream.encoding
-
-# Less conservative replacement for sys.getdefaultencoding, that will try
-# to match the environment.
-# Defined here as central function, so if we find better choices, we
-# won't need to make changes all over IPython.
-def getdefaultencoding():
-    """Return IPython's guess for the default encoding for bytes as text.
-
-    Asks for stdin.encoding first, to match the calling Terminal, but that
-    is often None for subprocesses.  Fall back on locale.getpreferredencoding()
-    which should be a sensible platform default (that respects LANG environment),
-    and finally to sys.getdefaultencoding() which is the most conservative option,
-    and usually ASCII.
-    """
-    enc = get_stream_enc(sys.stdin)
-    if not enc or enc=='ascii':
-        try:
-            # There are reports of getpreferredencoding raising errors
-            # in some cases, which may well be fixed, but let's be conservative here.
-            enc = locale.getpreferredencoding()
-        except Exception:
-            pass
-    return enc or sys.getdefaultencoding()
-
 def decode(s, encoding=None):
     encoding = encoding or getdefaultencoding()
     return s.decode(encoding, "replace")
 
 def encode(u, encoding=None):
     encoding = encoding or getdefaultencoding()
     return u.encode(encoding, "replace")
 
 
 def cast_unicode(s, encoding=None):
     if isinstance(s, bytes):
         return decode(s, encoding)
     return s
 
 def cast_bytes(s, encoding=None):
     if not isinstance(s, bytes):
         return encode(s, encoding)
     return s
 
 def _modify_str_or_docstring(str_change_func):
     @functools.wraps(str_change_func)
     def wrapper(func_or_str):
         if isinstance(func_or_str, basestring):
             func = None
             doc = func_or_str
         else:
             func = func_or_str
             doc = func.__doc__
         
         doc = str_change_func(doc)
         
         if func:
             func.__doc__ = doc
             return func
         return doc
     return wrapper
 
 if sys.version_info[0] >= 3:
     PY3 = True
     
     input = input
     builtin_mod_name = "builtins"
     
     str_to_unicode = no_code
     unicode_to_str = no_code
     str_to_bytes = encode
     bytes_to_str = decode
     cast_bytes_py2 = no_code
     
     def isidentifier(s, dotted=False):
         if dotted:
             return all(isidentifier(a) for a in s.split("."))
         return s.isidentifier()
     
     open = orig_open
     
     MethodType = types.MethodType
     
     def execfile(fname, glob, loc=None):
         loc = loc if (loc is not None) else glob
         exec compile(open(fname, 'rb').read(), fname, 'exec') in glob, loc
     
     # Refactor print statements in doctests.
     _print_statement_re = re.compile(r"\bprint (?P<expr>.*)$", re.MULTILINE)
     def _print_statement_sub(match):
         expr = match.groups('expr')
         return "print(%s)" % expr
     
     @_modify_str_or_docstring
     def doctest_refactor_print(doc):
         """Refactor 'print x' statements in a doctest to print(x) style. 2to3
         unfortunately doesn't pick up on our doctests.
         
         Can accept a string or a function, so it can be used as a decorator."""
         return _print_statement_re.sub(_print_statement_sub, doc)
     
     # Abstract u'abc' syntax:
     @_modify_str_or_docstring
     def u_format(s):
         """"{u}'abc'" --> "'abc'" (Python 3)
         
         Accepts a string or a function, so it can be used as a decorator."""
         return s.format(u='')
 
 else:
     PY3 = False
     
     input = raw_input
     builtin_mod_name = "__builtin__"
     
     str_to_unicode = decode
     unicode_to_str = encode
     str_to_bytes = no_code
     bytes_to_str = no_code
     cast_bytes_py2 = cast_bytes
     
     import re
     _name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$")
     def isidentifier(s, dotted=False):
         if dotted:
             return all(isidentifier(a) for a in s.split("."))
         return bool(_name_re.match(s))
     
     class open(object):
         """Wrapper providing key part of Python 3 open() interface."""
         def __init__(self, fname, mode="r", encoding="utf-8"):
             self.f = orig_open(fname, mode)
             self.enc = encoding
         
         def write(self, s):
             return self.f.write(s.encode(self.enc))
         
         def read(self, size=-1):
             return self.f.read(size).decode(self.enc)
         
         def close(self):
             return self.f.close()
         
         def __enter__(self):
             return self
         
         def __exit__(self, etype, value, traceback):
             self.f.close()
     
     def MethodType(func, instance):
         return types.MethodType(func, instance, type(instance))
     
     # don't override system execfile on 2.x:
     execfile = execfile
     
     def doctest_refactor_print(func_or_str):
         return func_or_str
 
 
     # Abstract u'abc' syntax:
     @_modify_str_or_docstring
     def u_format(s):
         """"{u}'abc'" --> "u'abc'" (Python 2)
         
         Accepts a string or a function, so it can be used as a decorator."""
         return s.format(u='u')
 
     if sys.platform == 'win32':
         def execfile(fname, glob=None, loc=None):
             loc = loc if (loc is not None) else glob
             # The rstrip() is necessary b/c trailing whitespace in files will
             # cause an IndentationError in Python 2.6 (this was fixed in 2.7,
             # but we still support 2.6).  See issue 1027.
             scripttext = __builtin__.open(fname).read().rstrip() + '\n'
             # compile converts unicode filename to str assuming
             # ascii. Let's do the conversion before calling compile
             if isinstance(fname, unicode):
                 filename = unicode_to_str(fname)
             else:
                 filename = fname
             exec compile(scripttext, filename, 'exec') in glob, loc
     else:
         def execfile(fname, *where):
             if isinstance(fname, unicode):
                 filename = fname.encode(sys.getfilesystemencoding())
             else:
                 filename = fname
             __builtin__.execfile(filename, *where)

IPython/zmq/iostream.py

@@ -1,91 +1,91 b''
 import sys
 import time
 from io import StringIO
 
 from session import extract_header, Message
 
-from IPython.utils import io, text, py3compat
+from IPython.utils import io, text, encoding
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Stream classes
 #-----------------------------------------------------------------------------
 
 class OutStream(object):
     """A file like object that publishes the stream to a 0MQ PUB socket."""
 
     # The time interval between automatic flushes, in seconds.
     flush_interval = 0.05
     topic=None
 
     def __init__(self, session, pub_socket, name):
         self.session = session
         self.pub_socket = pub_socket
         self.name = name
         self.parent_header = {}
         self._new_buffer()
 
     def set_parent(self, parent):
         self.parent_header = extract_header(parent)
 
     def close(self):
         self.pub_socket = None
 
     def flush(self):
         #io.rprint('>>>flushing output buffer: %s<<<' % self.name)  # dbg
         if self.pub_socket is None:
             raise ValueError(u'I/O operation on closed file')
         else:
             data = self._buffer.getvalue()
             if data:
                 content = {u'name':self.name, u'data':data}
                 msg = self.session.send(self.pub_socket, u'stream', content=content,
                                        parent=self.parent_header, ident=self.topic)
 
                 if hasattr(self.pub_socket, 'flush'):
                     # socket itself has flush (presumably ZMQStream)
                     self.pub_socket.flush()
                 self._buffer.close()
                 self._new_buffer()
 
     def isatty(self):
         return False
 
     def next(self):
         raise IOError('Read not supported on a write only stream.')
 
     def read(self, size=-1):
         raise IOError('Read not supported on a write only stream.')
 
     def readline(self, size=-1):
         raise IOError('Read not supported on a write only stream.')
 
     def write(self, string):
         if self.pub_socket is None:
             raise ValueError('I/O operation on closed file')
         else:
             # Make sure that we're handling unicode
             if not isinstance(string, unicode):
-                enc = py3compat.getdefaultencoding()
+                enc = encoding.getdefaultencoding()
                 string = string.decode(enc, 'replace')
             
             self._buffer.write(string)
             current_time = time.time()
             if self._start <= 0:
                 self._start = current_time
             elif current_time - self._start > self.flush_interval:
                 self.flush()
 
     def writelines(self, sequence):
         if self.pub_socket is None:
             raise ValueError('I/O operation on closed file')
         else:
             for string in sequence:
                 self.write(string)
 
     def _new_buffer(self):
         self._buffer = StringIO()
         self._start = -1