Unicode fixes (utf-8 used by default if ascii is not enough). This should fix some reported crashes.

Fix broken auto-quoting for pysh.
Remove old 2.1 specific hacks and move all 2.2 compatibility symbols into genutils.
Simplify i/o abstraction layer a bit.
Full details in changelog.

IPython/Extensions/InterpreterExec.py

@@ -1,270 +1,272 b''
 # -*- coding: utf-8 -*-
 """Modified input prompt for executing files.
 
 We define a special input line filter to allow typing lines which begin with
 '~', '/' or '.'. If one of those strings is encountered, it is automatically
 executed.
 
-$Id: InterpreterExec.py 573 2005-04-08 08:38:09Z fperez $"""
+$Id: InterpreterExec.py 638 2005-07-18 03:01:41Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2004 W.J. van der Laan <[e-mail]>
 #       Copyright (C) 2004 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__ = 'W.J. van der Laan <[e-mail]>, '\
              '%s <%s>' %  Release.authors['Fernando']
 __license__ = Release.license
 
 def prefilter_shell(self,line,continuation):
     """Alternate prefilter, modified for shell-like functionality.
 
     - Execute all lines beginning with '~', '/' or '.'
     - $var=cmd <=> %sc var=cmd
     - $$var=cmd <=> %sc -l var=cmd
     """
 
     if line:
         l0 = line[0]
         if l0 in '~/.':
             return self._prefilter("!%s"%line,continuation)
         elif l0=='$':
             lrest = line[1:]
             if lrest.startswith('$'):
                 # $$var=cmd <=> %sc -l var=cmd
                 return self._prefilter("%ssc -l %s" % (self.ESC_MAGIC,lrest[1:]),
                                        continuation)
             else:
                 # $var=cmd <=> %sc var=cmd
                 return self._prefilter("%ssc %s" % (self.ESC_MAGIC,lrest),
                                        continuation)
         else:
             return self._prefilter(line,continuation)
     else:
         return self._prefilter(line,continuation)
 
 # Rebind this to be the new IPython prefilter:
 from IPython.iplib import InteractiveShell
 InteractiveShell.prefilter = prefilter_shell
 # Clean up the namespace.
 del InteractiveShell,prefilter_shell
 
 # Provide pysh and further shell-oriented services
 import os,sys,shutil
 from IPython.genutils import system,shell,getoutput,getoutputerror
 
 # Short aliases for getting shell output as a string and a list
 sout = getoutput
 lout = lambda cmd: getoutput(cmd,split=1)
 
 # Empty function, meant as a docstring holder so help(pysh) works.
 def pysh():
     """Pysh is a set of modules and extensions to IPython which make shell-like
     usage with Python syntax more convenient.  Keep in mind that pysh is NOT a
     full-blown shell, so don't try to make it your /etc/passwd entry!
     
     In particular, it has no job control, so if you type Ctrl-Z (under Unix),
     you'll suspend pysh itself, not the process you just started.
 
     Since pysh is really nothing but a customized IPython, you should
     familiarize yourself with IPython's features.  This brief help mainly
     documents areas in which pysh differs from the normal IPython.
 
     ALIASES
     -------
     All of your $PATH has been loaded as IPython aliases, so you should be
     able to type any normal system command and have it executed.  See %alias? 
     and %unalias? for details on the alias facilities.
 
     SPECIAL SYNTAX
     --------------
     Any lines which begin with '~', '/' and '.' will be executed as shell
     commands instead of as Python code. The special escapes below are also
     recognized.  !cmd is valid in single or multi-line input, all others are
     only valid in single-line input:
 
     !cmd      - pass 'cmd' directly to the shell
     !!cmd     - execute 'cmd' and return output as a list (split on '\\n')
     $var=cmd  - capture output of cmd into var, as a string
     $$var=cmd - capture output of cmd into var, as a list (split on '\\n')
 
     The $/$$ syntaxes make Python variables from system output, which you can
     later use for further scripting.  The converse is also possible: when
     executing an alias or calling to the system via !/!!, you can expand any
     python variable or expression by prepending it with $.  Full details of
     the allowed syntax can be found in Python's PEP 215.
 
     A few brief examples will illustrate these:
 
         fperez[~/test]|3> !ls *s.py
         scopes.py  strings.py
 
     ls is an internal alias, so there's no need to use !:
         fperez[~/test]|4> ls *s.py
         scopes.py*  strings.py
 
     !!ls will return the output into a Python variable:
         fperez[~/test]|5> !!ls *s.py
                       <5> ['scopes.py', 'strings.py']
         fperez[~/test]|6> print _5
         ['scopes.py', 'strings.py']
 
     $ and $$ allow direct capture to named variables:
         fperez[~/test]|7> $astr = ls *s.py
         fperez[~/test]|8> astr
                       <8> 'scopes.py\\nstrings.py'
 
         fperez[~/test]|9> $$alist = ls *s.py
         fperez[~/test]|10> alist
                       <10> ['scopes.py', 'strings.py']
 
     alist is now a normal python list you can loop over.  Using $ will expand
     back the python values when alias calls are made:
         fperez[~/test]|11> for f in alist:
                       |..>     print 'file',f,
                       |..>     wc -l $f
                       |..>
         file scopes.py     13 scopes.py
         file strings.py      4 strings.py
 
     Note that you may need to protect your variables with braces if you want
     to append strings to their names.  To copy all files in alist to .bak
     extensions, you must use:
         fperez[~/test]|12> for f in alist:
                       |..>     cp $f ${f}.bak
 
     If you try using $f.bak, you'll get an AttributeError exception saying
     that your string object doesn't have a .bak attribute.  This is because
     the $ expansion mechanism allows you to expand full Python expressions:
         fperez[~/test]|13> echo "sys.platform is: $sys.platform"
         sys.platform is: linux2
 
     IPython's input history handling is still active, which allows you to
     rerun a single block of multi-line input by simply using exec:    
         fperez[~/test]|14> $$alist = ls *.eps
         fperez[~/test]|15> exec _i11
         file image2.eps    921 image2.eps
         file image.eps    921 image.eps
 
     While these are new special-case syntaxes, they are designed to allow very
     efficient use of the shell with minimal typing.  At an interactive shell
     prompt, conciseness of expression wins over readability.
 
     USEFUL FUNCTIONS AND MODULES
     ----------------------------
     The os, sys and shutil modules from the Python standard library are
     automatically loaded.  Some additional functions, useful for shell usage,
     are listed below.  You can request more help about them with '?'.
 
     shell   - execute a command in the underlying system shell    
     system  - like shell(), but return the exit status of the command
     sout    - capture the output of a command as a string
     lout    - capture the output of a command as a list (split on '\\n')
     getoutputerror - capture (output,error) of a shell command
 
     sout/lout are the functional equivalents of $/$$.  They are provided to
     allow you to capture system output in the middle of true python code,
     function definitions, etc (where $ and $$ are invalid).
 
     DIRECTORY MANAGEMENT
     --------------------
     Since each command passed by pysh to the underlying system is executed in
     a subshell which exits immediately, you can NOT use !cd to navigate the
     filesystem.
 
     Pysh provides its own builtin '%cd' magic command to move in the
     filesystem (the % is not required with automagic on).  It also maintains a
     list of visited directories (use %dhist to see it) and allows direct
     switching to any of them.  Type 'cd?' for more details.
 
     %pushd, %popd and %dirs are provided for directory stack handling.
 
     PROMPT CUSTOMIZATION
     --------------------
     
     The supplied ipythonrc-pysh profile comes with an example of a very
     colored and detailed prompt, mainly to serve as an illustration.  The
     valid escape sequences, besides color names, are:
 
         \\#  - Prompt number.
         \\D  - Dots, as many as there are digits in \\# (so they align).
         \\w  - Current working directory (cwd).
         \\W  - Basename of current working directory.
         \\XN - Where N=0..5. N terms of the cwd, with $HOME written as ~.
         \\YN - Where N=0..5. Like XN, but if ~ is term N+1 it's also shown.
         \\u  - Username.
         \\H  - Full hostname.
         \\h  - Hostname up to first '.'
         \\$  - Root symbol ($ or #).
         \\t  - Current time, in H:M:S format.
         \\v  - IPython release version.
         \\n  - Newline.
         \\r  - Carriage return.
         \\\\ - An explicitly escaped '\\'.
 
     You can configure your prompt colors using any ANSI color escape.  Each
     color escape sets the color for any subsequent text, until another escape
     comes in and changes things.  The valid color escapes are:
 
         \\C_Black
         \\C_Blue
         \\C_Brown
         \\C_Cyan
         \\C_DarkGray
         \\C_Green
         \\C_LightBlue
         \\C_LightCyan
         \\C_LightGray
         \\C_LightGreen
         \\C_LightPurple
         \\C_LightRed
         \\C_Purple
         \\C_Red
         \\C_White
         \\C_Yellow
         \\C_Normal - Stop coloring, defaults to your terminal settings.
     """
     pass
 
 # Configure a few things.  Much of this is fairly hackish, since IPython
 # doesn't really expose a clean API for it.  Be careful if you start making
 # many modifications here.
 
 print """\
 Welcome to pysh, a set of extensions to IPython for shell usage.
 help(pysh) -> help on the installed shell extensions and syntax.
 """
 
 #  Set the 'cd' command to quiet mode, a more shell-like behavior
 __IPYTHON__.default_option('cd','-q')
 
 # Load all of $PATH as aliases
 if os.name == 'posix':
     # %rehash is very fast, but it doesn't check for executability, it simply
     # dumps everything in $PATH as an alias. Use rehashx if you want more
     # checks.
     __IPYTHON__.magic_rehash()
 else:
     # Windows users: the list of extensions considered executable is read from
     # the environment variable 'pathext'.  If this is undefined, IPython
     # defaults to EXE, COM and BAT.
     # %rehashx is the one which does extension analysis, at the cost of
     # being much slower than %rehash.
     __IPYTHON__.magic_rehashx()
 
 # Remove %sc,%sx if present as aliases
 __IPYTHON__.magic_unalias('sc')
 __IPYTHON__.magic_unalias('sx')
 
 # We need different criteria for line-splitting, so that aliases such as
 # 'gnome-terminal' are interpreted as a single alias instead of variable
 # 'gnome' minus variable 'terminal'.
 import re
-__IPYTHON__.line_split = re.compile(r'^(\s*)([\?\w\.\-\+]+\w*\s*)(\(?.*$)')
+__IPYTHON__.line_split = re.compile(r'^([\s*,;/])'
+                                    r'([\?\w\.\-\+]+\w*\s*)'
+                                    r'(\(?.*$)')
 
 # Namespace cleanup
 del re

IPython/FlexCompleter.py

@@ -1,199 +1,192 b''
 # -*- coding: utf-8 -*-
 """Word completion for GNU readline 2.0.
 
 ---------------------------------------------------------------------------
 NOTE: This version is a re-implementation of rlcompleter with selectable
 namespace.
 
 The problem with rlcompleter is that it's hardwired to work with
 __main__.__dict__, and in some cases one may have 'sandboxed' namespaces. So
 this class is a ripoff of rlcompleter, with the namespace to work in as an
 optional parameter.
 
 This class can be used just like rlcompleter, but the Completer class now has
 a constructor with the optional 'namespace' parameter.
 
 A patch has been submitted to Python@sourceforge for these changes to go in
 the standard Python distribution.
 
 The patch went in for Python 2.3.  Once IPython drops support for Python 2.2,
 this file can be significantly reduced.
 ---------------------------------------------------------------------------
 
 Original rlcompleter documentation:
 
 This requires the latest extension to the readline module (the
 completes keywords, built-ins and globals in __main__; when completing
 NAME.NAME..., it evaluates (!) the expression up to the last dot and
 completes its attributes.
 
 It's very cool to do "import string" type "string.", hit the
 completion key (twice), and see the list of names defined by the
 string module!
 
 Tip: to use the tab key as the completion key, call
 
     readline.parse_and_bind("tab: complete")
 
 Notes:
 
 - Exceptions raised by the completer function are *ignored* (and
 generally cause the completion to fail).  This is a feature -- since
 readline sets the tty device in raw (or cbreak) mode, printing a
 traceback wouldn't work well without some complicated hoopla to save,
 reset and restore the tty state.
 
 - The evaluation of the NAME.NAME... form may cause arbitrary
 application defined code to be executed if an object with a
 __getattr__ hook is found.  Since it is the responsibility of the
 application (or the user) to enable this feature, I consider this an
 acceptable risk.  More complicated expressions (e.g. function calls or
 indexing operations) are *not* evaluated.
 
 - GNU readline is also used by the built-in functions input() and
 raw_input(), and thus these also benefit/suffer from the completer
 features.  Clearly an interactive application can benefit by
 specifying its own completer function and using raw_input() for all
 its input.
 
 - When the original stdin is not a tty device, GNU readline is never
 used, and this module (and the readline module) are silently inactive.
 
 """
 
 #*****************************************************************************
 #
 # Since this file is essentially a minimally modified copy of the rlcompleter
 # module which is part of the standard Python distribution, I assume that the
 # proper procedure is to maintain its copyright as belonging to the Python
 # Software Foundation:
 #
 #       Copyright (C) 2001 Python Software Foundation, www.python.org
 #
 #  Distributed under the terms of the Python Software Foundation license.
 #
 #  Full text available at:
 #
 #                  http://www.python.org/2.1/license.html
 #
 #*****************************************************************************
 
 import readline
 import __builtin__
 import __main__
 
 __all__ = ["Completer"]
 
-# declares Python 2.2 compatibility symbols:
-try:
-    basestring
-except NameError:
-    import types
-    basestring = (types.StringType, types.UnicodeType)
-
 class Completer:
     def __init__(self, namespace = None):
         """Create a new completer for the command line.
 
         Completer([namespace]) -> completer instance.
 
         If unspecified, the default namespace where completions are performed
         is __main__ (technically, __main__.__dict__). Namespaces should be
         given as dictionaries.
 
         Completer instances should be used as the completion mechanism of
         readline via the set_completer() call:
 
         readline.set_completer(Completer(my_namespace).complete)
         """
         
         if namespace and type(namespace) != type({}):
             raise TypeError,'namespace must be a dictionary'
 
         # Don't bind to namespace quite yet, but flag whether the user wants a
         # specific namespace or to use __main__.__dict__. This will allow us
         # to bind to __main__.__dict__ at completion time, not now.
         if namespace is None:
             self.use_main_ns = 1
         else:
             self.use_main_ns = 0
             self.namespace = namespace
 
     def complete(self, text, state):
         """Return the next possible completion for 'text'.
 
         This is called successively with state == 0, 1, 2, ... until it
         returns None.  The completion should begin with 'text'.
 
         """
         if self.use_main_ns:
             self.namespace = __main__.__dict__
             
         if state == 0:
             if "." in text:
                 self.matches = self.attr_matches(text)
             else:
                 self.matches = self.global_matches(text)
         try:
             return self.matches[state]
         except IndexError:
             return None
 
     def global_matches(self, text):
         """Compute matches when text is a simple name.
 
         Return a list of all keywords, built-in functions and names currently
         defined in self.namespace that match.
 
         """
         import keyword
         matches = []
         n = len(text)
         for list in [keyword.kwlist,
                      __builtin__.__dict__.keys(),
                      self.namespace.keys()]:
             for word in list:
                 if word[:n] == text and word != "__builtins__":
                     matches.append(word)
         return matches
 
     def attr_matches(self, text):
         """Compute matches when text contains a dot.
 
         Assuming the text is of the form NAME.NAME....[NAME], and is
         evaluatable in self.namespace, it will be evaluated and its attributes
         (as revealed by dir()) are used as possible completions.  (For class
         instances, class members are are also considered.)
 
         WARNING: this can still invoke arbitrary C code, if an object
         with a __getattr__ hook is evaluated.
 
         """
         import re
 
         # Another option, seems to work great. Catches things like ''.<tab>
         m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text)
 
         if not m:
             return []
         expr, attr = m.group(1, 3)
         object = eval(expr, self.namespace)
         words = [w for w in dir(object) if isinstance(w, basestring)]
         if hasattr(object,'__class__'):
             words.append('__class__')
             words.extend(get_class_members(object.__class__))
         n = len(attr)
         matches = []
         for word in words:
             if word[:n] == attr and word != "__builtins__":
                 matches.append("%s.%s" % (expr, word))
         return matches
 
 def get_class_members(klass):
     ret = dir(klass)
     if hasattr(klass,'__bases__'):
         for base in klass.__bases__:
             ret.extend(get_class_members(base))
     return ret
 
 readline.set_completer(Completer().complete)

IPython/Itpl.py

@@ -1,253 +1,277 b''
 # -*- coding: utf-8 -*-
 """String interpolation for Python (by Ka-Ping Yee, 14 Feb 2000).
 
 This module lets you quickly and conveniently interpolate values into
 strings (in the flavour of Perl or Tcl, but with less extraneous
 punctuation).  You get a bit more power than in the other languages,
 because this module allows subscripting, slicing, function calls,
 attribute lookup, or arbitrary expressions.  Variables and expressions
 are evaluated in the namespace of the caller.
 
 The itpl() function returns the result of interpolating a string, and
 printpl() prints out an interpolated string.  Here are some examples:
 
     from Itpl import printpl
     printpl("Here is a $string.")
     printpl("Here is a $module.member.")
     printpl("Here is an $object.member.")
     printpl("Here is a $functioncall(with, arguments).")
     printpl("Here is an ${arbitrary + expression}.")
     printpl("Here is an $array[3] member.")
     printpl("Here is a $dictionary['member'].")
 
 The filter() function filters a file object so that output through it
 is interpolated.  This lets you produce the illusion that Python knows
 how to do interpolation:
 
     import Itpl
     sys.stdout = Itpl.filter()
     f = "fancy"
     print "Isn't this $f?"
     print "Standard output has been replaced with a $sys.stdout object."
     sys.stdout = Itpl.unfilter()
     print "Okay, back $to $normal."
 
 Under the hood, the Itpl class represents a string that knows how to
 interpolate values.  An instance of the class parses the string once
 upon initialization; the evaluation and substitution can then be done
 each time the instance is evaluated with str(instance).  For example:
 
     from Itpl import Itpl
     s = Itpl("Here is $foo.")
     foo = 5
     print str(s)
     foo = "bar"
     print str(s)
 
-$Id: Itpl.py 542 2005-03-18 09:16:04Z fperez $
+$Id: Itpl.py 638 2005-07-18 03:01:41Z fperez $
 """                   # ' -> close an open quote for stupid emacs
 
 #*****************************************************************************
 #
 # Copyright (c) 2001 Ka-Ping Yee <[e-mail]>
 #
 #
 # Published under the terms of the MIT license, hereby reproduced:
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to
 # deal in the Software without restriction, including without limitation the
 # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 # sell copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
 #
 #*****************************************************************************
 
 __author__  = 'Ka-Ping Yee <[e-mail]>'
 __license__ = 'MIT'
 
 import sys, string
 from types import StringType
 from tokenize import tokenprog
 
 class ItplError(ValueError):
     def __init__(self, text, pos):
         self.text = text
         self.pos = pos
     def __str__(self):
         return "unfinished expression in %s at char %d" % (
             repr(self.text), self.pos)
 
 def matchorfail(text, pos):
     match = tokenprog.match(text, pos)
     if match is None:
         raise ItplError(text, pos)
     return match, match.end()
 
 class Itpl:
     """Class representing a string with interpolation abilities.
     
     Upon creation, an instance works out what parts of the format
     string are literal and what parts need to be evaluated.  The
     evaluation and substitution happens in the namespace of the
     caller when str(instance) is called."""
 
-    def __init__(self, format):
-        """The single argument to this constructor is a format string.
+    def __init__(self, format,codec='utf_8',encoding_errors='backslashreplace'):
+        """The single mandatory argument to this constructor is a format
+        string.
 
         The format string is parsed according to the following rules:
 
         1.  A dollar sign and a name, possibly followed by any of: 
               - an open-paren, and anything up to the matching paren 
               - an open-bracket, and anything up to the matching bracket 
               - a period and a name 
             any number of times, is evaluated as a Python expression.
 
         2.  A dollar sign immediately followed by an open-brace, and
             anything up to the matching close-brace, is evaluated as
             a Python expression.
 
         3.  Outside of the expressions described in the above two rules,
-            two dollar signs in a row give you one literal dollar sign."""
+            two dollar signs in a row give you one literal dollar sign.
 
-        if type(format) != StringType:
+        Optional arguments:
+
+        - codec('utf_8'): a string containing the name of a valid Python
+        codec.
+
+        - encoding_errors('backslashreplace'): a string with a valid error handling
+        policy.  See the codecs module documentation for details.
+
+        These are used to encode the format string if a call to str() fails on
+        the expanded result."""
+
+        if not isinstance(format,basestring):
             raise TypeError, "needs string initializer"
         self.format = format
+        self.codec = codec
+        self.encoding_errors = encoding_errors
 
         namechars = "abcdefghijklmnopqrstuvwxyz" \
             "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_";
         chunks = []
         pos = 0
 
         while 1:
             dollar = string.find(format, "$", pos)
             if dollar < 0: break
             nextchar = format[dollar+1]
 
             if nextchar == "{":
                 chunks.append((0, format[pos:dollar]))
                 pos, level = dollar+2, 1
                 while level:
                     match, pos = matchorfail(format, pos)
                     tstart, tend = match.regs[3]
                     token = format[tstart:tend]
                     if token == "{": level = level+1
                     elif token == "}": level = level-1
                 chunks.append((1, format[dollar+2:pos-1]))
 
             elif nextchar in namechars:
                 chunks.append((0, format[pos:dollar]))
                 match, pos = matchorfail(format, dollar+1)
                 while pos < len(format):
                     if format[pos] == "." and \
                         pos+1 < len(format) and format[pos+1] in namechars:
                         match, pos = matchorfail(format, pos+1)
                     elif format[pos] in "([":
                         pos, level = pos+1, 1
                         while level:
                             match, pos = matchorfail(format, pos)
                             tstart, tend = match.regs[3]
                             token = format[tstart:tend]
                             if token[0] in "([": level = level+1
                             elif token[0] in ")]": level = level-1
                     else: break
                 chunks.append((1, format[dollar+1:pos]))
 
             else:
                 chunks.append((0, format[pos:dollar+1]))
                 pos = dollar + 1 + (nextchar == "$")
 
         if pos < len(format): chunks.append((0, format[pos:]))
         self.chunks = chunks
 
     def __repr__(self):
         return "<Itpl %s >" % repr(self.format)
 
+    def _str(self,glob,loc):
+        """Evaluate to a string in the given globals/locals.
+
+        The final output is built by calling str(), but if this fails, the
+        result is encoded with the instance's codec and error handling policy,
+        via a call to out.encode(self.codec,self.encoding_errors)"""
+        result = []
+        app = result.append
+        for live, chunk in self.chunks:
+            if live: app(str(eval(chunk,glob,loc)))
+            else: app(chunk)
+        out = ''.join(result)
+        try:
+            return str(out)
+        except UnicodeError:
+            return out.encode(self.codec,self.encoding_errors)
+
     def __str__(self):
         """Evaluate and substitute the appropriate parts of the string."""
 
         # We need to skip enough frames to get to the actual caller outside of
         # Itpl.
         frame = sys._getframe(1)
         while frame.f_globals["__name__"] == __name__: frame = frame.f_back
         loc, glob = frame.f_locals, frame.f_globals
 
-        result = []
-        for live, chunk in self.chunks:
-            if live: result.append(str(eval(chunk,glob,loc)))
-            else: result.append(chunk)
-
-        return ''.join(result)
+        return self._str(glob,loc)
 
 class ItplNS(Itpl):
     """Class representing a string with interpolation abilities.
 
     This inherits from Itpl, but at creation time a namespace is provided
     where the evaluation will occur.  The interpolation becomes a bit more
     efficient, as no traceback needs to be extracte.  It also allows the
     caller to supply a different namespace for the interpolation to occur than
     its own."""
     
-    def __init__(self, format,globals,locals=None):
+    def __init__(self, format,globals,locals=None,
+                 codec='utf_8',encoding_errors='backslashreplace'):
         """ItplNS(format,globals[,locals]) -> interpolating string instance.
 
         This constructor, besides a format string, takes a globals dictionary
         and optionally a locals (which defaults to globals if not provided).
 
         For further details, see the Itpl constructor."""
 
         if locals is None:
             locals = globals
         self.globals = globals
         self.locals = locals
-        Itpl.__init__(self,format)
+        Itpl.__init__(self,format,codec,encoding_errors)
         
     def __str__(self):
         """Evaluate and substitute the appropriate parts of the string."""
-        glob = self.globals
-        loc = self.locals
-        result = []
-        for live, chunk in self.chunks:
-            if live: result.append(str(eval(chunk,glob,loc)))
-            else: result.append(chunk)
-        return ''.join(result)
+        return self._str(self.globals,self.locals)
+
+    def __repr__(self):
+        return "<ItplNS %s >" % repr(self.format)
 
 # utilities for fast printing
 def itpl(text): return str(Itpl(text))
 def printpl(text): print itpl(text)
 # versions with namespace
 def itplns(text,globals,locals=None): return str(ItplNS(text,globals,locals))
 def printplns(text,globals,locals=None): print itplns(text,globals,locals)
 
 class ItplFile:
     """A file object that filters each write() through an interpolator."""
     def __init__(self, file): self.file = file
     def __repr__(self): return "<interpolated " + repr(self.file) + ">"
     def __getattr__(self, attr): return getattr(self.file, attr)
     def write(self, text): self.file.write(str(Itpl(text)))
 
 def filter(file=sys.stdout):
     """Return an ItplFile that filters writes to the given file object.
     
     'file = filter(file)' replaces 'file' with a filtered object that
     has a write() method.  When called with no argument, this creates
     a filter to sys.stdout."""
     return ItplFile(file)
 
 def unfilter(ifile=None):
     """Return the original file that corresponds to the given ItplFile.
     
     'file = unfilter(file)' undoes the effect of 'file = filter(file)'.
     'sys.stdout = unfilter()' undoes the effect of 'sys.stdout = filter()'."""
     return ifile and ifile.file or sys.stdout.file

IPython/Prompts.py

@@ -1,574 +1,572 b''
 # -*- coding: utf-8 -*-
 """
 Classes for handling input/output prompts.
 
-$Id: Prompts.py 564 2005-03-26 22:43:58Z fperez $"""
+$Id: Prompts.py 638 2005-07-18 03:01:41Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2004 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Fernando']
 __license__ = Release.license
 __version__ = Release.version
 
 #****************************************************************************
 # Required modules
 import __builtin__
 import os,sys,socket
 import time
 from pprint import pprint,pformat
 
 # IPython's own
 from IPython.genutils import *
 from IPython.Struct import Struct
 from IPython.Magic import Macro
 from IPython.Itpl import ItplNS
 from IPython import ColorANSI
 
 #****************************************************************************
 #Color schemes for Prompts.
 
 PromptColors = ColorANSI.ColorSchemeTable()
 InputColors = ColorANSI.InputTermColors  # just a shorthand
 Colors = ColorANSI.TermColors  # just a shorthand
 
 PromptColors.add_scheme(ColorANSI.ColorScheme(
     'NoColor',
     in_prompt  = InputColors.NoColor,  # Input prompt
     in_number  = InputColors.NoColor,  # Input prompt number
     in_prompt2 = InputColors.NoColor, # Continuation prompt
     in_normal  = InputColors.NoColor,  # color off (usu. Colors.Normal)
     
     out_prompt = Colors.NoColor, # Output prompt
     out_number = Colors.NoColor, # Output prompt number
 
     normal = Colors.NoColor  # color off (usu. Colors.Normal)
     ))
 # make some schemes as instances so we can copy them for modification easily:
 __PColLinux =  ColorANSI.ColorScheme(
     'Linux',
     in_prompt  = InputColors.Green,
     in_number  = InputColors.LightGreen,
     in_prompt2 = InputColors.Green,
     in_normal  = InputColors.Normal,  # color off (usu. Colors.Normal)
 
     out_prompt = Colors.Red,
     out_number = Colors.LightRed,
 
     normal = Colors.Normal
     )
 # Don't forget to enter it into the table!
 PromptColors.add_scheme(__PColLinux)
 # Slightly modified Linux for light backgrounds
 __PColLightBG  = ColorANSI.ColorScheme('LightBG',**__PColLinux.colors.dict().copy())
 
 __PColLightBG.colors.update(
     in_prompt  = InputColors.Blue,
     in_number  = InputColors.LightBlue,
     in_prompt2 = InputColors.Blue
 )
 PromptColors.add_scheme(__PColLightBG)
 
 del Colors,InputColors
 
 #-----------------------------------------------------------------------------
 def multiple_replace(dict, text):
     """ Replace in 'text' all occurences of any key in the given
     dictionary by its corresponding value.  Returns the new string."""
 
     # Function by Xavier Defrang, originally found at:
     # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/81330
 
     # Create a regular expression  from the dictionary keys
     regex = re.compile("(%s)" % "|".join(map(re.escape, dict.keys())))
     # For each match, look-up corresponding value in dictionary
     return regex.sub(lambda mo: dict[mo.string[mo.start():mo.end()]], text)
 
 #-----------------------------------------------------------------------------
 # Special characters that can be used in prompt templates, mainly bash-like
 
 # If $HOME isn't defined (Windows), make it an absurd string so that it can
 # never be expanded out into '~'.  Basically anything which can never be a
 # reasonable directory name will do, we just want the $HOME -> '~' operation
 # to become a no-op.  We pre-compute $HOME here so it's not done on every
 # prompt call.
 
 # FIXME:
 
 # - This should be turned into a class which does proper namespace management,
 # since the prompt specials need to be evaluated in a certain namespace.
 # Currently it's just globals, which need to be managed manually by code
 # below.
 
 # - I also need to split up the color schemes from the prompt specials
 # somehow.  I don't have a clean design for that quite yet.
 
 HOME = os.environ.get("HOME","//////:::::ZZZZZ,,,~~~")
 
 # We precompute a few more strings here for the prompt_specials, which are
 # fixed once ipython starts.  This reduces the runtime overhead of computing
 # prompt strings.
 USER           = os.environ.get("USER")
 HOSTNAME       = socket.gethostname()
 HOSTNAME_SHORT = HOSTNAME.split(".")[0]
 ROOT_SYMBOL    = "$#"[os.name=='nt' or os.getuid()==0]
 
 prompt_specials_color = {
     # Prompt/history count
     '%n' : '${self.col_num}' '${self.cache.prompt_count}' '${self.col_p}',
     '\\#': '${self.col_num}' '${self.cache.prompt_count}' '${self.col_p}',
     # Prompt/history count, with the actual digits replaced by dots.  Used
     # mainly in continuation prompts (prompt_in2)
     '\\D': '${"."*len(str(self.cache.prompt_count))}',
     # Current working directory
     '\\w': '${os.getcwd()}',
     # Current time
     '\\t' : '${time.strftime("%H:%M:%S")}',
     # Basename of current working directory.
     # (use os.sep to make this portable across OSes)
     '\\W' : '${os.getcwd().split("%s")[-1]}' % os.sep,
     # These X<N> are an extension to the normal bash prompts.  They return
     # N terms of the path, after replacing $HOME with '~'
     '\\X0': '${os.getcwd().replace("%s","~")}' % HOME,
     '\\X1': '${self.cwd_filt(1)}',
     '\\X2': '${self.cwd_filt(2)}',
     '\\X3': '${self.cwd_filt(3)}',
     '\\X4': '${self.cwd_filt(4)}',
     '\\X5': '${self.cwd_filt(5)}',
     # Y<N> are similar to X<N>, but they show '~' if it's the directory
     # N+1 in the list.  Somewhat like %cN in tcsh.
     '\\Y0': '${self.cwd_filt2(0)}',
     '\\Y1': '${self.cwd_filt2(1)}',
     '\\Y2': '${self.cwd_filt2(2)}',
     '\\Y3': '${self.cwd_filt2(3)}',
     '\\Y4': '${self.cwd_filt2(4)}',
     '\\Y5': '${self.cwd_filt2(5)}',
     # Hostname up to first .
     '\\h': HOSTNAME_SHORT,
     # Full hostname
     '\\H': HOSTNAME,
     # Username of current user
     '\\u': USER,
     # Escaped '\'
     '\\\\': '\\',
     # Newline
     '\\n': '\n',
     # Carriage return
     '\\r': '\r',
     # Release version
     '\\v': __version__,
     # Root symbol ($ or #)
     '\\$': ROOT_SYMBOL,
     }
 
 # A copy of the prompt_specials dictionary but with all color escapes removed,
 # so we can correctly compute the prompt length for the auto_rewrite method.
 prompt_specials_nocolor = prompt_specials_color.copy()
 prompt_specials_nocolor['%n'] = '${self.cache.prompt_count}'
 prompt_specials_nocolor['\\#'] = '${self.cache.prompt_count}'
 
 # Add in all the InputTermColors color escapes as valid prompt characters.
 # They all get added as \\C_COLORNAME, so that we don't have any conflicts
 # with a color name which may begin with a letter used by any other of the
 # allowed specials.  This of course means that \\C will never be allowed for
 # anything else.
 input_colors = ColorANSI.InputTermColors
 for _color in dir(input_colors):
     if _color[0] != '_':
         c_name = '\\C_'+_color
         prompt_specials_color[c_name] = getattr(input_colors,_color)
         prompt_specials_nocolor[c_name] = ''
 
 # we default to no color for safety.  Note that prompt_specials is a global
 # variable used by all prompt objects.
 prompt_specials = prompt_specials_nocolor
 
 #-----------------------------------------------------------------------------
 def str_safe(arg):
     """Convert to a string, without ever raising an exception.
 
     If str(arg) fails, <ERROR: ... > is returned, where ... is the exception
     error message."""
     
     try:
-        return str(arg)
+        out = str(arg)
+    except UnicodeError:
+        try:
+            out = arg.encode('utf_8','replace')
+        except Exception,msg:
+            # let's keep this little duplication here, so that the most common
+            # case doesn't suffer from a double try wrapping.
+            out = '<ERROR: %s>' % msg
     except Exception,msg:
-        return '<ERROR: %s>' % msg
+        out = '<ERROR: %s>' % msg
+    return out
 
 class BasePrompt:
     """Interactive prompt similar to Mathematica's."""
     def __init__(self,cache,sep,prompt,pad_left=False):
 
         # Hack: we access information about the primary prompt through the
         # cache argument.  We need this, because we want the secondary prompt
         # to be aligned with the primary one.  Color table info is also shared
         # by all prompt classes through the cache.  Nice OO spaghetti code!
         self.cache = cache
         self.sep = sep
         
         # regexp to count the number of spaces at the end of a prompt
         # expression, useful for prompt auto-rewriting
         self.rspace = re.compile(r'(\s*)$')
         # Flag to left-pad prompt strings to match the length of the primary
         # prompt
         self.pad_left = pad_left
         # Set template to create each actual prompt (where numbers change)
         self.p_template = prompt
         self.set_p_str()
 
     def set_p_str(self):
         """ Set the interpolating prompt strings.
 
         This must be called every time the color settings change, because the
         prompt_specials global may have changed."""
 
         import os,time  # needed in locals for prompt string handling
         loc = locals()
         self.p_str = ItplNS('%s%s%s' %
                             ('${self.sep}${self.col_p}',
                              multiple_replace(prompt_specials, self.p_template),
                              '${self.col_norm}'),self.cache.user_ns,loc)
         
         self.p_str_nocolor = ItplNS(multiple_replace(prompt_specials_nocolor,
                                                      self.p_template),
                                     self.cache.user_ns,loc)
 
     def write(self,msg):  # dbg
         sys.stdout.write(msg)
         return ''
 
     def __str__(self):
         """Return a string form of the prompt.
 
         This for is useful for continuation and output prompts, since it is
         left-padded to match lengths with the primary one (if the
         self.pad_left attribute is set)."""
 
         out_str = str_safe(self.p_str)
         if self.pad_left:
             # We must find the amount of padding required to match lengths,
             # taking the color escapes (which are invisible on-screen) into
             # account.
             esc_pad = len(out_str) - len(str_safe(self.p_str_nocolor))
             format = '%%%ss' % (len(str(self.cache.last_prompt))+esc_pad)
             return format % out_str
         else:
             return out_str
 
     # these path filters are put in as methods so that we can control the
     # namespace where the prompt strings get evaluated
     def cwd_filt(self,depth):
         """Return the last depth elements of the current working directory.
 
         $HOME is always replaced with '~'.
         If depth==0, the full path is returned."""
 
         cwd = os.getcwd().replace(HOME,"~")
         out = os.sep.join(cwd.split(os.sep)[-depth:])
         if out:
             return out
         else:
             return os.sep
 
     def cwd_filt2(self,depth):
         """Return the last depth elements of the current working directory.
 
         $HOME is always replaced with '~'.
         If depth==0, the full path is returned."""
 
         cwd = os.getcwd().replace(HOME,"~").split(os.sep)
         if '~' in cwd and len(cwd) == depth+1:
             depth += 1
         out = os.sep.join(cwd[-depth:])
         if out:
             return out
         else:
             return os.sep
 
 class Prompt1(BasePrompt):
     """Input interactive prompt similar to Mathematica's."""
 
     def __init__(self,cache,sep='\n',prompt='In [\\#]: ',pad_left=True):
         BasePrompt.__init__(self,cache,sep,prompt,pad_left)
 
     def set_colors(self):
         self.set_p_str()
         Colors = self.cache.color_table.active_colors # shorthand
         self.col_p = Colors.in_prompt
         self.col_num = Colors.in_number
         self.col_norm = Colors.in_normal
         # We need a non-input version of these escapes for the '--->'
         # auto-call prompts used in the auto_rewrite() method.
         self.col_p_ni = self.col_p.replace('\001','').replace('\002','') 
         self.col_norm_ni = Colors.normal        
         
     def __str__(self):
         self.cache.prompt_count += 1
         self.cache.last_prompt = str_safe(self.p_str_nocolor).split('\n')[-1]
         return str_safe(self.p_str)
 
     def auto_rewrite(self):
         """Print a string of the form '--->' which lines up with the previous
         input string. Useful for systems which re-write the user input when
         handling automatically special syntaxes."""
 
         curr = str(self.cache.last_prompt)
         nrspaces = len(self.rspace.search(curr).group())
         return '%s%s>%s%s' % (self.col_p_ni,'-'*(len(curr)-nrspaces-1),
                               ' '*nrspaces,self.col_norm_ni)
 
 class PromptOut(BasePrompt):
     """Output interactive prompt similar to Mathematica's."""
 
     def __init__(self,cache,sep='',prompt='Out[\\#]: ',pad_left=True):
         BasePrompt.__init__(self,cache,sep,prompt,pad_left)
         if not self.p_template:
             self.__str__ = lambda: ''
 
     def set_colors(self):
         self.set_p_str()
         Colors = self.cache.color_table.active_colors # shorthand
         self.col_p = Colors.out_prompt
         self.col_num = Colors.out_number
         self.col_norm = Colors.normal
 
 class Prompt2(BasePrompt):
     """Interactive continuation prompt."""
     
     def __init__(self,cache,prompt='   .\\D.: ',pad_left=True):
         self.cache = cache
         self.p_template = prompt
         self.pad_left = pad_left
         self.set_p_str()
 
     def set_p_str(self):
         import os,time  # needed in locals for prompt string handling
         loc = locals()
         self.p_str = ItplNS('%s%s%s' %
                             ('${self.col_p2}',
                              multiple_replace(prompt_specials, self.p_template),
                              '$self.col_norm'),
                             self.cache.user_ns,loc)
         self.p_str_nocolor = ItplNS(multiple_replace(prompt_specials_nocolor,
                                                      self.p_template),
                                     self.cache.user_ns,loc)
 
     def set_colors(self):
         self.set_p_str()
         Colors = self.cache.color_table.active_colors
         self.col_p2 = Colors.in_prompt2
         self.col_norm = Colors.in_normal
         # FIXME (2004-06-16) HACK: prevent crashes for users who haven't
         # updated their prompt_in2 definitions.  Remove eventually.
         self.col_p = Colors.out_prompt
         self.col_num = Colors.out_number
 
 #-----------------------------------------------------------------------------
 class CachedOutput:
     """Class for printing output from calculations while keeping a cache of
     reults. It dynamically creates global variables prefixed with _ which
     contain these results.
 
     Meant to be used as a sys.displayhook replacement, providing numbered
     prompts and cache services.
 
     Initialize with initial and final values for cache counter (this defines
     the maximum size of the cache."""
 
     def __init__(self,cache_size,Pprint,colors='NoColor',input_sep='\n',
                  output_sep='\n',output_sep2='',user_ns={},
                  ps1 = None, ps2 = None,ps_out = None,
                  input_hist = None,pad_left=True):
 
         cache_size_min = 20
         if cache_size <= 0:
             self.do_full_cache = 0
             cache_size = 0
         elif cache_size < cache_size_min:
             self.do_full_cache = 0
             cache_size = 0
             warn('caching was disabled (min value for cache size is %s).' %
                  cache_size_min,level=3)
         else:
             self.do_full_cache = 1
 
         self.cache_size = cache_size
         self.input_sep = input_sep
 
         # we need a reference to the user-level namespace
         self.user_ns = user_ns
         # and to the user's input
         self.input_hist = input_hist
 
         # Set input prompt strings and colors
         if cache_size == 0:
             if ps1.find('%n') > -1 or ps1.find('\\#') > -1: ps1 = '>>> '
             if ps2.find('%n') > -1 or ps2.find('\\#') > -1: ps2 = '... '
         self.ps1_str = self._set_prompt_str(ps1,'In [\\#]: ','>>> ')
         self.ps2_str = self._set_prompt_str(ps2,'   .\\D.: ','... ')
         self.ps_out_str = self._set_prompt_str(ps_out,'Out[\\#]: ','')
 
+        self.color_table = PromptColors
         self.prompt1 = Prompt1(self,sep=input_sep,prompt=self.ps1_str,
                                pad_left=pad_left)
         self.prompt2 = Prompt2(self,prompt=self.ps2_str,pad_left=pad_left)
         self.prompt_out = PromptOut(self,sep='',prompt=self.ps_out_str,
                                     pad_left=pad_left)
-        self.color_table = PromptColors
         self.set_colors(colors)
 
         # other more normal stuff
         # b/c each call to the In[] prompt raises it by 1, even the first.
         self.prompt_count = 0
         self.cache_count = 1
         # Store the last prompt string each time, we need it for aligning
         # continuation and auto-rewrite prompts
         self.last_prompt = ''
         self.entries = [None]  # output counter starts at 1 for the user
         self.Pprint = Pprint
         self.output_sep = output_sep
         self.output_sep2 = output_sep2
         self._,self.__,self.___ = '','',''
         self.pprint_types = map(type,[(),[],{}])
         
         # these are deliberately global:
         to_user_ns = {'_':self._,'__':self.__,'___':self.___}
         self.user_ns.update(to_user_ns)
 
     def _set_prompt_str(self,p_str,cache_def,no_cache_def):
         if p_str is None:
             if self.do_full_cache:
                 return cache_def
             else:
                 return no_cache_def
         else:
             return p_str
                 
     def set_colors(self,colors):
         """Set the active color scheme and configure colors for the three
         prompt subsystems."""
 
         # FIXME: the prompt_specials global should be gobbled inside this
         # class instead.  Do it when cleaning up the whole 3-prompt system.
         global prompt_specials
         if colors.lower()=='nocolor':
             prompt_specials = prompt_specials_nocolor
         else:
             prompt_specials = prompt_specials_color
         
         self.color_table.set_active_scheme(colors)
         self.prompt1.set_colors()
         self.prompt2.set_colors()
         self.prompt_out.set_colors()
 
     def __call__(self,arg=None):
         """Printing with history cache management.
         
         This is invoked everytime the interpreter needs to print, and is
         activated by setting the variable sys.displayhook to it."""
 
         # If something injected a '_' variable in __builtin__, delete
         # ipython's automatic one so we don't clobber that.  gettext() in
         # particular uses _, so we need to stay away from it.
         if '_' in __builtin__.__dict__:
             try:
                 del self.user_ns['_']
             except KeyError:
                 pass
         if arg is not None:
+            cout_write = Term.cout.write # fast lookup
             # first handle the cache and counters
             self.update(arg)
             # do not print output if input ends in ';'
             if self.input_hist[self.prompt_count].endswith(';\n'):
                 return
             # don't use print, puts an extra space
-            Term.cout.write(self.output_sep)
+            cout_write(self.output_sep)
             if self.do_full_cache:
-                Term.cout.write(str(self.prompt_out))
+                cout_write(str(self.prompt_out))
 
             if isinstance(arg,Macro):
                 print 'Executing Macro...'
                 # in case the macro takes a long time to execute
                 Term.cout.flush()
                 exec arg.value in self.user_ns
                 return None
 
             # and now call a possibly user-defined print mechanism
             self.display(arg)
-            Term.cout.write(self.output_sep2)
+            cout_write(self.output_sep2)
             Term.cout.flush()
 
     def _display(self,arg):
         """Default printer method, uses pprint.
 
         This can be over-ridden by the users to implement special formatting
         of certain types of output."""
 
         if self.Pprint:
-            # The following is an UGLY kludge, b/c python fails to properly
-            # identify instances of classes imported in the user namespace
-            # (they have different memory locations, I guess). Structs are
-            # essentially dicts but pprint doesn't know what to do with them.
-            try:
-                if arg.__class__.__module__ == 'Struct' and \
-                   arg.__class__.__name__ == 'Struct':
-                    out = 'Struct:\n%s' % pformat(arg.dict())
-                else:
-                    out = pformat(arg)
-            except:
                 out = pformat(arg)
             if '\n' in out:
                 # So that multi-line strings line up with the left column of
                 # the screen, instead of having the output prompt mess up
                 # their first line.                
                 Term.cout.write('\n')
             print >>Term.cout, out
         else:
             print >>Term.cout, arg
 
     # Assign the default display method:
     display = _display
 
     def update(self,arg):
         #print '***cache_count', self.cache_count # dbg
         if self.cache_count >= self.cache_size and self.do_full_cache:
             self.flush()
         # Don't overwrite '_' and friends if '_' is in __builtin__ (otherwise
         # we cause buggy behavior for things like gettext).
         if '_' not in __builtin__.__dict__:
             self.___ = self.__
             self.__ = self._
             self._ = arg
             self.user_ns.update({'_':self._,'__':self.__,'___':self.___})
             
         # hackish access to top-level  namespace to create _1,_2... dynamically
         to_main = {}
         if self.do_full_cache:
             self.cache_count += 1
             self.entries.append(arg)
             new_result = '_'+`self.prompt_count`
             to_main[new_result] = self.entries[-1]
         self.user_ns.update(to_main)
         self.user_ns['_oh'][self.prompt_count] = arg
 
     def flush(self):
         if not self.do_full_cache:
             raise ValueError,"You shouldn't have reached the cache flush "\
                   "if full caching is not enabled!"
         warn('Output cache limit (currently '+\
               `self.cache_count`+' entries) hit.\n'
              'Flushing cache and resetting history counter...\n'
              'The only history variables available will be _,__,___ and _1\n'
              'with the current result.')
         # delete auto-generated vars from global namespace
         for n in range(1,self.prompt_count + 1):
             key = '_'+`n`
             try:
                 del self.user_ns[key]
             except: pass
         self.prompt_count = 1
         self.cache_count = 1

IPython/Struct.py

@@ -1,376 +1,375 b''
 # -*- coding: utf-8 -*-
 """Mimic C structs with lots of extra functionality.
 
-$Id: Struct.py 410 2004-11-04 07:58:17Z fperez $"""
+$Id: Struct.py 638 2005-07-18 03:01:41Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2004 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Fernando']
 __license__ = Release.license
 
 __all__ = ['Struct']
 
 import types
 from IPython.genutils import list2dict2
 
 class Struct:
     """Class to mimic C structs but also provide convenient dictionary-like
     functionality.
 
     Instances can be initialized with a dictionary, a list of key=value pairs
     or both. If both are present, the dictionary must come first.
 
     Because Python classes provide direct assignment to their members, it's
     easy to overwrite normal methods (S.copy = 1 would destroy access to
     S.copy()). For this reason, all builtin method names are protected and
     can't be assigned to. An attempt to do s.copy=1 or s['copy']=1 will raise
     a KeyError exception. If you really want to, you can bypass this
     protection by directly assigning to __dict__: s.__dict__['copy']=1 will
     still work. Doing this will break functionality, though. As in most of
     Python, namespace protection is weakly enforced, so feel free to shoot
     yourself if you really want to.
 
     Note that this class uses more memory and is *much* slower than a regular
     dictionary, so be careful in situations where memory or performance are
     critical. But for day to day use it should behave fine. It is particularly
     convenient for storing configuration data in programs.
 
     +,+=,- and -= are implemented. +/+= do merges (non-destructive updates),
     -/-= remove keys from the original. See the method descripitions.
 
     This class allows a quick access syntax: both s.key and s['key'] are
     valid.  This syntax has a limitation: each 'key' has to be explicitly
     accessed by its original name. The normal s.key syntax doesn't provide
     access to the keys via variables whose values evaluate to the desired
     keys. An example should clarify this:
 
     Define a dictionary and initialize both with dict and k=v pairs:
     >>> d={'a':1,'b':2}
     >>> s=Struct(d,hi=10,ho=20)
     The return of __repr__ can be used to create a new instance:
     >>> s
     Struct({'ho': 20, 'b': 2, 'hi': 10, 'a': 1})
     __str__ (called by print) shows it's not quite a regular dictionary:
     >>> print s
     Struct {a: 1, b: 2, hi: 10, ho: 20}
     Access by explicitly named key with dot notation:
     >>> s.a
     1
     Or like a dictionary:
     >>> s['a']
     1
     If you want a variable to hold the key value, only dictionary access works:
     >>> key='hi'
     >>> s.key
     Traceback (most recent call last):
       File "<stdin>", line 1, in ?
     AttributeError: Struct instance has no attribute 'key'
     >>> s[key]
     10
 
     Another limitation of the s.key syntax (and Struct(key=val)
     initialization): keys can't be numbers. But numeric keys can be used and
     accessed using the dictionary syntax. Again, an example:
 
     This doesn't work:
     >>> s=Struct(4='hi')
     SyntaxError: keyword can't be an expression
     But this does:
     >>> s=Struct()
     >>> s[4]='hi'
     >>> s
     Struct({4: 'hi'})
     >>> s[4]
     'hi'
     """
 
     # Attributes to which __setitem__ and __setattr__ will block access.
     # Note: much of this will be moot in Python 2.2 and will be done in a much
     # cleaner way.
     __protected = ('copy dict dictcopy get has_attr has_key items keys '
                    'merge popitem setdefault update values '
                    '__make_dict __dict_invert ').split()
 
     def __init__(self,dict=None,**kw):
         """Initialize with a dictionary, another Struct, or by giving
         explicitly the list of attributes.
 
         Both can be used, but the dictionary must come first:
         Struct(dict), Struct(k1=v1,k2=v2) or Struct(dict,k1=v1,k2=v2).
         """
         if dict is None:
             dict = {}
         if isinstance(dict,Struct):
             dict = dict.dict()
         elif dict and  type(dict) is not types.DictType:
             raise TypeError,\
                   'Initialize with a dictionary or key=val pairs.'
         dict.update(kw)
         # do the updating by hand to guarantee that we go through the
         # safety-checked __setitem__
         for k,v in dict.items():
             self[k] = v
 
     def __setitem__(self,key,value):
         """Used when struct[key] = val calls are made."""
         if key in Struct.__protected:
             raise KeyError,'Key '+`key`+' is a protected key of class Struct.'
         self.__dict__[key] = value
 
     def __setattr__(self, key, value):
         """Used when struct.key = val calls are made."""
         self.__setitem__(key,value)
 
     def __str__(self):
         """Gets called by print."""
         
         return 'Struct('+str(self.__dict__)+')'
 
     def __repr__(self):
         """Gets called by repr.
         
         A Struct can be recreated with S_new=eval(repr(S_old))."""
         return 'Struct('+str(self.__dict__)+')'
 
     def __getitem__(self,key):
         """Allows struct[key] access."""
         return self.__dict__[key]
 
     def __contains__(self,key):
         """Allows use of the 'in' operator."""
         return self.__dict__.has_key(key)
 
     def __iadd__(self,other):
         """S += S2 is a shorthand for S.merge(S2)."""
         self.merge(other)
         return self
 
     def __add__(self,other):
         """S + S2 -> New Struct made form S and S.merge(S2)"""
         Sout = self.copy()
         Sout.merge(other)
         return Sout
 
     def __sub__(self,other):
         """Return S1-S2, where all keys in S2 have been deleted (if present)
         from S1."""
         Sout = self.copy()
         Sout -= other
         return Sout
 
     def __isub__(self,other):
         """Do in place S = S - S2, meaning all keys in S2 have been deleted
         (if present) from S1."""
 
         for k in other.keys():
             if self.has_key(k):
                 del self.__dict__[k]
 
     def __make_dict(self,__loc_data__,**kw):
         "Helper function for update and merge. Return a dict from data."
 
         if __loc_data__ == None:
             dict = {}
         elif type(__loc_data__) is types.DictType:
             dict = __loc_data__
         elif isinstance(__loc_data__,Struct):
             dict = __loc_data__.__dict__
         else:
             raise TypeError, 'Update with a dict, a Struct or key=val pairs.'
         if kw:
             dict.update(kw)
         return dict
 
     def __dict_invert(self,dict):
         """Helper function for merge. Takes a dictionary whose values are
         lists and returns a dict. with the elements of each list as keys and
         the original keys as values."""
 
         outdict = {}
         for k,lst in dict.items():
             if type(lst) is types.StringType:
                 lst = lst.split()
             for entry in lst:
                 outdict[entry] = k
         return outdict
 
     def clear(self):
         """Clear all attributes."""
         self.__dict__.clear()
 
     def copy(self):
         """Return a (shallow) copy of a Struct."""
         return Struct(self.__dict__.copy())
 
     def dict(self):
         """Return the Struct's dictionary."""
         return self.__dict__
 
     def dictcopy(self):
         """Return a (shallow) copy of the Struct's dictionary."""
         return self.__dict__.copy()
 
     def popitem(self):
         """S.popitem() -> (k, v), remove and return some (key, value) pair as
         a 2-tuple; but raise KeyError if S is empty."""
         return self.__dict__.popitem()
 
     def update(self,__loc_data__=None,**kw):
         """Update (merge) with data from another Struct or from a dictionary.
         Optionally, one or more key=value pairs can be given at the end for
         direct update."""
 
         # The funny name __loc_data__ is to prevent a common variable name which
         # could be a fieled of a Struct to collide with this parameter. The problem
         # would arise if the function is called with a keyword with this same name
         # that a user means to add as a Struct field.
         newdict = Struct.__make_dict(self,__loc_data__,**kw)
         for k,v in newdict.items():
             self[k] = v
 
     def merge(self,__loc_data__=None,__conflict_solve=None,**kw):
         """S.merge(data,conflict,k=v1,k=v2,...) -> merge data and k=v into S.
 
         This is similar to update(), but much more flexible.  First, a dict is
         made from data+key=value pairs. When merging this dict with the Struct
         S, the optional dictionary 'conflict' is used to decide what to do.
 
         If conflict is not given, the default behavior is to preserve any keys
         with their current value (the opposite of the update method's
         behavior).
 
         conflict is a dictionary of binary functions which will be used to
         solve key conflicts. It must have the following structure:
 
           conflict == { fn1 : [Skey1,Skey2,...], fn2 : [Skey3], etc }
 
         Values must be lists or whitespace separated strings which are
         automatically converted to lists of strings by calling string.split().
 
         Each key of conflict is a function which defines a policy for
         resolving conflicts when merging with the input data. Each fn must be
         a binary function which returns the desired outcome for a key
         conflict. These functions will be called as fn(old,new).
 
         An example is probably in order. Suppose you are merging the struct S
         with a dict D and the following conflict policy dict:
 
             S.merge(D,{fn1:['a','b',4], fn2:'key_c key_d'})
 
         If the key 'a' is found in both S and D, the merge method will call:
 
             S['a'] = fn1(S['a'],D['a'])
 
         As a convenience, merge() provides five (the most commonly needed)
         pre-defined policies: preserve, update, add, add_flip and add_s. The
         easiest explanation is their implementation:
 
           preserve = lambda old,new: old
           update   = lambda old,new: new
           add      = lambda old,new: old + new
           add_flip = lambda old,new: new + old  # note change of order!
           add_s    = lambda old,new: old + ' ' + new  # only works for strings!
 
         You can use those four words (as strings) as keys in conflict instead
         of defining them as functions, and the merge method will substitute
         the appropriate functions for you. That is, the call
 
           S.merge(D,{'preserve':'a b c','add':[4,5,'d'],my_function:[6]})
 
         will automatically substitute the functions preserve and add for the
         names 'preserve' and 'add' before making any function calls.
 
         For more complicated conflict resolution policies, you still need to
         construct your own functions. """
 
         data_dict = Struct.__make_dict(self,__loc_data__,**kw)
 
         # policies for conflict resolution: two argument functions which return
         # the value that will go in the new struct
         preserve = lambda old,new: old
         update   = lambda old,new: new
         add      = lambda old,new: old + new
         add_flip = lambda old,new: new + old  # note change of order!
         add_s    = lambda old,new: old + ' ' + new
 
         # default policy is to keep current keys when there's a conflict
         conflict_solve = list2dict2(self.keys(),default = preserve)
 
         # the conflict_solve dictionary is given by the user 'inverted': we
         # need a name-function mapping, it comes as a function -> names
         # dict. Make a local copy (b/c we'll make changes), replace user
         # strings for the three builtin policies and invert it.
         if __conflict_solve:
             inv_conflict_solve_user = __conflict_solve.copy()
             for name, func in [('preserve',preserve), ('update',update),
                                ('add',add), ('add_flip',add_flip), ('add_s',add_s)]:
                 if name in inv_conflict_solve_user.keys():
                     inv_conflict_solve_user[func] = inv_conflict_solve_user[name]
                     del inv_conflict_solve_user[name]
             conflict_solve.update(Struct.__dict_invert(self,inv_conflict_solve_user))
         #print 'merge. conflict_solve: '; pprint(conflict_solve) # dbg
-        # after Python 2.2, use iterators: for key in data_dict will then work
         #print '*'*50,'in merger. conflict_solver:';  pprint(conflict_solve)
-        for key in data_dict.keys():
+        for key in data_dict:
             if key not in self:
                 self[key] = data_dict[key]
             else:
                 self[key] = conflict_solve[key](self[key],data_dict[key])
 
     def has_key(self,key):
         """Like has_key() dictionary method."""
         return self.__dict__.has_key(key)
 
     def hasattr(self,key):
         """hasattr function available as a method.
 
         Implemented like has_key, to make sure that all available keys in the
         internal dictionary of the Struct appear also as attributes (even
         numeric keys)."""
         return self.__dict__.has_key(key)
 
     def items(self):
         """Return the items in the Struct's dictionary, in the same format
         as a call to {}.items()."""
         return self.__dict__.items()
 
     def keys(self):
         """Return the keys in the Struct's dictionary, in the same format
         as a call to {}.keys()."""
         return self.__dict__.keys()
 
     def values(self,keys=None):
         """Return the values in the Struct's dictionary, in the same format
         as a call to {}.values().
 
         Can be called with an optional argument keys, which must be a list or
         tuple of keys. In this case it returns only the values corresponding
         to those keys (allowing a form of 'slicing' for Structs)."""
         if not keys:
             return self.__dict__.values()
         else:
             ret=[]
             for k in keys:
                 ret.append(self[k])
             return ret
 
     def get(self,attr,val=None):
         """S.get(k[,d]) -> S[k] if S.has_key(k), else d.  d defaults to None."""
         try:
             return self[attr]
         except KeyError:
             return val
 
     def setdefault(self,attr,val=None):
         """S.setdefault(k[,d]) -> S.get(k,d), also set S[k]=d if not S.has_key(k)"""
         if not self.has_key(attr):
             self[attr] = val
         return self.get(attr,val)
 # end class Struct
 

IPython/background_jobs.py

@@ -1,504 +1,495 b''
 # -*- coding: utf-8 -*-
 """Manage background (threaded) jobs conveniently from an interactive shell.
 
 This module provides a BackgroundJobManager class.  This is the main class
 meant for public usage, it implements an object which can create and manage
 new background jobs.
 
 It also provides the actual job classes managed by these BackgroundJobManager
 objects, see their docstrings below.
 
 
 This system was inspired by discussions with B. Granger and the
 BackgroundCommand class described in the book Python Scripting for
 Computational Science, by H. P. Langtangen:
 
 http://folk.uio.no/hpl/scripting
 
 (although ultimately no code from this text was used, as IPython's system is a
 separate implementation).
 
-$Id: background_jobs.py 515 2005-02-15 07:41:41Z fperez $
+$Id: background_jobs.py 638 2005-07-18 03:01:41Z fperez $
 """
 
 #*****************************************************************************
 #       Copyright (C) 2005 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Fernando']
 __license__ = Release.license
 
 # Code begins
 import threading,sys
 
 from IPython.ultraTB import AutoFormattedTB
 from IPython.genutils import warn,error
 
-# declares Python 2.2 compatibility symbols:
-try:
-    basestring
-except NameError:
-    import types
-    basestring = (types.StringType, types.UnicodeType)
-    True = 1==1
-    False = 1==0
-
 class BackgroundJobManager:
     """Class to manage a pool of backgrounded threaded jobs.
 
     Below, we assume that 'jobs' is a BackgroundJobManager instance.
     
     Usage summary (see the method docstrings for details):
 
       jobs.new(...) -> start a new job
       
       jobs() or jobs.status() -> print status summary of all jobs
 
       jobs[N] -> returns job number N.
 
       foo = jobs[N].result -> assign to variable foo the result of job N
 
       jobs[N].traceback() -> print the traceback of dead job N
 
       jobs.remove(N) -> remove (finished) job N
 
       jobs.flush_finished() -> remove all finished jobs
       
     As a convenience feature, BackgroundJobManager instances provide the
     utility result and traceback methods which retrieve the corresponding
     information from the jobs list:
 
       jobs.result(N) <--> jobs[N].result
       jobs.traceback(N) <--> jobs[N].traceback()
 
     While this appears minor, it allows you to use tab completion
     interactively on the job manager instance.
 
     In interactive mode, IPython provides the magic fuction %bg for quick
     creation of backgrounded expression-based jobs. Type bg? for details."""
 
     def __init__(self):
         # Lists for job management
         self.jobs_run  = []
         self.jobs_comp = []
         self.jobs_dead = []
         # A dict of all jobs, so users can easily access any of them
         self.jobs_all = {}
         # For reporting
         self._comp_report = []
         self._dead_report = []
         # Store status codes locally for fast lookups
         self._s_created   = BackgroundJobBase.stat_created_c
         self._s_running   = BackgroundJobBase.stat_running_c
         self._s_completed = BackgroundJobBase.stat_completed_c
         self._s_dead      = BackgroundJobBase.stat_dead_c
 
     def new(self,func_or_exp,*args,**kwargs):
         """Add a new background job and start it in a separate thread.
 
         There are two types of jobs which can be created:
 
         1. Jobs based on expressions which can be passed to an eval() call.
         The expression must be given as a string.  For example:
 
           job_manager.new('myfunc(x,y,z=1)'[,glob[,loc]])
 
         The given expression is passed to eval(), along with the optional
         global/local dicts provided.  If no dicts are given, they are
         extracted automatically from the caller's frame.
         
         A Python statement is NOT a valid eval() expression.  Basically, you
         can only use as an eval() argument something which can go on the right
         of an '=' sign and be assigned to a variable.
 
         For example,"print 'hello'" is not valid, but '2+3' is.
 
         2. Jobs given a function object, optionally passing additional
         positional arguments:
 
           job_manager.new(myfunc,x,y)
 
         The function is called with the given arguments.
 
         If you need to pass keyword arguments to your function, you must
         supply them as a dict named kw:
 
           job_manager.new(myfunc,x,y,kw=dict(z=1))
 
         The reason for this assymmetry is that the new() method needs to
         maintain access to its own keywords, and this prevents name collisions
         between arguments to new() and arguments to your own functions.
 
         In both cases, the result is stored in the job.result field of the
         background job object.
 
 
         Notes and caveats:
 
         1. All threads running share the same standard output.  Thus, if your
         background jobs generate output, it will come out on top of whatever
         you are currently writing.  For this reason, background jobs are best
         used with silent functions which simply return their output.
 
         2. Threads also all work within the same global namespace, and this
         system does not lock interactive variables.  So if you send job to the
         background which operates on a mutable object for a long time, and
         start modifying that same mutable object interactively (or in another
         backgrounded job), all sorts of bizarre behaviour will occur.
 
         3. If a background job is spending a lot of time inside a C extension
         module which does not release the Python Global Interpreter Lock
         (GIL), this will block the IPython prompt.  This is simply because the
         Python interpreter can only switch between threads at Python
         bytecodes.  While the execution is inside C code, the interpreter must
         simply wait unless the extension module releases the GIL.
 
         4. There is no way, due to limitations in the Python threads library,
         to kill a thread once it has started."""
         
         if callable(func_or_exp):
             kw  = kwargs.get('kw',{})
             job = BackgroundJobFunc(func_or_exp,*args,**kw)
         elif isinstance(func_or_exp,basestring):
             if not args:
                 frame = sys._getframe(1)
                 glob, loc = frame.f_globals, frame.f_locals
             elif len(args)==1:
                 glob = loc = args[0]
             elif len(args)==2:
                 glob,loc = args
             else:
                 raise ValueError,\
                       'Expression jobs take at most 2 args (globals,locals)'
             job = BackgroundJobExpr(func_or_exp,glob,loc)
         else:
             raise
         jkeys = self.jobs_all.keys()
         if jkeys:
             job.num = max(jkeys)+1
         else:
             job.num = 0
         self.jobs_run.append(job)
         self.jobs_all[job.num] = job
         print 'Starting job # %s in a separate thread.' % job.num
         job.start()
         return job
 
     def __getitem__(self,key):
         return self.jobs_all[key]
 
     def __call__(self):
         """An alias to self.status(),
 
         This allows you to simply call a job manager instance much like the
         Unix jobs shell command."""
 
         return self.status()
 
     def _update_status(self):
         """Update the status of the job lists.
 
         This method moves finished jobs to one of two lists:
           - self.jobs_comp: jobs which completed successfully
           - self.jobs_dead: jobs which finished but died.
 
         It also copies those jobs to corresponding _report lists.  These lists
         are used to report jobs completed/dead since the last update, and are
         then cleared by the reporting function after each call."""
         
         run,comp,dead = self._s_running,self._s_completed,self._s_dead
         jobs_run = self.jobs_run
         for num in range(len(jobs_run)):
             job  = jobs_run[num]
             stat = job.stat_code
             if stat == run:
                 continue
             elif stat == comp:
                 self.jobs_comp.append(job)
                 self._comp_report.append(job)
                 jobs_run[num] = False
             elif stat == dead:
                 self.jobs_dead.append(job)
                 self._dead_report.append(job)
                 jobs_run[num] = False
         self.jobs_run = filter(None,self.jobs_run)
 
     def _group_report(self,group,name):
         """Report summary for a given job group.
 
         Return True if the group had any elements."""
 
         if group:
             print '%s jobs:' % name
             for job in group:
                 print '%s : %s' % (job.num,job)
             print
             return True
 
     def _group_flush(self,group,name):
         """Flush a given job group
 
         Return True if the group had any elements."""
 
         njobs = len(group)
         if njobs:
             plural = {1:''}.setdefault(njobs,'s')
             print 'Flushing %s %s job%s.' % (njobs,name,plural)
             group[:] = []
             return True
         
     def _status_new(self):
         """Print the status of newly finished jobs.
 
         Return True if any new jobs are reported.
 
         This call resets its own state every time, so it only reports jobs
         which have finished since the last time it was called."""
 
         self._update_status()
         new_comp = self._group_report(self._comp_report,'Completed')
         new_dead = self._group_report(self._dead_report,
                                       'Dead, call job.traceback() for details')
         self._comp_report[:] = []
         self._dead_report[:] = []
         return new_comp or new_dead
                 
     def status(self,verbose=0):
         """Print a status of all jobs currently being managed."""
 
         self._update_status()
         self._group_report(self.jobs_run,'Running')
         self._group_report(self.jobs_comp,'Completed')
         self._group_report(self.jobs_dead,'Dead')
         # Also flush the report queues
         self._comp_report[:] = []
         self._dead_report[:] = []
 
     def remove(self,num):
         """Remove a finished (completed or dead) job."""
 
         try:
             job = self.jobs_all[num]
         except KeyError:
             error('Job #%s not found' % num)
         else:
             stat_code = job.stat_code
             if stat_code == self._s_running:
                 error('Job #%s is still running, it can not be removed.' % num)
                 return
             elif stat_code == self._s_completed:
                 self.jobs_comp.remove(job)
             elif stat_code == self._s_dead:
                 self.jobs_dead.remove(job)
 
     def flush_finished(self):
         """Flush all jobs finished (completed and dead) from lists.
 
         Running jobs are never flushed.
 
         It first calls _status_new(), to update info. If any jobs have
         completed since the last _status_new() call, the flush operation
         aborts."""
 
         if self._status_new():
             error('New jobs completed since last '\
                   '_status_new(), aborting flush.')
             return
 
         # Remove the finished jobs from the master dict
         jobs_all = self.jobs_all
         for job in self.jobs_comp+self.jobs_dead:
             del(jobs_all[job.num])
 
         # Now flush these lists completely
         fl_comp = self._group_flush(self.jobs_comp,'Completed')
         fl_dead = self._group_flush(self.jobs_dead,'Dead')
         if not (fl_comp or fl_dead):
             print 'No jobs to flush.'
 
     def result(self,num):
         """result(N) -> return the result of job N."""
         try:
             return self.jobs_all[num].result
         except KeyError:
             error('Job #%s not found' % num)
 
     def traceback(self,num):
         try:
             self.jobs_all[num].traceback()
         except KeyError:
             error('Job #%s not found' % num)
 
 
 class BackgroundJobBase(threading.Thread):
     """Base class to build BackgroundJob classes.
 
     The derived classes must implement:
 
     - Their own __init__, since the one here raises NotImplementedError.  The
     derived constructor must call self._init() at the end, to provide common
     initialization.
 
     - A strform attribute used in calls to __str__.
 
     - A call() method, which will make the actual execution call and must
     return a value to be held in the 'result' field of the job object."""
 
     # Class constants for status, in string and as numerical codes (when
     # updating jobs lists, we don't want to do string comparisons).  This will
     # be done at every user prompt, so it has to be as fast as possible
     stat_created   = 'Created'; stat_created_c = 0
     stat_running   = 'Running'; stat_running_c = 1
     stat_completed = 'Completed'; stat_completed_c = 2
     stat_dead      = 'Dead (Exception), call job.traceback() for details'
     stat_dead_c = -1
 
     def __init__(self):
         raise NotImplementedError, \
               "This class can not be instantiated directly."
 
     def _init(self):
         """Common initialization for all BackgroundJob objects"""
         
         for attr in ['call','strform']:
             assert hasattr(self,attr), "Missing attribute <%s>" % attr
         
         # The num tag can be set by an external job manager
         self.num = None
       
         self.status    = BackgroundJobBase.stat_created
         self.stat_code = BackgroundJobBase.stat_created_c
         self.finished  = False
         self.result    = '<BackgroundJob has not completed>'
         # reuse the ipython traceback handler if we can get to it, otherwise
         # make a new one
         try:
             self._make_tb = __IPYTHON__.InteractiveTB.text
         except:
             self._make_tb = AutoFormattedTB(mode = 'Context',
                                            color_scheme='NoColor',
                                            tb_offset = 1).text
         # Hold a formatted traceback if one is generated.
         self._tb = None
         
         threading.Thread.__init__(self)
 
     def __str__(self):
         return self.strform
 
     def __repr__(self):
         return '<BackgroundJob: %s>' % self.strform
 
     def traceback(self):
         print self._tb
         
     def run(self):
         try:
             self.status    = BackgroundJobBase.stat_running
             self.stat_code = BackgroundJobBase.stat_running_c
             self.result    = self.call()
         except:
             self.status    = BackgroundJobBase.stat_dead
             self.stat_code = BackgroundJobBase.stat_dead_c
             self.finished  = None
             self.result    = ('<BackgroundJob died, call job.traceback() for details>')
             self._tb       = self._make_tb()
         else:
             self.status    = BackgroundJobBase.stat_completed
             self.stat_code = BackgroundJobBase.stat_completed_c
             self.finished  = True
 
 class BackgroundJobExpr(BackgroundJobBase):
     """Evaluate an expression as a background job (uses a separate thread)."""
 
     def __init__(self,expression,glob=None,loc=None):
         """Create a new job from a string which can be fed to eval().
 
         global/locals dicts can be provided, which will be passed to the eval
         call."""
 
         # fail immediately if the given expression can't be compiled
         self.code = compile(expression,'<BackgroundJob compilation>','eval')
                 
         if glob is None:
             glob = {}
         if loc is None:
             loc = {}
             
         self.expression = self.strform = expression
         self.glob = glob
         self.loc = loc
         self._init()
         
     def call(self):
         return eval(self.code,self.glob,self.loc)
 
 class BackgroundJobFunc(BackgroundJobBase):
     """Run a function call as a background job (uses a separate thread)."""
 
     def __init__(self,func,*args,**kwargs):
         """Create a new job from a callable object.
 
         Any positional arguments and keyword args given to this constructor
         after the initial callable are passed directly to it."""
 
         assert callable(func),'first argument must be callable'
         
         if args is None:
             args = []
         if kwargs is None:
             kwargs = {}
         
         self.func = func
         self.args = args
         self.kwargs = kwargs
         # The string form will only include the function passed, because
         # generating string representations of the arguments is a potentially
         # _very_ expensive operation (e.g. with large arrays).
         self.strform = str(func)
         self._init()
 
     def call(self):
         return self.func(*self.args,**self.kwargs)
 
 
 if __name__=='__main__':
 
     import time
 
     def sleepfunc(interval=2,*a,**kw):
         args = dict(interval=interval,
                     args=a,
                     kwargs=kw)
         time.sleep(interval)
         return args
 
     def diefunc(interval=2,*a,**kw):
         time.sleep(interval)
         die
 
     def printfunc(interval=1,reps=5):
         for n in range(reps):
             time.sleep(interval)
             print 'In the background...'
 
     jobs = BackgroundJobManager()
     # first job will have # 0
     jobs.new(sleepfunc,4)
     jobs.new(sleepfunc,kw={'reps':2})
     # This makes a job which will die
     jobs.new(diefunc,1)
     jobs.new('printfunc(1,3)')
 
     # after a while, you can get the traceback of a dead job.  Run the line
     # below again interactively until it prints a traceback (check the status
     # of the job):
     print jobs[1].status
     jobs[1].traceback()
     
     # Run this line again until the printed result changes
     print "The result of job #0 is:",jobs[0].result

IPython/genutils.py

@@ -1,1519 +1,1508 b''
 # -*- coding: utf-8 -*-
 """
 General purpose utilities.
 
 This is a grab-bag of stuff I find useful in most programs I write. Some of
 these things are also convenient when working at the command line.
 
-$Id: genutils.py 633 2005-07-17 01:03:15Z tzanko $"""
+$Id: genutils.py 638 2005-07-18 03:01:41Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2004 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Fernando']
 __license__ = Release.license
 
 #****************************************************************************
 # required modules
 import __main__
 import types,commands,time,sys,os,re,shutil
 import tempfile
+import codecs
 from IPython.Itpl import Itpl,itpl,printpl
 from IPython import DPyGetOpt
 
+# Build objects which appeared in Python 2.3 for 2.2, to make ipython
+# 2.2-friendly
+try:
+    basestring
+except NameError:
+    import types
+    basestring = (types.StringType, types.UnicodeType)
+    True = 1==1
+    False = 1==0
+
+    def enumerate(obj):
+        i = -1
+        for item in obj:
+            i += 1
+            yield i, item
+
+    # add these to the builtin namespace, so that all modules find them
+    import __builtin__
+    __builtin__.basestring = basestring
+    __builtin__.True = True
+    __builtin__.False = False
+    __builtin__.enumerate = enumerate
+
 #****************************************************************************
 # Exceptions
 class Error(Exception):
     """Base class for exceptions in this module."""
     pass
 
 #----------------------------------------------------------------------------
-class Stream:
-    """Simple class to hold the various I/O streams in Term"""
-
-    def __init__(self,stream,name):
+class IOStream:
+    def __init__(self,stream,fallback):
+        if not hasattr(stream,'write') or not hasattr(stream,'flush'):
+            stream = fallback
         self.stream = stream
-        self.name = name
+        self._swrite = stream.write
+        self.flush = stream.flush
+
+    def write(self,data):
         try:
-            self.fileno = stream.fileno()
-        except AttributeError:
-            msg = ("Stream <%s> looks suspicious: it lacks a 'fileno' attribute."
-                   % name)
-            print >> sys.stderr, 'WARNING:',msg
+            self._swrite(data)
+        except:
         try:
-            self.mode = stream.mode
-        except AttributeError:
-            msg = ("Stream <%s> looks suspicious: it lacks a 'mode' attribute."
-                   % name)
-            print >> sys.stderr, 'WARNING:',msg
+                # print handles some unicode issues which may trip a plain
+                # write() call.  Attempt to emulate write() by using a
+                # trailing comma
+                print >> self.stream, data,
+            except:
+                # if we get here, something is seriously broken.
+                print >> sys.stderr, \
+                      'ERROR - failed to write data to stream:', stream
 
-class Term:
+class IOTerm:
     """ Term holds the file or file-like objects for handling I/O operations.
 
     These are normally just sys.stdin, sys.stdout and sys.stderr but for
     Windows they can can replaced to allow editing the strings before they are
     displayed."""
 
     # In the future, having IPython channel all its I/O operations through
     # this class will make it easier to embed it into other environments which
     # are not a normal terminal (such as a GUI-based shell)
-    in_s  = Stream(sys.stdin,'cin')
-    out_s = Stream(sys.stdout,'cout')
-    err_s = Stream(sys.stderr,'cerr')
-
-    # Store the three streams in (err,out,in) order so that if we need to reopen
-    # them, the error channel is reopened first to provide info.
-    streams = [err_s,out_s,in_s]
-
-    # The class globals should be the actual 'bare' streams for normal I/O to work
-    cin  = streams[2].stream
-    cout = streams[1].stream
-    cerr = streams[0].stream
-    
-    def reopen_all(cls):
-        """Reopen all streams if necessary.
-
-        This should only be called if it is suspected that someting closed
-        accidentally one of the I/O streams."""
-
-        any_closed = 0
-
-        for sn in range(len(cls.streams)):
-            st = cls.streams[sn]
-            if st.stream.closed:
-                any_closed = 1
-                new_stream = os.fdopen(os.dup(st.fileno), st.mode,0)
-                cls.streams[sn] = Stream(new_stream,st.name)
-                print >> cls.streams[0].stream, \
-                      '\nWARNING:\nStream Term.%s had to be reopened!' % st.name
-
-        # Rebuild the class globals
-        cls.cin = cls.streams[2].stream
-        cls.cout = cls.streams[1].stream
-        cls.cerr = cls.streams[0].stream
-
-    reopen_all = classmethod(reopen_all)
-
-    def set_stdout(cls,stream):
-        """Set the stream """
-        cls.cout = stream
-    set_stdout = classmethod(set_stdout)
+    def __init__(self,cin=None,cout=None,cerr=None):
+        self.cin  = IOStream(cin,sys.stdin)
+        self.cout = IOStream(cout,sys.stdout)
+        self.cerr = IOStream(cerr,sys.stderr)
 
-    def set_stderr(cls,stream):
-        cls.cerr = stream
-    set_stderr = classmethod(set_stderr)
+# Global variable to be used for all I/O
+Term = IOTerm()
 
 # Windows-specific code to load Gary Bishop's readline and configure it
 # automatically for the users
 # Note: os.name on cygwin returns posix, so this should only pick up 'native'
 # windows.  Cygwin returns 'cygwin' for sys.platform.
 if os.name == 'nt':
     try:
         import readline
     except ImportError:
         pass
     else:
         try:
             _out = readline.GetOutputFile()
         except AttributeError:
             pass
         else:
-            Term.set_stdout(_out)
-            Term.set_stderr(_out)
+            # Remake Term to use the readline i/o facilities
+            Term = IOTerm(cout=_out,cerr=_out)
             del _out
 
 #****************************************************************************
 # Generic warning/error printer, used by everything else
 def warn(msg,level=2,exit_val=1):
     """Standard warning printer. Gives formatting consistency.
 
     Output is sent to Term.cerr (sys.stderr by default).
     
     Options:
     
     -level(2): allows finer control:
       0 -> Do nothing, dummy function.
       1 -> Print message.
       2 -> Print 'WARNING:' + message. (Default level).
       3 -> Print 'ERROR:' + message.
       4 -> Print 'FATAL ERROR:' + message and trigger a sys.exit(exit_val).
 
     -exit_val (1): exit value returned by sys.exit() for a level 4
     warning. Ignored for all other levels."""
     
     if level>0:
         header = ['','','WARNING: ','ERROR: ','FATAL ERROR: ']
         print >> Term.cerr, '%s%s' % (header[level],msg)
         if level == 4:
             print >> Term.cerr,'Exiting.\n'
             sys.exit(exit_val)
 
 def info(msg):
     """Equivalent to warn(msg,level=1)."""
 
     warn(msg,level=1)
 
 def error(msg):
     """Equivalent to warn(msg,level=3)."""
 
     warn(msg,level=3)
 
 def fatal(msg,exit_val=1):
     """Equivalent to warn(msg,exit_val=exit_val,level=4)."""
 
     warn(msg,exit_val=exit_val,level=4)
 
 #----------------------------------------------------------------------------
 StringTypes = types.StringTypes
 
 # Basic timing functionality
 
 # If possible (Unix), use the resource module instead of time.clock()
 try:
     import resource
     def clock():
         """clock() -> floating point number
 
         Return the CPU time in seconds (user time only, system time is
         ignored) since the start of the process.  This is done via a call to
         resource.getrusage, so it avoids the wraparound problems in
         time.clock()."""
         
         return resource.getrusage(resource.RUSAGE_SELF)[0]
     
     def clock2():
         """clock2() -> (t_user,t_system)
 
         Similar to clock(), but return a tuple of user/system times."""
         return resource.getrusage(resource.RUSAGE_SELF)[:2]
 
 except ImportError:
     clock = time.clock
     def clock2():
         """Under windows, system CPU time can't be measured.
 
         This just returns clock() and zero."""
         return time.clock(),0.0
 
 def timings_out(reps,func,*args,**kw):
     """timings_out(reps,func,*args,**kw) -> (t_total,t_per_call,output)
 
     Execute a function reps times, return a tuple with the elapsed total
     CPU time in seconds, the time per call and the function's output.
 
     Under Unix, the return value is the sum of user+system time consumed by
     the process, computed via the resource module.  This prevents problems
     related to the wraparound effect which the time.clock() function has.
     
     Under Windows the return value is in wall clock seconds. See the
     documentation for the time module for more details."""
 
     reps = int(reps)
     assert reps >=1, 'reps must be >= 1'
     if reps==1:
         start = clock()
         out = func(*args,**kw)
         tot_time = clock()-start
     else:
         rng = xrange(reps-1) # the last time is executed separately to store output
         start = clock()
         for dummy in rng: func(*args,**kw)
         out = func(*args,**kw)  # one last time
         tot_time = clock()-start
     av_time = tot_time / reps
     return tot_time,av_time,out
 
 def timings(reps,func,*args,**kw):
     """timings(reps,func,*args,**kw) -> (t_total,t_per_call)
 
     Execute a function reps times, return a tuple with the elapsed total CPU
     time in seconds and the time per call. These are just the first two values
     in timings_out()."""
 
     return timings_out(reps,func,*args,**kw)[0:2]
 
 def timing(func,*args,**kw):
     """timing(func,*args,**kw) -> t_total
 
     Execute a function once, return the elapsed total CPU time in
     seconds. This is just the first value in timings_out()."""
 
     return timings_out(1,func,*args,**kw)[0]
 
 #****************************************************************************
 # file and system
 
 def system(cmd,verbose=0,debug=0,header=''):
     """Execute a system command, return its exit status.
 
     Options:
 
     - verbose (0): print the command to be executed.
     
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     stat = 0
     if verbose or debug: print header+cmd
     sys.stdout.flush()
     if not debug: stat = os.system(cmd)
     return stat
 
 def shell(cmd,verbose=0,debug=0,header=''):
     """Execute a command in the system shell, always return None.
 
     Options:
 
     - verbose (0): print the command to be executed.
     
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: this is similar to genutils.system(), but it returns None so it can
     be conveniently used in interactive loops without getting the return value
     (typically 0) printed many times."""
 
     stat = 0
     if verbose or debug: print header+cmd
     # flush stdout so we don't mangle python's buffering
     sys.stdout.flush()
     if not debug:
         os.system(cmd)
 
 def getoutput(cmd,verbose=0,debug=0,header='',split=0):
     """Dummy substitute for perl's backquotes.
 
     Executes a command and returns the output.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, the output is returned as a list split on newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     if verbose or debug: print header+cmd
     if not debug:
         output = commands.getoutput(cmd)
         if split:
             return output.split('\n')
         else:
             return output
 
 def getoutputerror(cmd,verbose=0,debug=0,header='',split=0):
     """Return (standard output,standard error) of executing cmd in a shell.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, each of stdout/err is returned as a list split on
     newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     if verbose or debug: print header+cmd
     if not cmd:
         if split:
             return [],[]
         else:
             return '',''
     if not debug:
         pin,pout,perr = os.popen3(cmd)
         tout = pout.read().rstrip()
         terr = perr.read().rstrip()
         pin.close()
         pout.close()
         perr.close()
         if split:
             return tout.split('\n'),terr.split('\n')
         else:
             return tout,terr
 
 # for compatibility with older naming conventions
 xsys = system
 bq = getoutput
 
 class SystemExec:
     """Access the system and getoutput functions through a stateful interface.
 
     Note: here we refer to the system and getoutput functions from this
     library, not the ones from the standard python library.
     
     This class offers the system and getoutput functions as methods, but the
     verbose, debug and header parameters can be set for the instance (at
     creation time or later) so that they don't need to be specified on each
     call.
 
     For efficiency reasons, there's no way to override the parameters on a
     per-call basis other than by setting instance attributes. If you need
     local overrides, it's best to directly call system() or getoutput().
 
     The following names are provided as alternate options:
      - xsys: alias to system
      - bq: alias to getoutput
 
     An instance can then be created as:
      >>> sysexec = SystemExec(verbose=1,debug=0,header='Calling: ')
 
     And used as:
      >>> sysexec.xsys('pwd')
      >>> dirlist = sysexec.bq('ls -l')
     """
     
     def __init__(self,verbose=0,debug=0,header='',split=0):
         """Specify the instance's values for verbose, debug and header."""
         setattr_list(self,'verbose debug header split')
 
     def system(self,cmd):
         """Stateful interface to system(), with the same keyword parameters."""
 
         system(cmd,self.verbose,self.debug,self.header)
 
     def shell(self,cmd):
         """Stateful interface to shell(), with the same keyword parameters."""
 
         shell(cmd,self.verbose,self.debug,self.header)
 
     xsys = system  # alias
 
     def getoutput(self,cmd):
         """Stateful interface to getoutput()."""
 
         return getoutput(cmd,self.verbose,self.debug,self.header,self.split)
 
     def getoutputerror(self,cmd):
         """Stateful interface to getoutputerror()."""
 
         return getoutputerror(cmd,self.verbose,self.debug,self.header,self.split)
 
     bq = getoutput  # alias
 
 #-----------------------------------------------------------------------------
 def mutex_opts(dict,ex_op):
     """Check for presence of mutually exclusive keys in a dict.
 
     Call: mutex_opts(dict,[[op1a,op1b],[op2a,op2b]...]"""
     for op1,op2 in ex_op:
         if op1 in dict and op2 in dict:
             raise ValueError,'\n*** ERROR in Arguments *** '\
                   'Options '+op1+' and '+op2+' are mutually exclusive.'
 
 #-----------------------------------------------------------------------------
 def filefind(fname,alt_dirs = None):
     """Return the given filename either in the current directory, if it
     exists, or in a specified list of directories.
 
     ~ expansion is done on all file and directory names.
 
     Upon an unsuccessful search, raise an IOError exception."""
 
     if alt_dirs is None:
         try:
             alt_dirs = get_home_dir()
         except HomeDirError:
             alt_dirs = os.getcwd()
     search = [fname] + list_strings(alt_dirs)
     search = map(os.path.expanduser,search)
     #print 'search list for',fname,'list:',search  # dbg
     fname = search[0]
     if os.path.isfile(fname):
         return fname
     for direc in search[1:]:
         testname = os.path.join(direc,fname)
         #print 'testname',testname  # dbg
         if os.path.isfile(testname):
             return testname
     raise IOError,'File' + `fname` + \
           ' not found in current or supplied directories:' + `alt_dirs`
 
 #----------------------------------------------------------------------------
 def target_outdated(target,deps):
     """Determine whether a target is out of date.
 
     target_outdated(target,deps) -> 1/0
 
     deps: list of filenames which MUST exist.
     target: single filename which may or may not exist.
 
     If target doesn't exist or is older than any file listed in deps, return
     true, otherwise return false.
     """
     try:
         target_time = os.path.getmtime(target)
     except os.error:
         return 1
     for dep in deps:
         dep_time = os.path.getmtime(dep)
         if dep_time > target_time:
             #print "For target",target,"Dep failed:",dep # dbg
             #print "times (dep,tar):",dep_time,target_time # dbg
             return 1
     return 0
 
 #-----------------------------------------------------------------------------
 def target_update(target,deps,cmd):
     """Update a target with a given command given a list of dependencies.
 
     target_update(target,deps,cmd) -> runs cmd if target is outdated.
 
     This is just a wrapper around target_outdated() which calls the given
     command if target is outdated."""
 
     if target_outdated(target,deps):
         xsys(cmd)
 
 #----------------------------------------------------------------------------
 def unquote_ends(istr):
     """Remove a single pair of quotes from the endpoints of a string."""
 
     if not istr:
         return istr
     if (istr[0]=="'" and istr[-1]=="'") or \
        (istr[0]=='"' and istr[-1]=='"'):
         return istr[1:-1]
     else:
         return istr
 
 #----------------------------------------------------------------------------
 def process_cmdline(argv,names=[],defaults={},usage=''):
     """ Process command-line options and arguments.
 
     Arguments:
 
     - argv: list of arguments, typically sys.argv.
     
     - names: list of option names. See DPyGetOpt docs for details on options
     syntax.
 
     - defaults: dict of default values.
 
     - usage: optional usage notice to print if a wrong argument is passed.
 
     Return a dict of options and a list of free arguments."""
 
     getopt = DPyGetOpt.DPyGetOpt()
     getopt.setIgnoreCase(0)
     getopt.parseConfiguration(names)
 
     try:
         getopt.processArguments(argv)
     except:
         print usage
         warn(`sys.exc_value`,level=4)
 
     defaults.update(getopt.optionValues)
     args = getopt.freeValues
     
     return defaults,args
 
 #----------------------------------------------------------------------------
 def optstr2types(ostr):
     """Convert a string of option names to a dict of type mappings.
 
     optstr2types(str) -> {None:'string_opts',int:'int_opts',float:'float_opts'}
 
     This is used to get the types of all the options in a string formatted
     with the conventions of DPyGetOpt. The 'type' None is used for options
     which are strings (they need no further conversion). This function's main
     use is to get a typemap for use with read_dict().
     """
 
     typeconv = {None:'',int:'',float:''}
     typemap = {'s':None,'i':int,'f':float}
     opt_re = re.compile(r'([\w]*)([^:=]*:?=?)([sif]?)')
 
     for w in ostr.split():
         oname,alias,otype = opt_re.match(w).groups()
         if otype == '' or alias == '!':   # simple switches are integers too
             otype = 'i'
         typeconv[typemap[otype]] += oname + ' '
     return typeconv
 
 #----------------------------------------------------------------------------
 def read_dict(filename,type_conv=None,**opt):
 
     """Read a dictionary of key=value pairs from an input file, optionally
     performing conversions on the resulting values.
 
     read_dict(filename,type_conv,**opt) -> dict
 
     Only one value per line is accepted, the format should be
      # optional comments are ignored
      key value\n
 
     Args:
 
       - type_conv: A dictionary specifying which keys need to be converted to
       which types. By default all keys are read as strings. This dictionary
       should have as its keys valid conversion functions for strings
       (int,long,float,complex, or your own).  The value for each key
       (converter) should be a whitespace separated string containing the names
       of all the entries in the file to be converted using that function. For
       keys to be left alone, use None as the conversion function (only needed
       with purge=1, see below).
 
       - opt: dictionary with extra options as below (default in parens)
 
         purge(0): if set to 1, all keys *not* listed in type_conv are purged out
         of the dictionary to be returned. If purge is going to be used, the
         set of keys to be left as strings also has to be explicitly specified
         using the (non-existent) conversion function None.
 
         fs(None): field separator. This is the key/value separator to be used
         when parsing the file. The None default means any whitespace [behavior
         of string.split()].
 
         strip(0): if 1, strip string values of leading/trailinig whitespace.
 
         warn(1): warning level if requested keys are not found in file.
           - 0: silently ignore.
           - 1: inform but proceed.
           - 2: raise KeyError exception.
 
         no_empty(0): if 1, remove keys with whitespace strings as a value.
 
         unique([]): list of keys (or space separated string) which can't be
         repeated. If one such key is found in the file, each new instance
         overwrites the previous one. For keys not listed here, the behavior is
         to make a list of all appearances.
 
     Example:
     If the input file test.ini has:
       i 3
       x 4.5
       y 5.5
       s hi ho
     Then:
 
     >>> type_conv={int:'i',float:'x',None:'s'}
     >>> read_dict('test.ini')
     {'i': '3', 's': 'hi ho', 'x': '4.5', 'y': '5.5'}
     >>> read_dict('test.ini',type_conv)
     {'i': 3, 's': 'hi ho', 'x': 4.5, 'y': '5.5'}
     >>> read_dict('test.ini',type_conv,purge=1)
     {'i': 3, 's': 'hi ho', 'x': 4.5}
     """
 
     # starting config
     opt.setdefault('purge',0)
     opt.setdefault('fs',None)  # field sep defaults to any whitespace
     opt.setdefault('strip',0)
     opt.setdefault('warn',1)
     opt.setdefault('no_empty',0)
     opt.setdefault('unique','')
     if type(opt['unique']) in StringTypes:
         unique_keys = qw(opt['unique'])
     elif type(opt['unique']) in (types.TupleType,types.ListType):
         unique_keys = opt['unique']
     else:
         raise ValueError, 'Unique keys must be given as a string, List or Tuple'
 
     dict = {}
     # first read in table of values as strings
     file = open(filename,'r')
     for line in file.readlines():
         line = line.strip()
         if len(line) and line[0]=='#': continue
         if len(line)>0:
             lsplit = line.split(opt['fs'],1)
             try:
                 key,val = lsplit
             except ValueError:
                 key,val = lsplit[0],''
             key = key.strip()
             if opt['strip']: val = val.strip()
             if val == "''" or val == '""': val = ''
             if opt['no_empty'] and (val=='' or val.isspace()):
                 continue
             # if a key is found more than once in the file, build a list
             # unless it's in the 'unique' list. In that case, last found in file
             # takes precedence. User beware.
             try:
                 if dict[key] and key in unique_keys:
                     dict[key] = val
                 elif type(dict[key]) is types.ListType:
                     dict[key].append(val)
                 else:
                     dict[key] = [dict[key],val]
             except KeyError:
                 dict[key] = val
     # purge if requested
     if opt['purge']:
         accepted_keys = qwflat(type_conv.values())
         for key in dict.keys():
             if key in accepted_keys: continue
             del(dict[key])
     # now convert if requested
     if type_conv==None: return dict
     conversions = type_conv.keys()
     try: conversions.remove(None)
     except: pass
     for convert in conversions:
         for val in qw(type_conv[convert]):
             try:
                 dict[val] = convert(dict[val])
             except KeyError,e:
                 if opt['warn'] == 0:
                     pass
                 elif opt['warn'] == 1:
                     print >>sys.stderr, 'Warning: key',val,\
                           'not found in file',filename
                 elif opt['warn'] == 2:
                     raise KeyError,e
                 else:
                     raise ValueError,'Warning level must be 0,1 or 2'
 
     return dict
 
 #----------------------------------------------------------------------------
 def flag_calls(func):
     """Wrap a function to detect and flag when it gets called.
 
     This is a decorator which takes a function and wraps it in a function with
     a 'called' attribute. wrapper.called is initialized to False.
 
     The wrapper.called attribute is set to False right before each call to the
     wrapped function, so if the call fails it remains False.  After the call
     completes, wrapper.called is set to True and the output is returned.
 
     Testing for truth in wrapper.called allows you to determine if a call to
     func() was attempted and succeeded."""
     
     def wrapper(*args,**kw):
         wrapper.called = False
         out = func(*args,**kw)
         wrapper.called = True
         return out
 
     wrapper.called = False
     wrapper.__doc__ = func.__doc__
     return wrapper
 
 #----------------------------------------------------------------------------
 class HomeDirError(Error):
     pass
 
 def get_home_dir():
     """Return the closest possible equivalent to a 'home' directory.
 
     We first try $HOME.  Absent that, on NT it's $HOMEDRIVE\$HOMEPATH.
 
     Currently only Posix and NT are implemented, a HomeDirError exception is
     raised for all other OSes. """
 
     try:
         return os.environ['HOME']
     except KeyError:
         if os.name == 'posix':
             raise HomeDirError,'undefined $HOME, IPython can not proceed.'
         elif os.name == 'nt':
             # For some strange reason, win9x returns 'nt' for os.name.
             try:
                 return os.path.join(os.environ['HOMEDRIVE'],os.environ['HOMEPATH'])
             except:
                 try:
                     # Use the registry to get the 'My Documents' folder.
                     import _winreg as wreg
                     key = wreg.OpenKey(wreg.HKEY_CURRENT_USER,
                                        "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
                     homedir = wreg.QueryValueEx(key,'Personal')[0]
                     key.Close()
                     return homedir
                 except:
                     return 'C:\\'
         elif os.name == 'dos':
             # Desperate, may do absurd things in classic MacOS. May work under DOS.
             return 'C:\\'
         else:
             raise HomeDirError,'support for your operating system not implemented.'
 
 #****************************************************************************
 # strings and text
 
 class LSString(str):
     """String derivative with a special access attributes.
 
     These are normal strings, but with the special attributes:
 
         .l (or .list) : value as list (split on newlines).
         .n (or .nlstr): original value (the string itself).
         .s (or .spstr): value as whitespace-separated string.
         
     Any values which require transformations are computed only once and
     cached.
 
     Such strings are very useful to efficiently interact with the shell, which
     typically only understands whitespace-separated options for commands."""
 
     def get_list(self):
         try:
             return self.__list
         except AttributeError:
             self.__list = self.split('\n')
             return self.__list
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = self.replace('\n',' ')
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         return self
 
     n = nlstr = property(get_nlstr)
 
 class SList(list):
     """List derivative with a special access attributes.
 
     These are normal lists, but with the special attributes:
 
         .l (or .list) : value as list (the list itself).
         .n (or .nlstr): value as a string, joined on newlines.
         .s (or .spstr): value as a string, joined on spaces.
 
     Any values which require transformations are computed only once and
     cached."""
 
     def get_list(self):
         return self
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = ' '.join(self)
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         try:
             return self.__nlstr
         except AttributeError:
             self.__nlstr = '\n'.join(self)
             return self.__nlstr
 
     n = nlstr = property(get_nlstr)
 
 def raw_input_multi(header='', ps1='==> ', ps2='..> ',terminate_str = '.'):
     """Take multiple lines of input.
 
     A list with each line of input as a separate element is returned when a
     termination string is entered (defaults to a single '.'). Input can also
     terminate via EOF (^D in Unix, ^Z-RET in Windows).
 
     Lines of input which end in \\ are joined into single entries (and a
     secondary continuation prompt is issued as long as the user terminates
     lines with \\). This allows entering very long strings which are still
     meant to be treated as single entities.
     """
 
     try:
         if header:
             header += '\n'
         lines = [raw_input(header + ps1)]
     except EOFError:
         return []
     terminate = [terminate_str]
     try:
         while lines[-1:] != terminate:
             new_line = raw_input(ps1)
             while new_line.endswith('\\'):
                 new_line = new_line[:-1] + raw_input(ps2)
             lines.append(new_line)
                 
         return lines[:-1]  # don't return the termination command
     except EOFError:
         print
         return lines
 
 #----------------------------------------------------------------------------
 def raw_input_ext(prompt='',  ps2='... '):
     """Similar to raw_input(), but accepts extended lines if input ends with \\."""
 
     line = raw_input(prompt)
     while line.endswith('\\'):
         line = line[:-1] + raw_input(ps2)
     return line
 
 #----------------------------------------------------------------------------
 def ask_yes_no(prompt,default=None):
     """Asks a question and returns an integer 1/0 (y/n) answer.
 
     If default is given (one of 'y','n'), it is used if the user input is
     empty. Otherwise the question is repeated until an answer is given.
     If EOF occurs 20 times consecutively, the default answer is assumed,
     or if there is no default, an exception is raised to prevent infinite
     loops.
 
     Valid answers are: y/yes/n/no (match is not case sensitive)."""
 
     answers = {'y':1,'n':0,'yes':1,'no':0}
     ans = None
     eofs, max_eofs = 0, 20
     while ans not in answers.keys():
         try:
             ans = raw_input(prompt+' ').lower()
             if not ans:  # response was an empty string
                 ans = default
             eofs = 0
         except (EOFError,KeyboardInterrupt):
             eofs = eofs + 1
             if eofs >= max_eofs:
                 if default in answers.keys():
                     ans = default
                 else:
                     raise
             
     return answers[ans]
 
 #----------------------------------------------------------------------------
 class EvalDict:
     """
     Emulate a dict which evaluates its contents in the caller's frame.
 
     Usage:
     >>>number = 19
     >>>text = "python"
     >>>print "%(text.capitalize())s %(number/9.0).1f rules!" % EvalDict()
     """
 
     # This version is due to [e-mail] on c.l.py, and is basically a
     # modified (shorter) version of:
     # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66018 by
     # Skip Montanaro ([e-mail]).
 
     def __getitem__(self, name):
         frame = sys._getframe(1)
         return eval(name, frame.f_globals, frame.f_locals)
 
 EvalString = EvalDict  # for backwards compatibility
 #----------------------------------------------------------------------------
 def qw(words,flat=0,sep=None,maxsplit=-1):
     """Similar to Perl's qw() operator, but with some more options.
 
     qw(words,flat=0,sep=' ',maxsplit=-1) -> words.split(sep,maxsplit)
 
     words can also be a list itself, and with flat=1, the output will be
     recursively flattened. Examples:
 
     >>> qw('1 2')
     ['1', '2']
     >>> qw(['a b','1 2',['m n','p q']])
     [['a', 'b'], ['1', '2'], [['m', 'n'], ['p', 'q']]]
     >>> qw(['a b','1 2',['m n','p q']],flat=1)
     ['a', 'b', '1', '2', 'm', 'n', 'p', 'q']    """
 
     if type(words) in StringTypes:
         return [word.strip() for word in words.split(sep,maxsplit)
                 if word and not word.isspace() ]
     if flat:
         return flatten(map(qw,words,[1]*len(words)))
     return map(qw,words)
 
 #----------------------------------------------------------------------------
 def qwflat(words,sep=None,maxsplit=-1):
     """Calls qw(words) in flat mode. It's just a convenient shorthand."""
     return qw(words,1,sep,maxsplit)
 
 #-----------------------------------------------------------------------------
 def list_strings(arg):
     """Always return a list of strings, given a string or list of strings
     as input."""
 
     if type(arg) in StringTypes: return [arg]
     else: return arg
 
 #----------------------------------------------------------------------------
 def grep(pat,list,case=1):
     """Simple minded grep-like function.
     grep(pat,list) returns occurrences of pat in list, None on failure.
 
     It only does simple string matching, with no support for regexps. Use the
     option case=0 for case-insensitive matching."""
 
     # This is pretty crude. At least it should implement copying only references
     # to the original data in case it's big. Now it copies the data for output.
     out=[]
     if case:
         for term in list:
             if term.find(pat)>-1: out.append(term)
     else:
         lpat=pat.lower()
         for term in list:
             if term.lower().find(lpat)>-1: out.append(term)
 
     if len(out): return out
     else: return None
 
 #----------------------------------------------------------------------------
 def dgrep(pat,*opts):
     """Return grep() on dir()+dir(__builtins__).
 
     A very common use of grep() when working interactively."""
 
     return grep(pat,dir(__main__)+dir(__main__.__builtins__),*opts)
 
 #----------------------------------------------------------------------------
 def idgrep(pat):
     """Case-insensitive dgrep()"""
 
     return dgrep(pat,0)
 
 #----------------------------------------------------------------------------
 def igrep(pat,list):
     """Synonym for case-insensitive grep."""
 
     return grep(pat,list,case=0)
 
 #----------------------------------------------------------------------------
 def indent(str,nspaces=4,ntabs=0):
     """Indent a string a given number of spaces or tabstops.
     
     indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
     """
     if str is None:
         return
     ind = '\t'*ntabs+' '*nspaces
     outstr = '%s%s' % (ind,str.replace(os.linesep,os.linesep+ind))
     if outstr.endswith(os.linesep+ind):
         return outstr[:-len(ind)]
     else:
         return outstr
     
 #-----------------------------------------------------------------------------
 def native_line_ends(filename,backup=1):
     """Convert (in-place) a file to line-ends native to the current OS.
 
     If the optional backup argument is given as false, no backup of the
     original file is left.  """
 
     backup_suffixes = {'posix':'~','dos':'.bak','nt':'.bak','mac':'.bak'}
 
     bak_filename = filename + backup_suffixes[os.name]
 
     original = open(filename).read()
     shutil.copy2(filename,bak_filename)
     try:
         new = open(filename,'wb')
         new.write(os.linesep.join(original.splitlines()))
         new.write(os.linesep) # ALWAYS put an eol at the end of the file
         new.close()
     except:
         os.rename(bak_filename,filename)
     if not backup:
         try:
             os.remove(bak_filename)
         except:
             pass
     
 #----------------------------------------------------------------------------
 def get_pager_cmd(pager_cmd = None):
     """Return a pager command.
 
     Makes some attempts at finding an OS-correct one."""
     
     if os.name == 'posix':
         default_pager_cmd = 'less -r'  # -r for color control sequences
     elif os.name in ['nt','dos']:
         default_pager_cmd = 'type'
 
     if pager_cmd is None:
         try:
             pager_cmd = os.environ['PAGER']
         except:
             pager_cmd = default_pager_cmd
     return pager_cmd
 
 #-----------------------------------------------------------------------------
 def get_pager_start(pager,start):
     """Return the string for paging files with an offset.
 
     This is the '+N' argument which less and more (under Unix) accept.
     """
 
     if pager in ['less','more']:
         if start:
             start_string = '+' + str(start)
         else:
             start_string = ''
     else:
         start_string = ''
     return start_string
 
 #----------------------------------------------------------------------------
 def page_dumb(strng,start=0,screen_lines=25):
     """Very dumb 'pager' in Python, for when nothing else works.
 
     Only moves forward, same interface as page(), except for pager_cmd and
     mode."""
 
     out_ln  = strng.splitlines()[start:]
     screens = chop(out_ln,screen_lines-1)
     if len(screens) == 1:
         print >>Term.cout, os.linesep.join(screens[0])
     else:
         for scr in screens[0:-1]:
             print >>Term.cout, os.linesep.join(scr)
             ans = raw_input('---Return to continue, q to quit--- ')
             if ans.lower().startswith('q'):
                 return
         print >>Term.cout, os.linesep.join(screens[-1])
 
 #----------------------------------------------------------------------------
 def page(strng,start=0,screen_lines=0,pager_cmd = None):
     """Print a string, piping through a pager after a certain length.
 
     The screen_lines parameter specifies the number of *usable* lines of your
     terminal screen (total lines minus lines you need to reserve to show other
     information).
 
     If you set screen_lines to a number <=0, page() will try to auto-determine
     your screen size and will only use up to (screen_size+screen_lines) for
     printing, paging after that. That is, if you want auto-detection but need
     to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for
     auto-detection without any lines reserved simply use screen_lines = 0.
 
     If a string won't fit in the allowed lines, it is sent through the
     specified pager command. If none given, look for PAGER in the environment,
     and ultimately default to less.
 
     If no system pager works, the string is sent through a 'dumb pager'
     written in python, very simplistic.
     """
     
     # Ugly kludge, but calling curses.initscr() flat out crashes in emacs
     TERM = os.environ.get('TERM','dumb')
     if TERM in ['dumb','emacs'] and os.name != 'nt':
         print strng
         return
     # chop off the topmost part of the string we don't want to see
     str_lines = strng.split(os.linesep)[start:]
     str_toprint = os.linesep.join(str_lines)
     num_newlines = len(str_lines)
     len_str = len(str_toprint)
     
     # Dumb heuristics to guesstimate number of on-screen lines the string
     # takes.  Very basic, but good enough for docstrings in reasonable
     # terminals. If someone later feels like refining it, it's not hard.
     numlines = max(num_newlines,int(len_str/80)+1)
 
     screen_lines_def = 25 # default value if we can't auto-determine
 
     # auto-determine screen size
     if screen_lines <= 0:
         if TERM=='xterm':
             try:
                 import curses
                 if hasattr(curses,'initscr'):
                     use_curses = 1
                 else:
                     use_curses = 0
             except ImportError:
                 use_curses = 0
         else:
             # curses causes problems on many terminals other than xterm.
             use_curses = 0
         if use_curses:
                 scr = curses.initscr()
                 screen_lines_real,screen_cols = scr.getmaxyx()
                 curses.endwin()
                 screen_lines += screen_lines_real
                 #print '***Screen size:',screen_lines_real,'lines x',\
                 #screen_cols,'columns.' # dbg
         else:
                 screen_lines += screen_lines_def
 
     #print 'numlines',numlines,'screenlines',screen_lines  # dbg
     if numlines <= screen_lines :
         #print '*** normal print'  # dbg
         print >>Term.cout, str_toprint
     else:
         # Try to open pager and default to internal one if that fails.
         # All failure modes are tagged as 'retval=1', to match the return
         # value of a failed system command.  If any intermediate attempt
         # sets retval to 1, at the end we resort to our own page_dumb() pager.
         pager_cmd = get_pager_cmd(pager_cmd)
         pager_cmd += ' ' + get_pager_start(pager_cmd,start)
         if os.name == 'nt':
             if pager_cmd.startswith('type'):
                 # The default WinXP 'type' command is failing on complex strings.
                 retval = 1
             else:
                 tmpname = tempfile.mktemp('.txt')
                 tmpfile = file(tmpname,'wt')
                 tmpfile.write(strng)
                 tmpfile.close()
                 cmd = "%s < %s" % (pager_cmd,tmpname)
                 if os.system(cmd):
                   retval = 1
                 else:
                   retval = None
                 os.remove(tmpname)
         else:
             try:
                 retval = None
                 # if I use popen4, things hang. No idea why.
                 #pager,shell_out = os.popen4(pager_cmd)
                 pager = os.popen(pager_cmd,'w')
                 pager.write(strng)
                 pager.close()
                 retval = pager.close()  # success returns None
             except IOError,msg:  # broken pipe when user quits
                 if msg.args == (32,'Broken pipe'):
                     retval = None
                 else:
                     retval = 1
             except OSError:
                 # Other strange problems, sometimes seen in Win2k/cygwin
                 retval = 1
         if retval is not None:
             page_dumb(strng,screen_lines=screen_lines)
 
 #----------------------------------------------------------------------------
 def page_file(fname,start = 0, pager_cmd = None):
     """Page a file, using an optional pager command and starting line.
     """
 
     pager_cmd = get_pager_cmd(pager_cmd)
     pager_cmd += ' ' + get_pager_start(pager_cmd,start)
 
     try:
         if os.environ['TERM'] in ['emacs','dumb']:
             raise EnvironmentError
         xsys(pager_cmd + ' ' + fname)
     except:
         try:
             if start > 0:
                 start -= 1
             page(open(fname).read(),start)
         except:
             print 'Unable to show file',`fname`
 
 #----------------------------------------------------------------------------
 def snip_print(str,width = 75,print_full = 0,header = ''):
     """Print a string snipping the midsection to fit in width.
 
     print_full: mode control:
       - 0: only snip long strings
       - 1: send to page() directly.
       - 2: snip long strings and ask for full length viewing with page()
     Return 1 if snipping was necessary, 0 otherwise."""
 
     if print_full == 1:
         page(header+str)
         return 0
 
     print header,
     if len(str) < width:
         print str
         snip = 0
     else:
         whalf = int((width -5)/2)
         print str[:whalf] + ' <...> ' + str[-whalf:]
         snip = 1
     if snip and print_full == 2:
         if raw_input(header+' Snipped. View (y/n)? [N]').lower() == 'y':
             page(str)
     return snip
 
 #****************************************************************************
 # lists, dicts and structures
 
 def belong(candidates,checklist):
     """Check whether a list of items appear in a given list of options.
 
     Returns a list of 1 and 0, one for each candidate given."""
 
     return [x in checklist for x in candidates]
 
 #----------------------------------------------------------------------------
 def uniq_stable(elems):
     """uniq_stable(elems) -> list
 
     Return from an iterable, a list of all the unique elements in the input,
     but maintaining the order in which they first appear.
 
     A naive solution to this problem which just makes a dictionary with the
     elements as keys fails to respect the stability condition, since
     dictionaries are unsorted by nature.
 
     Note: All elements in the input must be valid dictionary keys for this
     routine to work, as it internally uses a dictionary for efficiency
     reasons."""
     
     unique = []
     unique_dict = {}
     for nn in elems:
         if nn not in unique_dict:
             unique.append(nn)
             unique_dict[nn] = None
     return unique
 
 #----------------------------------------------------------------------------
 class NLprinter:
     """Print an arbitrarily nested list, indicating index numbers.
 
     An instance of this class called nlprint is available and callable as a
     function.
     
     nlprint(list,indent=' ',sep=': ') -> prints indenting each level by 'indent'
     and using 'sep' to separate the index from the value. """
 
     def __init__(self):
         self.depth = 0
     
     def __call__(self,lst,pos='',**kw):
         """Prints the nested list numbering levels."""
         kw.setdefault('indent',' ')
         kw.setdefault('sep',': ')
         kw.setdefault('start',0)
         kw.setdefault('stop',len(lst))
         # we need to remove start and stop from kw so they don't propagate
         # into a recursive call for a nested list.
         start = kw['start']; del kw['start']
         stop = kw['stop']; del kw['stop']
         if self.depth == 0 and 'header' in kw.keys():
             print kw['header']
             
         for idx in range(start,stop):
             elem = lst[idx]
             if type(elem)==type([]):
                 self.depth += 1
                 self.__call__(elem,itpl('$pos$idx,'),**kw)
                 self.depth -= 1
             else:
                 printpl(kw['indent']*self.depth+'$pos$idx$kw["sep"]$elem')
 
 nlprint = NLprinter()
 #----------------------------------------------------------------------------
 def all_belong(candidates,checklist):
     """Check whether a list of items ALL appear in a given list of options.
 
     Returns a single 1 or 0 value."""
 
     return 1-(0 in [x in checklist for x in candidates])
 
 #----------------------------------------------------------------------------
 def sort_compare(lst1,lst2,inplace = 1):
     """Sort and compare two lists.
 
     By default it does it in place, thus modifying the lists. Use inplace = 0
     to avoid that (at the cost of temporary copy creation)."""
     if not inplace:
         lst1 = lst1[:]
         lst2 = lst2[:]
     lst1.sort(); lst2.sort()
     return lst1 == lst2
 
 #----------------------------------------------------------------------------
 def mkdict(**kwargs):
     """Return a dict from a keyword list.
 
     It's just syntactic sugar for making ditcionary creation more convenient:
     # the standard way
     >>>data = { 'red' : 1, 'green' : 2, 'blue' : 3 }
     # a cleaner way
     >>>data = dict(red=1, green=2, blue=3)
 
     If you need more than this, look at the Struct() class."""
 
     return kwargs
 
 #----------------------------------------------------------------------------
 def list2dict(lst):
     """Takes a list of (key,value) pairs and turns it into a dict."""
 
     dic = {}
     for k,v in lst: dic[k] = v
     return dic
 
 #----------------------------------------------------------------------------
 def list2dict2(lst,default=''):
     """Takes a list and turns it into a dict.
     Much slower than list2dict, but more versatile. This version can take
     lists with sublists of arbitrary length (including sclars)."""
 
     dic = {}
     for elem in lst:
         if type(elem) in (types.ListType,types.TupleType):
             size = len(elem)
             if  size == 0:
                 pass
             elif size == 1:
                 dic[elem] = default
             else:
                 k,v = elem[0], elem[1:]
                 if len(v) == 1: v = v[0]
                 dic[k] = v
         else:
             dic[elem] = default
     return dic
 
 #----------------------------------------------------------------------------
 def flatten(seq):
     """Flatten a list of lists (NOT recursive, only works for 2d lists)."""
 
     # bug in python??? (YES. Fixed in 2.2, let's leave the kludgy fix in).
 
     # if the x=0 isn't made, a *global* variable x is left over after calling
     # this function, with the value of the last element in the return
     # list. This does seem like a bug big time to me.
 
     # the problem is fixed with the x=0, which seems to force the creation of
     # a local name
 
     x = 0 
     return [x for subseq in seq for x in subseq]
 
 #----------------------------------------------------------------------------
 def get_slice(seq,start=0,stop=None,step=1):
     """Get a slice of a sequence with variable step. Specify start,stop,step."""
     if stop == None:
         stop = len(seq)
     item = lambda i: seq[i]
     return map(item,xrange(start,stop,step))
 
 #----------------------------------------------------------------------------
 def chop(seq,size):
     """Chop a sequence into chunks of the given size."""
     chunk = lambda i: seq[i:i+size]
     return map(chunk,xrange(0,len(seq),size))
 
 #----------------------------------------------------------------------------
 def with(object, **args):
     """Set multiple attributes for an object, similar to Pascal's with.
 
     Example:
     with(jim,
          born = 1960,
          haircolour = 'Brown',
          eyecolour = 'Green')
 
     Credit: Greg Ewing, in
     http://mail.python.org/pipermail/python-list/2001-May/040703.html"""
 
     object.__dict__.update(args)
 
 #----------------------------------------------------------------------------
 def setattr_list(obj,alist,nspace = None):
     """Set a list of attributes for an object taken from a namespace.
 
     setattr_list(obj,alist,nspace) -> sets in obj all the attributes listed in
     alist with their values taken from nspace, which must be a dict (something
     like locals() will often do) If nspace isn't given, locals() of the
     *caller* is used, so in most cases you can omit it.
 
     Note that alist can be given as a string, which will be automatically
     split into a list on whitespace. If given as a list, it must be a list of
     *strings* (the variable names themselves), not of variables."""
 
     # this grabs the local variables from the *previous* call frame -- that is
     # the locals from the function that called setattr_list().
     # - snipped from weave.inline()
     if nspace is None:
         call_frame = sys._getframe().f_back
         nspace = call_frame.f_locals
 
     if type(alist) in StringTypes:
         alist = alist.split()
     for attr in alist:
         val = eval(attr,nspace)
         setattr(obj,attr,val)
 
 #----------------------------------------------------------------------------
 def getattr_list(obj,alist,*args):
     """getattr_list(obj,alist[, default]) -> attribute list.
 
     Get a list of named attributes for an object. When a default argument is
     given, it is returned when the attribute doesn't exist; without it, an
     exception is raised in that case.
 
     Note that alist can be given as a string, which will be automatically
     split into a list on whitespace. If given as a list, it must be a list of
     *strings* (the variable names themselves), not of variables."""
 
     if type(alist) in StringTypes:
         alist = alist.split()
     if args:
         if len(args)==1:
             default = args[0]
             return map(lambda attr: getattr(obj,attr,default),alist)
         else:
             raise ValueError,'getattr_list() takes only one optional argument'
     else:
         return map(lambda attr: getattr(obj,attr),alist)
     
 #----------------------------------------------------------------------------
 def map_method(method,object_list,*argseq,**kw):
     """map_method(method,object_list,*args,**kw) -> list
 
     Return a list of the results of applying the methods to the items of the
     argument sequence(s).  If more than one sequence is given, the method is
     called with an argument list consisting of the corresponding item of each
     sequence. All sequences must be of the same length.
 
     Keyword arguments are passed verbatim to all objects called.
 
     This is Python code, so it's not nearly as fast as the builtin map()."""
 
     out_list = []
     idx = 0
     for object in object_list:
         try:
             handler = getattr(object, method)
         except AttributeError:
             out_list.append(None)
         else:
             if argseq:
                 args = map(lambda lst:lst[idx],argseq)
                 #print 'ob',object,'hand',handler,'ar',args # dbg
                 out_list.append(handler(args,**kw))
             else:
                 out_list.append(handler(**kw))
         idx += 1
     return out_list
 
 #----------------------------------------------------------------------------
 # Proposed popitem() extension, written as a method
 
 class NotGiven: pass
 
 def popkey(dct,key,default=NotGiven):
     """Return dct[key] and delete dct[key].
 
     If default is given, return it if dct[key] doesn't exist, otherwise raise
     KeyError.  """
 
     try:
         val = dct[key]
     except KeyError:
         if default is NotGiven:
             raise
         else:
             return default
     else:
         del dct[key]
         return val
 #*************************** end of file <genutils.py> **********************
 

IPython/iplib.py

@@ -1,2084 +1,1999 b''
 # -*- coding: utf-8 -*-
 """
 IPython -- An enhanced Interactive Python
 
 Requires Python 2.1 or newer.
 
 This file contains all the classes and helper functions specific to IPython.
 
-$Id: iplib.py 602 2005-06-09 03:02:30Z fperez $
+$Id: iplib.py 638 2005-07-18 03:01:41Z fperez $
 """
 
 #*****************************************************************************
 #       Copyright (C) 2001 Janko Hauser <[e-mail]> and
 #       Copyright (C) 2001-2004 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #
 # Note: this code originally subclassed code.InteractiveConsole from the
 # Python standard library.  Over time, much of that class has been copied
 # verbatim here for modifications which could not be accomplished by
 # subclassing.  The Python License (sec. 2) allows for this, but it's always
 # nice to acknowledge credit where credit is due.
 #*****************************************************************************
 
 #****************************************************************************
 # Modules and globals
 
 from __future__ import generators  # for 2.2 backwards-compatibility
 
 from IPython import Release
 __author__  = '%s <%s>\n%s <%s>' % \
               ( Release.authors['Janko'] + Release.authors['Fernando'] )
 __license__ = Release.license
 __version__ = Release.version
 
 # Python standard modules
 import __main__
 import __builtin__
 import exceptions
 import keyword
 import new
 import os, sys, shutil
 import code, glob, types, re
 import string, StringIO
 import inspect, pydoc
 import bdb, pdb
 import UserList # don't subclass list so this works with Python2.1
 from pprint import pprint, pformat
 import cPickle as pickle
 import traceback
 
 # IPython's own modules
 import IPython
 from IPython import OInspect,PyColorize,ultraTB
 from IPython.ultraTB import ColorScheme,ColorSchemeTable  # too long names
 from IPython.Logger import Logger
 from IPython.Magic import Magic,magic2python,shlex_split
 from IPython.usage import cmd_line_usage,interactive_usage
 from IPython.Struct import Struct
 from IPython.Itpl import Itpl,itpl,printpl,ItplNS,itplns
 from IPython.FakeModule import FakeModule
 from IPython.background_jobs import BackgroundJobManager
 from IPython.genutils import *
 
 # Global pointer to the running 
 
 # store the builtin raw_input globally, and use this always, in case user code
 # overwrites it (like wx.py.PyShell does)
 raw_input_original = raw_input
 
-# declares Python 2.2 compatibility symbols:
-try:
-    enumerate
-except NameError:
-    def enumerate(obj):
-        i = -1
-        for item in obj:
-            i += 1
-            yield i, item
 #****************************************************************************
 # Some utility function definitions
 
 class Bunch: pass
 
 def esc_quotes(strng):
     """Return the input string with single and double quotes escaped out"""
 
     return strng.replace('"','\\"').replace("'","\\'")
 
 def import_fail_info(mod_name,fns=None):
     """Inform load failure for a module."""
 
     if fns == None:
         warn("Loading of %s failed.\n" % (mod_name,))
     else:
         warn("Loading of %s from %s failed.\n" % (fns,mod_name))
 
 def qw_lol(indata):
     """qw_lol('a b') -> [['a','b']],
     otherwise it's just a call to qw().
 
     We need this to make sure the modules_some keys *always* end up as a
     list of lists."""
 
     if type(indata) in StringTypes:
         return [qw(indata)]
     else:
         return qw(indata)
 
 def ipmagic(arg_s):
     """Call a magic function by name.
 
     Input: a string containing the name of the magic function to call and any
     additional arguments to be passed to the magic.
 
     ipmagic('name -opt foo bar') is equivalent to typing at the ipython
     prompt:
 
     In[1]: %name -opt foo bar
 
     To call a magic without arguments, simply use ipmagic('name').
 
     This provides a proper Python function to call IPython's magics in any
     valid Python code you can type at the interpreter, including loops and
     compound statements.  It is added by IPython to the Python builtin
     namespace upon initialization."""
 
     args = arg_s.split(' ',1)
     magic_name = args[0]
     if magic_name.startswith(__IPYTHON__.ESC_MAGIC):
         magic_name = magic_name[1:]
     try:
         magic_args = args[1]
     except IndexError:
         magic_args = ''
     fn = getattr(__IPYTHON__,'magic_'+magic_name,None)
     if fn is None:
         error("Magic function `%s` not found." % magic_name)
     else:
         magic_args = __IPYTHON__.var_expand(magic_args)
         return fn(magic_args)
 
 def ipalias(arg_s):
     """Call an alias by name.
 
     Input: a string containing the name of the alias to call and any
     additional arguments to be passed to the magic.
 
     ipalias('name -opt foo bar') is equivalent to typing at the ipython
     prompt:
 
     In[1]: name -opt foo bar
 
     To call an alias without arguments, simply use ipalias('name').
 
     This provides a proper Python function to call IPython's aliases in any
     valid Python code you can type at the interpreter, including loops and
     compound statements.  It is added by IPython to the Python builtin
     namespace upon initialization."""
 
     args = arg_s.split(' ',1)
     alias_name = args[0]
     try:
         alias_args = args[1]
     except IndexError:
         alias_args = ''
     if alias_name in __IPYTHON__.alias_table:
         __IPYTHON__.call_alias(alias_name,alias_args)
     else:
         error("Alias `%s` not found." % alias_name)
 
 #-----------------------------------------------------------------------------
 # Local use classes
 try:
     from IPython import FlexCompleter
 
     class MagicCompleter(FlexCompleter.Completer):
         """Extension of the completer class to work on %-prefixed lines."""
 
         def __init__(self,shell,namespace=None,omit__names=0,alias_table=None):
             """MagicCompleter() -> completer
 
             Return a completer object suitable for use by the readline library
             via readline.set_completer().
 
             Inputs:
 
             - shell: a pointer to the ipython shell itself.  This is needed
             because this completer knows about magic functions, and those can
             only be accessed via the ipython instance.
 
             - namespace: an optional dict where completions are performed.
             
             - The optional omit__names parameter sets the completer to omit the
             'magic' names (__magicname__) for python objects unless the text
             to be completed explicitly starts with one or more underscores.
 
             - If alias_table is supplied, it should be a dictionary of aliases
             to complete. """
 
             FlexCompleter.Completer.__init__(self,namespace)
             self.magic_prefix = shell.name+'.magic_'
             self.magic_escape = shell.ESC_MAGIC
             self.readline = FlexCompleter.readline
             delims = self.readline.get_completer_delims()
             delims = delims.replace(self.magic_escape,'')
             self.readline.set_completer_delims(delims)
             self.get_line_buffer = self.readline.get_line_buffer
             self.omit__names = omit__names
             self.merge_completions = shell.rc.readline_merge_completions
             
             if alias_table is None:
                 alias_table = {}
             self.alias_table = alias_table
             # Regexp to split filenames with spaces in them
             self.space_name_re = re.compile(r'([^\\] )')
             # Hold a local ref. to glob.glob for speed
             self.glob = glob.glob
             # Special handling of backslashes needed in win32 platforms
             if sys.platform == "win32":
                 self.clean_glob = self._clean_glob_win32
             else:
                 self.clean_glob = self._clean_glob
             self.matchers = [self.python_matches,
                              self.file_matches,
                              self.alias_matches,
                              self.python_func_kw_matches]
 
         # Code contributed by Alex Schmolck, for ipython/emacs integration
         def all_completions(self, text):
             """Return all possible completions for the benefit of emacs."""
             
             completions = []
             try:
                 for i in xrange(sys.maxint):
                     res = self.complete(text, i)
 
                     if not res: break
 
                     completions.append(res)
             #XXX workaround for ``notDefined.<tab>``
             except NameError:
                 pass
             return completions
         # /end Alex Schmolck code.
 
         def _clean_glob(self,text):
             return self.glob("%s*" % text)
             
         def _clean_glob_win32(self,text):
             return [f.replace("\\","/")
                     for f in self.glob("%s*" % text)]            
 
         def file_matches(self, text):
             """Match filneames, expanding ~USER type strings.
 
             Most of the seemingly convoluted logic in this completer is an
             attempt to handle filenames with spaces in them.  And yet it's not
             quite perfect, because Python's readline doesn't expose all of the
             GNU readline details needed for this to be done correctly.
 
             For a filename with a space in it, the printed completions will be
             only the parts after what's already been typed (instead of the
             full completions, as is normally done).  I don't think with the
             current (as of Python 2.3) Python readline it's possible to do
             better."""
             
             #print 'Completer->file_matches: <%s>' % text # dbg
 
             # chars that require escaping with backslash - i.e. chars
             # that readline treats incorrectly as delimiters, but we
             # don't want to treat as delimiters in filename matching
             # when escaped with backslash
             
             protectables = ' ()[]{}'
 
             def protect_filename(s):
                 return "".join([(ch in protectables and '\\' + ch or ch)
                                 for ch in s])
 
             lbuf = self.get_line_buffer()[:self.readline.get_endidx()]
             open_quotes = 0  # track strings with open quotes
             try:
                 lsplit = shlex_split(lbuf)[-1]
             except ValueError:
                 # typically an unmatched ", or backslash without escaped char.
                 if lbuf.count('"')==1:
                     open_quotes = 1
                     lsplit = lbuf.split('"')[-1]
                 elif lbuf.count("'")==1:
                     open_quotes = 1
                     lsplit = lbuf.split("'")[-1]
                 else:
                     return None
             except IndexError:
                 # tab pressed on empty line
                 lsplit = ""
 
             if lsplit != protect_filename(lsplit):
                 # if protectables are found, do matching on the whole escaped
                 # name
                 has_protectables = 1
                 text0,text = text,lsplit
             else:
                 has_protectables = 0
                 text = os.path.expanduser(text)
             
             if text == "":
                 return [protect_filename(f) for f in self.glob("*")]
 
             m0 = self.clean_glob(text.replace('\\',''))
             if has_protectables:
                 # If we had protectables, we need to revert our changes to the
                 # beginning of filename so that we don't double-write the part
                 # of the filename we have so far
                 len_lsplit = len(lsplit)
                 matches = [text0 + protect_filename(f[len_lsplit:]) for f in m0]
             else:
                 if open_quotes:
                     # if we have a string with an open quote, we don't need to
                     # protect the names at all (and we _shouldn't_, as it
                     # would cause bugs when the filesystem call is made).
                     matches = m0
                 else:
                     matches = [protect_filename(f) for f in m0]
             if len(matches) == 1 and os.path.isdir(matches[0]):
                 # Takes care of links to directories also.  Use '/'
                 # explicitly, even under Windows, so that name completions
                 # don't end up escaped.
                 matches[0] += '/'
             return matches
 
         def alias_matches(self, text):
             """Match internal system aliases"""
             #print 'Completer->alias_matches:',text # dbg
             text = os.path.expanduser(text)
             aliases =  self.alias_table.keys()
             if text == "":
                 return aliases
             else:
                 return [alias for alias in aliases if alias.startswith(text)]
             
         def python_matches(self,text):
             """Match attributes or global python names"""
             #print 'Completer->python_matches' # dbg
             if "." in text:
                 try:
                     matches = self.attr_matches(text)
                     if text.endswith('.') and self.omit__names:
                         if self.omit__names == 1:
                             # true if txt is _not_ a __ name, false otherwise:
                             no__name = (lambda txt:
                                         re.match(r'.*\.__.*?__',txt) is None)
                         else:
                             # true if txt is _not_ a _ name, false otherwise:
                             no__name = (lambda txt:
                                         re.match(r'.*\._.*?',txt) is None)
                         matches = filter(no__name, matches)
                 except NameError:
                     # catches <undefined attributes>.<tab>
                     matches = []
             else:
                 matches = self.global_matches(text)
                 # this is so completion finds magics when automagic is on:
                 if matches == [] and not text.startswith(os.sep):
                     matches = self.attr_matches(self.magic_prefix+text)
             return matches
 
         def _default_arguments(self, obj):
             """Return the list of default arguments of obj if it is callable,
             or empty list otherwise."""
             
             if not (inspect.isfunction(obj) or inspect.ismethod(obj)):
                 # for classes, check for __init__,__new__
                 if inspect.isclass(obj):
                     obj = (getattr(obj,'__init__',None) or
                            getattr(obj,'__new__',None))
                 # for all others, check if they are __call__able
                 elif hasattr(obj, '__call__'):
                     obj = obj.__call__
                 # XXX: is there a way to handle the builtins ?
             try:
                 args,_,_1,defaults = inspect.getargspec(obj)
                 if defaults:
                     return args[-len(defaults):]
             except TypeError: pass
             return []
 
         def python_func_kw_matches(self,text):
             """Match named parameters (kwargs) of the last open function"""
 
             if "." in text: # a parameter cannot be dotted
                 return []
             try: regexp = self.__funcParamsRegex
             except AttributeError:
                 regexp = self.__funcParamsRegex = re.compile(r'''
                     '.*?' |    # single quoted strings or
                     ".*?" |    # double quoted strings or
                     \w+   |    # identifier
                     \S         # other characters
                     ''', re.VERBOSE | re.DOTALL)
             # 1. find the nearest identifier that comes before an unclosed
             # parenthesis e.g. for "foo (1+bar(x), pa", the candidate is "foo"
             tokens = regexp.findall(self.get_line_buffer())
             tokens.reverse()
             iterTokens = iter(tokens); openPar = 0
             for token in iterTokens:
                 if token == ')':
                     openPar -= 1
                 elif token == '(':
                     openPar += 1
                     if openPar > 0:
                         # found the last unclosed parenthesis
                         break
             else:
                 return []
             # 2. Concatenate any dotted names (e.g. "foo.bar" for "foo.bar(x, pa" )
             ids = []
             isId = re.compile(r'\w+$').match
             while True:
                 try:
                     ids.append(iterTokens.next())
                     if not isId(ids[-1]):
                         ids.pop(); break
                     if not iterTokens.next() == '.':
                         break
                 except StopIteration:
                     break
             # lookup the candidate callable matches either using global_matches
             # or attr_matches for dotted names
             if len(ids) == 1:
                 callableMatches = self.global_matches(ids[0])
             else:
                 callableMatches = self.attr_matches('.'.join(ids[::-1]))
             argMatches = []
             for callableMatch in callableMatches:
                 try: namedArgs = self._default_arguments(eval(callableMatch,
                                                              self.namespace))
                 except: continue
                 for namedArg in namedArgs:
                     if namedArg.startswith(text):
                         argMatches.append("%s=" %namedArg)
             return argMatches
 
         def complete(self, text, state):
             """Return the next possible completion for 'text'.
 
             This is called successively with state == 0, 1, 2, ... until it
             returns None.  The completion should begin with 'text'.  """
             
             #print '\n*** COMPLETE: <%s> (%s)' % (text,state)  # dbg
             magic_escape = self.magic_escape
             magic_prefix = self.magic_prefix
             
             try:
                 if text.startswith(magic_escape):
                     text = text.replace(magic_escape,magic_prefix)
                 elif text.startswith('~'):
                     text = os.path.expanduser(text)
                 if state == 0:
                     # Extend the list of completions with the results of each
                     # matcher, so we return results to the user from all
                     # namespaces.
                     if self.merge_completions:
                         self.matches = []
                         for matcher in self.matchers:
                             self.matches.extend(matcher(text))
                     else:
                         for matcher in self.matchers:
                             self.matches = matcher(text)
                             if self.matches:
                                 break
                         
                 try:
                     return self.matches[state].replace(magic_prefix,magic_escape)
                 except IndexError:
                     return None
             except:
                 # If completion fails, don't annoy the user.
                 pass
 
 except ImportError:
     pass  # no readline support
 
 except KeyError:
     pass  # Windows doesn't set TERM, it doesn't matter
 
 
 class InputList(UserList.UserList):
     """Class to store user input.
 
     It's basically a list, but slices return a string instead of a list, thus
     allowing things like (assuming 'In' is an instance):
 
     exec In[4:7]
 
     or
 
     exec In[5:9] + In[14] + In[21:25]"""
 
     def __getslice__(self,i,j):
         return ''.join(UserList.UserList.__getslice__(self,i,j))
 
 #****************************************************************************
 # Local use exceptions
 class SpaceInInput(exceptions.Exception):
     pass
 
 #****************************************************************************
 # Main IPython class
 
 class InteractiveShell(code.InteractiveConsole, Logger, Magic):
     """An enhanced console for Python."""
 
     def __init__(self,name,usage=None,rc=Struct(opts=None,args=None),
                  user_ns = None,banner2='',
                  custom_exceptions=((),None)):
 
         # Put a reference to self in builtins so that any form of embedded or
         # imported code can test for being inside IPython.
         __builtin__.__IPYTHON__ = self
 
         # And load into builtins ipmagic/ipalias as well
         __builtin__.ipmagic = ipmagic
         __builtin__.ipalias = ipalias
 
         # Add to __builtin__ other parts of IPython's public API
         __builtin__.ip_set_hook = self.set_hook
 
         # Keep in the builtins a flag for when IPython is active.  We set it
         # with setdefault so that multiple nested IPythons don't clobber one
         # another.  Each will increase its value by one upon being activated,
         # which also gives us a way to determine the nesting level.
         __builtin__.__dict__.setdefault('__IPYTHON__active',0)
 
         # Inform the user of ipython's fast exit magics.
         _exit = ' Use %Exit or %Quit to exit without confirmation.'
         __builtin__.exit += _exit
         __builtin__.quit += _exit
 
         # Create the namespace where the user will operate:
 
         # FIXME. For some strange reason, __builtins__ is showing up at user
         # level as a dict instead of a module. This is a manual fix, but I
         # should really track down where the problem is coming from. Alex
         # Schmolck reported this problem first.
 
         # A useful post by Alex Martelli on this topic:
         # Re: inconsistent value from __builtins__
         # Von: Alex Martelli <[e-mail]>
         # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
         # Gruppen: comp.lang.python
         # Referenzen: 1
 
         # Michael Hohn <[e-mail]> wrote:
         # > >>> print type(builtin_check.get_global_binding('__builtins__'))
         # > <type 'dict'>
         # > >>> print type(__builtins__)
         # > <type 'module'>
         # > Is this difference in return value intentional?
 
         # Well, it's documented that '__builtins__' can be either a dictionary
         # or a module, and it's been that way for a long time. Whether it's
         # intentional (or sensible), I don't know. In any case, the idea is that
         # if you need to access the built-in namespace directly, you should start
         # with "import __builtin__" (note, no 's') which will definitely give you
         # a module. Yeah, it's somewhat confusing:-(.
         
         if user_ns is None:
             # Set __name__ to __main__ to better match the behavior of the
             # normal interpreter.
             self.user_ns = {'__name__'     :'__main__',
                             '__builtins__' : __builtin__,
                             }
         else:
             self.user_ns = user_ns
 
         # The user namespace MUST have a pointer to the shell itself.
         self.user_ns[name] = self
 
         # We need to insert into sys.modules something that looks like a
         # module but which accesses the IPython namespace, for shelve and
         # pickle to work interactively. Normally they rely on getting
         # everything out of __main__, but for embedding purposes each IPython
         # instance has its own private namespace, so we can't go shoving
         # everything into __main__.
 
         try:
             main_name = self.user_ns['__name__']
         except KeyError:
             raise KeyError,'user_ns dictionary MUST have a "__name__" key'
         else:
             #print "pickle hack in place"  # dbg
             sys.modules[main_name] = FakeModule(self.user_ns)
 
         # List of input with multi-line handling.
         # Fill its zero entry, user counter starts at 1
         self.input_hist = InputList(['\n'])
 
         # list of visited directories
         self.dir_hist = [os.getcwd()]
 
         # dict of output history
         self.output_hist = {}
 
         # dict of names to be treated as system aliases.  Each entry in the
         # alias table must be a 2-tuple of the form (N,name), where N is the
         # number of positional arguments of the alias.
         self.alias_table = {}
 
         # dict of things NOT to alias (keywords and builtins)
         self.no_alias = {}
         for key in keyword.kwlist:
             self.no_alias[key] = 1
         self.no_alias.update(__builtin__.__dict__)
         
         # make global variables for user access to these
         self.user_ns['_ih'] = self.input_hist
         self.user_ns['_oh'] = self.output_hist
         self.user_ns['_dh'] = self.dir_hist
 
         # user aliases to input and output histories
         self.user_ns['In']  = self.input_hist
         self.user_ns['Out'] = self.output_hist
 
         # Store the actual shell's name
         self.name = name
 
         # Object variable to store code object waiting execution.  This is
         # used mainly by the multithreaded shells, but it can come in handy in
         # other situations.  No need to use a Queue here, since it's a single
         # item which gets cleared once run.
         self.code_to_run = None
         self.code_to_run_src = ''  # corresponding source
         
         # Job manager (for jobs run as background threads)
         self.jobs = BackgroundJobManager()
         # Put the job manager into builtins so it's always there.
         __builtin__.jobs = self.jobs
 
         # escapes for automatic behavior on the command line
         self.ESC_SHELL = '!'
         self.ESC_HELP  = '?'
         self.ESC_MAGIC = '%'
         self.ESC_QUOTE = ','
         self.ESC_QUOTE2 = ';'
         self.ESC_PAREN = '/'
 
         # And their associated handlers
         self.esc_handlers = {self.ESC_PAREN:self.handle_auto,
                              self.ESC_QUOTE:self.handle_auto,
                              self.ESC_QUOTE2:self.handle_auto,
                              self.ESC_MAGIC:self.handle_magic,
                              self.ESC_HELP:self.handle_help,
                              self.ESC_SHELL:self.handle_shell_escape,
                              }
 
         # class initializations
         code.InteractiveConsole.__init__(self,locals = self.user_ns)
         Logger.__init__(self,log_ns = self.user_ns)
         Magic.__init__(self,self)
 
         # an ugly hack to get a pointer to the shell, so I can start writing
         # magic code via this pointer instead of the current mixin salad.
         Magic.set_shell(self,self)
 
         # hooks holds pointers used for user-side customizations
         self.hooks = Struct()
         
         # Set all default hooks, defined in the IPython.hooks module.
         hooks = IPython.hooks
         for hook_name in hooks.__all__:
             self.set_hook(hook_name,getattr(hooks,hook_name))
 
         # Flag to mark unconditional exit
         self.exit_now = False
 
         self.usage_min =  """\
         An enhanced console for Python.
         Some of its features are:
         - Readline support if the readline library is present.
         - Tab completion in the local namespace.
         - Logging of input, see command-line options.
         - System shell escape via ! , eg !ls.
         - Magic commands, starting with a % (like %ls, %pwd, %cd, etc.)
         - Keeps track of locally defined variables via %who, %whos.
         - Show object information with a ? eg ?x or x? (use ?? for more info).
         """
         if usage: self.usage = usage
         else: self.usage = self.usage_min
 
         # Storage
         self.rc = rc   # This will hold all configuration information
         self.inputcache = []
         self._boundcache = []
         self.pager = 'less'
         # temporary files used for various purposes.  Deleted at exit.
         self.tempfiles = []
 
         # for pushd/popd management
         try:
             self.home_dir = get_home_dir()
         except HomeDirError,msg:
             fatal(msg)
 
         self.dir_stack = [os.getcwd().replace(self.home_dir,'~')]
 
         # Functions to call the underlying shell.
 
         # utility to expand user variables via Itpl
         self.var_expand = lambda cmd: str(ItplNS(cmd.replace('#','\#'),
                                                  self.user_ns))
         # The first is similar to os.system, but it doesn't return a value,
         # and it allows interpolation of variables in the user's namespace.
         self.system = lambda cmd: shell(self.var_expand(cmd),
                                         header='IPython system call: ',
                                         verbose=self.rc.system_verbose)
         # These are for getoutput and getoutputerror:
         self.getoutput = lambda cmd: \
                          getoutput(self.var_expand(cmd),
                                    header='IPython system call: ',
                                    verbose=self.rc.system_verbose)
         self.getoutputerror = lambda cmd: \
                               getoutputerror(str(ItplNS(cmd.replace('#','\#'),
                                                         self.user_ns)),
                                              header='IPython system call: ',
                                              verbose=self.rc.system_verbose)
  
         # RegExp for splitting line contents into pre-char//first
         # word-method//rest.  For clarity, each group in on one line.
 
         # WARNING: update the regexp if the above escapes are changed, as they
         # are hardwired in.
 
         # Don't get carried away with trying to make the autocalling catch too
         # much:  it's better to be conservative rather than to trigger hidden
         # evals() somewhere and end up causing side effects.
 
         self.line_split = re.compile(r'^([\s*,;/])'
                                      r'([\?\w\.]+\w*\s*)'
                                      r'(\(?.*$)')
 
         # Original re, keep around for a while in case changes break something
         #self.line_split = re.compile(r'(^[\s*!\?%,/]?)'
         #                             r'(\s*[\?\w\.]+\w*\s*)'
         #                             r'(\(?.*$)')
 
         # RegExp to identify potential function names
         self.re_fun_name = re.compile(r'[a-zA-Z_]([a-zA-Z0-9_.]*) *$')
         # RegExp to exclude strings with this start from autocalling
         self.re_exclude_auto = re.compile('^[!=()<>,\*/\+-]|^is ')
         # try to catch also methods for stuff in lists/tuples/dicts: off
         # (experimental). For this to work, the line_split regexp would need
         # to be modified so it wouldn't break things at '['. That line is
         # nasty enough that I shouldn't change it until I can test it _well_.
         #self.re_fun_name = re.compile (r'[a-zA-Z_]([a-zA-Z0-9_.\[\]]*) ?$')
 
         # keep track of where we started running (mainly for crash post-mortem)
         self.starting_dir = os.getcwd()
 
         # Attributes for Logger mixin class, make defaults here
         self._dolog = 0
         self.LOG = ''
         self.LOGDEF = '.InteractiveShell.log'
         self.LOGMODE = 'over'
         self.LOGHEAD = Itpl(
 """#log# Automatic Logger file. *** THIS MUST BE THE FIRST LINE ***
 #log# DO NOT CHANGE THIS LINE OR THE TWO BELOW
 #log# opts = $self.rc.opts
 #log# args = $self.rc.args
 #log# It is safe to make manual edits below here.
 #log#-----------------------------------------------------------------------
 """)
         # Various switches which can be set
         self.CACHELENGTH = 5000  # this is cheap, it's just text
         self.BANNER = "Python %(version)s on %(platform)s\n" % sys.__dict__
         self.banner2 = banner2
 
         # TraceBack handlers:
         # Need two, one for syntax errors and one for other exceptions.
         self.SyntaxTB = ultraTB.ListTB(color_scheme='NoColor')
         # This one is initialized with an offset, meaning we always want to
         # remove the topmost item in the traceback, which is our own internal
         # code. Valid modes: ['Plain','Context','Verbose']
         self.InteractiveTB = ultraTB.AutoFormattedTB(mode = 'Plain',
                                                      color_scheme='NoColor',
                                                      tb_offset = 1)
         # and add any custom exception handlers the user may have specified
         self.set_custom_exc(*custom_exceptions)
 
         # Object inspector
         ins_colors = OInspect.InspectColors
         code_colors = PyColorize.ANSICodeColors
         self.inspector = OInspect.Inspector(ins_colors,code_colors,'NoColor')
         self.autoindent = 0
 
         # Make some aliases automatically
         # Prepare list of shell aliases to auto-define
         if os.name == 'posix':            
             auto_alias = ('mkdir mkdir', 'rmdir rmdir',
                           'mv mv -i','rm rm -i','cp cp -i',
                           'cat cat','less less','clear clear',
                           # a better ls
                           'ls ls -F',
                           # long ls
                           'll ls -lF',
                           # color ls
                           'lc ls -F -o --color',
                           # ls normal files only
                           'lf ls -F -o --color %l | grep ^-',
                           # ls symbolic links 
                           'lk ls -F -o --color %l | grep ^l',
                           # directories or links to directories,
                           'ldir ls -F -o --color %l | grep /$',
                           # things which are executable
                           'lx ls -F -o --color %l | grep ^-..x',
                           )
         elif os.name in ['nt','dos']:
             auto_alias = ('dir dir /on', 'ls dir /on',
                           'ddir dir /ad /on', 'ldir dir /ad /on',
                           'mkdir mkdir','rmdir rmdir','echo echo',
                           'ren ren','cls cls','copy copy')
         else:
             auto_alias = ()
         self.auto_alias = map(lambda s:s.split(None,1),auto_alias)
         # Call the actual (public) initializer
         self.init_auto_alias()
     # end __init__
 
     def set_hook(self,name,hook):
         """set_hook(name,hook) -> sets an internal IPython hook.
 
         IPython exposes some of its internal API as user-modifiable hooks.  By
         resetting one of these hooks, you can modify IPython's behavior to
         call at runtime your own routines."""
 
         # At some point in the future, this should validate the hook before it
         # accepts it.  Probably at least check that the hook takes the number
         # of args it's supposed to.
         setattr(self.hooks,name,new.instancemethod(hook,self,self.__class__))
 
     def set_custom_exc(self,exc_tuple,handler):
         """set_custom_exc(exc_tuple,handler)
 
         Set a custom exception handler, which will be called if any of the
         exceptions in exc_tuple occur in the mainloop (specifically, in the
         runcode() method.
 
         Inputs:
 
           - exc_tuple: a *tuple* of valid exceptions to call the defined
           handler for.  It is very important that you use a tuple, and NOT A
           LIST here, because of the way Python's except statement works.  If
           you only want to trap a single exception, use a singleton tuple:
 
             exc_tuple == (MyCustomException,)
 
           - handler: this must be defined as a function with the following
           basic interface: def my_handler(self,etype,value,tb).
 
           This will be made into an instance method (via new.instancemethod)
           of IPython itself, and it will be called if any of the exceptions
           listed in the exc_tuple are caught.  If the handler is None, an
           internal basic one is used, which just prints basic info.
 
         WARNING: by putting in your own exception handler into IPython's main
         execution loop, you run a very good chance of nasty crashes.  This
         facility should only be used if you really know what you are doing."""
 
         assert type(exc_tuple)==type(()) , \
                "The custom exceptions must be given AS A TUPLE."
 
         def dummy_handler(self,etype,value,tb):
             print '*** Simple custom exception handler ***'
             print 'Exception type :',etype
             print 'Exception value:',value
             print 'Traceback      :',tb
             print 'Source code    :',self.code_to_run_src
 
         if handler is None: handler = dummy_handler
 
         self.CustomTB = new.instancemethod(handler,self,self.__class__)
         self.custom_exceptions = exc_tuple
 
     def set_custom_completer(self,completer,pos=0):
         """set_custom_completer(completer,pos=0)
 
         Adds a new custom completer function.
 
         The position argument (defaults to 0) is the index in the completers
         list where you want the completer to be inserted."""
 
         newcomp = new.instancemethod(completer,self.Completer,
                                      self.Completer.__class__)
         self.Completer.matchers.insert(pos,newcomp)
 
     def post_config_initialization(self):
         """Post configuration init method
 
         This is called after the configuration files have been processed to
         'finalize' the initialization."""
         
         # dynamic data that survives through sessions
         # XXX make the filename a config option?
         persist_base = 'persist'
         if self.rc.profile:
             persist_base += '_%s' % self.rc.profile
         self.persist_fname =  os.path.join(self.rc.ipythondir,persist_base)
 
         try:
             self.persist = pickle.load(file(self.persist_fname))
         except:
             self.persist = {}
             
     def init_auto_alias(self):
         """Define some aliases automatically.
 
         These are ALL parameter-less aliases"""
         for alias,cmd in self.auto_alias:
             self.alias_table[alias] = (0,cmd)
 
     def alias_table_validate(self,verbose=0):
         """Update information about the alias table.
 
         In particular, make sure no Python keywords/builtins are in it."""
 
         no_alias = self.no_alias
         for k in self.alias_table.keys():
             if k in no_alias:
                 del self.alias_table[k]
                 if verbose:
                     print ("Deleting alias <%s>, it's a Python "
                            "keyword or builtin." % k)
     
     def set_autoindent(self,value=None):
         """Set the autoindent flag, checking for readline support.
 
         If called with no arguments, it acts as a toggle."""
 
         if not self.has_readline:
             if os.name == 'posix':
                 warn("The auto-indent feature requires the readline library")
             self.autoindent = 0
             return
         if value is None:
             self.autoindent = not self.autoindent
         else:
             self.autoindent = value
 
     def rc_set_toggle(self,rc_field,value=None):
         """Set or toggle a field in IPython's rc config. structure.
 
         If called with no arguments, it acts as a toggle.
 
         If called with a non-existent field, the resulting AttributeError
         exception will propagate out."""
 
         rc_val = getattr(self.rc,rc_field)
         if value is None:
             value = not rc_val
         setattr(self.rc,rc_field,value)
 
     def user_setup(self,ipythondir,rc_suffix,mode='install'):
         """Install the user configuration directory.
 
         Can be called when running for the first time or to upgrade the user's
         .ipython/ directory with the mode parameter. Valid modes are 'install'
         and 'upgrade'."""
 
         def wait():
             try:
                 raw_input("Please press <RETURN> to start IPython.")
             except EOFError:
                 print >> Term.cout
             print '*'*70
 
         cwd = os.getcwd()  # remember where we started
         glb = glob.glob
         print '*'*70
         if mode == 'install':
             print \
 """Welcome to IPython. I will try to create a personal configuration directory
 where you can customize many aspects of IPython's functionality in:\n"""
         else:
             print 'I am going to upgrade your configuration in:'
 
         print ipythondir
 
         rcdirend = os.path.join('IPython','UserConfig')
         cfg = lambda d: os.path.join(d,rcdirend)
         try:
             rcdir = filter(os.path.isdir,map(cfg,sys.path))[0]
         except IOError:
             warning = """
 Installation error. IPython's directory was not found.
 
 Check the following:
 
 The ipython/IPython directory should be in a directory belonging to your
 PYTHONPATH environment variable (that is, it should be in a directory
 belonging to sys.path). You can copy it explicitly there or just link to it.
 
 IPython will proceed with builtin defaults.
 """
             warn(warning)
             wait()
             return
 
         if mode == 'install':
             try:
                 shutil.copytree(rcdir,ipythondir)
                 os.chdir(ipythondir)
                 rc_files = glb("ipythonrc*")
                 for rc_file in rc_files:
                     os.rename(rc_file,rc_file+rc_suffix)
             except:
                 warning = """
 
 There was a problem with the installation:
 %s
 Try to correct it or contact the developers if you think it's a bug.
 IPython will proceed with builtin defaults.""" % sys.exc_info()[1]
                 warn(warning)
                 wait()
                 return
 
         elif mode == 'upgrade':
             try:
                 os.chdir(ipythondir)
             except:
                 print """
 Can not upgrade: changing to directory %s failed. Details:
 %s
 """ % (ipythondir,sys.exc_info()[1])
                 wait()
                 return
             else:
                 sources = glb(os.path.join(rcdir,'[A-Za-z]*'))
                 for new_full_path in sources:
                     new_filename = os.path.basename(new_full_path)
                     if new_filename.startswith('ipythonrc'):
                         new_filename = new_filename + rc_suffix
                     # The config directory should only contain files, skip any
                     # directories which may be there (like CVS)
                     if os.path.isdir(new_full_path):
                         continue
                     if os.path.exists(new_filename):
                         old_file = new_filename+'.old'
                         if os.path.exists(old_file):
                             os.remove(old_file)
                         os.rename(new_filename,old_file)
                     shutil.copy(new_full_path,new_filename)
         else:
             raise ValueError,'unrecognized mode for install:',`mode`
 
         # Fix line-endings to those native to each platform in the config
         # directory.
         try:
             os.chdir(ipythondir)
         except:
             print """
 Problem: changing to directory %s failed.
 Details:
 %s
 
 Some configuration files may have incorrect line endings.  This should not
 cause any problems during execution.  """ % (ipythondir,sys.exc_info()[1])
             wait()
         else:
             for fname in glb('ipythonrc*'):
                 try:
                     native_line_ends(fname,backup=0)
                 except IOError:
                     pass
 
         if mode == 'install':
             print """
 Successful installation!
 
 Please read the sections 'Initial Configuration' and 'Quick Tips' in the
 IPython manual (there are both HTML and PDF versions supplied with the
 distribution) to make sure that your system environment is properly configured
 to take advantage of IPython's features."""
         else:
             print """
 Successful upgrade!
 
 All files in your directory:
 %(ipythondir)s
 which would have been overwritten by the upgrade were backed up with a .old
 extension.  If you had made particular customizations in those files you may
 want to merge them back into the new files.""" % locals()
         wait()
         os.chdir(cwd)
         # end user_setup()
 
     def atexit_operations(self):
         """This will be executed at the time of exit.
 
         Saving of persistent data should be performed here. """
 
         # input history
         self.savehist()
 
         # Cleanup all tempfiles left around
         for tfile in self.tempfiles:
             try:
                 os.unlink(tfile)
             except OSError:
                 pass
 
         # save the "persistent data" catch-all dictionary
         try:
             pickle.dump(self.persist, open(self.persist_fname,"w"))
         except:
             print "*** ERROR *** persistent data saving failed."
         
     def savehist(self):
         """Save input history to a file (via readline library)."""
         try:
             self.readline.write_history_file(self.histfile)
         except:
             print 'Unable to save IPython command history to file: ' + \
                   `self.histfile`
 
     def pre_readline(self):
         """readline hook to be used at the start of each line.
 
         Currently it handles auto-indent only."""
         
         self.readline.insert_text(' '* self.readline_indent)
 
     def init_readline(self):
         """Command history completion/saving/reloading."""
         try:
             import readline
             self.Completer = MagicCompleter(self,
                                             self.user_ns,
                                             self.rc.readline_omit__names,
                                             self.alias_table)
         except ImportError,NameError:
             # If FlexCompleter failed to import, MagicCompleter won't be 
             # defined.  This can happen because of a problem with readline
             self.has_readline = 0
             # no point in bugging windows users with this every time:
             if os.name == 'posix':
                 warn('Readline services not available on this platform.')
         else:
             import atexit
 
             # Platform-specific configuration
             if os.name == 'nt':
                 # readline under Windows modifies the default exit behavior
                 # from being Ctrl-Z/Return to the Unix Ctrl-D one.
                 __builtin__.exit = __builtin__.quit = \
                      ('Use Ctrl-D (i.e. EOF) to exit. '
                       'Use %Exit or %Quit to exit without confirmation.')
                 self.readline_startup_hook = readline.set_pre_input_hook
             else:
                 self.readline_startup_hook = readline.set_startup_hook
 
             # Load user's initrc file (readline config)
             inputrc_name = os.environ.get('INPUTRC')
             if inputrc_name is None:
                 home_dir = get_home_dir()
                 if home_dir is not None:
                     inputrc_name = os.path.join(home_dir,'.inputrc')
             if os.path.isfile(inputrc_name):
                 try:
                     readline.read_init_file(inputrc_name)
                 except:
                     warn('Problems reading readline initialization file <%s>'
                          % inputrc_name)
             
             self.has_readline = 1
             self.readline = readline
             self.readline_indent = 0  # for auto-indenting via readline
             # save this in sys so embedded copies can restore it properly
             sys.ipcompleter = self.Completer.complete
             readline.set_completer(self.Completer.complete)
 
             # Configure readline according to user's prefs
             for rlcommand in self.rc.readline_parse_and_bind:
                 readline.parse_and_bind(rlcommand)
 
             # remove some chars from the delimiters list
             delims = readline.get_completer_delims()
             delims = delims.translate(string._idmap,
                                       self.rc.readline_remove_delims)
             readline.set_completer_delims(delims)
             # otherwise we end up with a monster history after a while:
             readline.set_history_length(1000)
             try:
                 #print '*** Reading readline history'  # dbg
                 readline.read_history_file(self.histfile)
             except IOError:
                 pass  # It doesn't exist yet.
 
             atexit.register(self.atexit_operations)
             del atexit
 
         # Configure auto-indent for all platforms
         self.set_autoindent(self.rc.autoindent)
 
     def showsyntaxerror(self, filename=None):
         """Display the syntax error that just occurred.
 
         This doesn't display a stack trace because there isn't one.
 
         If a filename is given, it is stuffed in the exception instead
         of what was there before (because Python's parser always uses
         "<string>" when reading from a string).
         """
         type, value, sys.last_traceback = sys.exc_info()
         sys.last_type = type
         sys.last_value = value
         if filename and type is SyntaxError:
             # Work hard to stuff the correct filename in the exception
             try:
                 msg, (dummy_filename, lineno, offset, line) = value
             except:
                 # Not the format we expect; leave it alone
                 pass
             else:
                 # Stuff in the right filename
                 try:
                     # Assume SyntaxError is a class exception
                     value = SyntaxError(msg, (filename, lineno, offset, line))
                 except:
                     # If that failed, assume SyntaxError is a string
                     value = msg, (filename, lineno, offset, line)
         self.SyntaxTB(type,value,[])
 
     def debugger(self):
         """Call the pdb debugger."""
 
         if not self.rc.pdb:
             return
         pdb.pm()
 
     def showtraceback(self,exc_tuple = None):
         """Display the exception that just occurred."""
 
         # Though this won't be called by syntax errors in the input line,
         # there may be SyntaxError cases whith imported code.
         if exc_tuple is None:
             type, value, tb = sys.exc_info()
         else:
             type, value, tb = exc_tuple
         if type is SyntaxError:
             self.showsyntaxerror()
         else:
             sys.last_type = type
             sys.last_value = value
             sys.last_traceback = tb
             self.InteractiveTB()
             if self.InteractiveTB.call_pdb and self.has_readline:
                 # pdb mucks up readline, fix it back
                 self.readline.set_completer(self.Completer.complete)
 
     def update_cache(self, line):
         """puts line into cache"""
         self.inputcache.insert(0, line) # This copies the cache every time ... :-(
         if len(self.inputcache) >= self.CACHELENGTH:
             self.inputcache.pop()    # This not :-)
 
     def name_space_init(self):
         """Create local namespace."""
         # We want this to be a method to facilitate embedded initialization.
         code.InteractiveConsole.__init__(self,self.user_ns)
 
     def mainloop(self,banner=None):
         """Creates the local namespace and starts the mainloop.
 
         If an optional banner argument is given, it will override the
         internally created default banner."""
         
         self.name_space_init()
         if self.rc.c:  # Emulate Python's -c option
             self.exec_init_cmd()
         if banner is None:
             if self.rc.banner:
                 banner = self.BANNER+self.banner2
             else:
                 banner = ''
         self.interact(banner)
 
     def exec_init_cmd(self):
         """Execute a command given at the command line.
 
         This emulates Python's -c option."""
 
         sys.argv = ['-c']
         self.push(self.rc.c)
 
     def embed_mainloop(self,header='',local_ns=None,global_ns=None,stack_depth=0):
         """Embeds IPython into a running python program.
 
         Input:
 
           - header: An optional header message can be specified.
 
           - local_ns, global_ns: working namespaces. If given as None, the
           IPython-initialized one is updated with __main__.__dict__, so that
           program variables become visible but user-specific configuration
           remains possible.
 
           - stack_depth: specifies how many levels in the stack to go to
           looking for namespaces (when local_ns and global_ns are None).  This
           allows an intermediate caller to make sure that this function gets
           the namespace from the intended level in the stack.  By default (0)
           it will get its locals and globals from the immediate caller.
 
         Warning: it's possible to use this in a program which is being run by
         IPython itself (via %run), but some funny things will happen (a few
         globals get overwritten). In the future this will be cleaned up, as
         there is no fundamental reason why it can't work perfectly."""
 
         # Patch for global embedding to make sure that things don't overwrite
         # user globals accidentally. Thanks to Richard <[e-mail]>
         # FIXME. Test this a bit more carefully (the if.. is new)
         if local_ns is None and global_ns is None:
             self.user_ns.update(__main__.__dict__)
 
         # Get locals and globals from caller
         if local_ns is None or global_ns is None:
             call_frame = sys._getframe(stack_depth).f_back
 
             if local_ns is None:
                 local_ns = call_frame.f_locals
             if global_ns is None:
                 global_ns = call_frame.f_globals
 
         # Update namespaces and fire up interpreter
         self.user_ns.update(local_ns)
         self.interact(header)
 
         # Remove locals from namespace
-        for k in local_ns.keys():
+        for k in local_ns:
             del self.user_ns[k]
 
     def interact(self, banner=None):
         """Closely emulate the interactive Python console.
 
         The optional banner argument specify the banner to print
         before the first interaction; by default it prints a banner
         similar to the one printed by the real Python interpreter,
         followed by the current class name in parentheses (so as not
         to confuse this with the real interpreter -- since it's so
         close!).
 
         """
         cprt = 'Type "copyright", "credits" or "license" for more information.'
         if banner is None:
             self.write("Python %s on %s\n%s\n(%s)\n" %
                        (sys.version, sys.platform, cprt,
                         self.__class__.__name__))
         else:
             self.write(banner)
 
         more = 0
 
         # Mark activity in the builtins
         __builtin__.__dict__['__IPYTHON__active'] += 1
-        while 1:
-            # This is set by a call to %Exit or %Quit
-            if self.exit_now:
-                break
+
+        # exit_now is set by a call to %Exit or %Quit
+        while not self.exit_now:
             try:
                 if more:
                     prompt = self.outputcache.prompt2
                     if self.autoindent:
                         self.readline_startup_hook(self.pre_readline)
                 else:
                     prompt = self.outputcache.prompt1
                 try:
                     line = self.raw_input(prompt)
                     if self.autoindent:
                         self.readline_startup_hook(None)
                 except EOFError:
                     if self.autoindent:
                         self.readline_startup_hook(None)
                     self.write("\n")
                     if self.rc.confirm_exit:
                         if ask_yes_no('Do you really want to exit ([y]/n)?','y'):
                             break
                     else:
                         break
                 else:
                     more = self.push(line)
                     # Auto-indent management
                     if self.autoindent:
                         if line:
                             ini_spaces = re.match('^(\s+)',line)
                             if ini_spaces:
                                 nspaces = ini_spaces.end()
                             else:
                                 nspaces = 0
                             self.readline_indent = nspaces
 
                             if line[-1] == ':':
                                 self.readline_indent += 4
                             elif re.match(r'^\s+raise|^\s+return',line):
                                 self.readline_indent -= 4
                         else:
                             self.readline_indent = 0
 
             except KeyboardInterrupt:
                 self.write("\nKeyboardInterrupt\n")
                 self.resetbuffer()
                 more = 0
                 # keep cache in sync with the prompt counter:
                 self.outputcache.prompt_count -= 1
 
                 if self.autoindent:
                     self.readline_indent = 0
 
             except bdb.BdbQuit:
                 warn("The Python debugger has exited with a BdbQuit exception.\n"
                      "Because of how pdb handles the stack, it is impossible\n"
                      "for IPython to properly format this particular exception.\n"
                      "IPython will resume normal operation.")
-            except:
-                # We should never get here except in fairly bizarre situations
-                # (or b/c of an IPython bug). One reasonable exception is if
-                # the user sets stdin/out/err to a broken object (or closes
-                # any of them!)
-
-                fixed_in_out_err = 0
-
-                # Call the Term I/O class and have it reopen any stream which
-                # the user might have closed.
-                Term.reopen_all()
-
-                # Do the same manually for sys.stderr/out/in
-                
-                # err first, so we can print at least warnings
-                if sys.__stderr__.closed:
-                    sys.__stderr__ = os.fdopen(os.dup(2),'w',0)
-                    fixed_err_err = 1
-                    print >> sys.__stderr__,"""
-WARNING:
-sys.__stderr__ was closed!
-I've tried to reopen it, but bear in mind that things may not work normally
-from now.  In particular, readline support may have broken.
-"""
-                # Next, check stdin/out    
-                if sys.__stdin__.closed:
-                    sys.__stdin__ = os.fdopen(os.dup(0),'r',0)
-                    fixed_in_out_err = 1
-                    print >> sys.__stderr__,"""
-WARNING:
-sys.__stdin__ was closed!
-I've tried to reopen it, but bear in mind that things may not work normally
-from now.  In particular, readline support may have broken.
-"""
-                if sys.__stdout__.closed:
-                    sys.__stdout__ = os.fdopen(os.dup(1),'w',0)
-                    fixed_in_out_err = 1
-                    print >> sys.__stderr__,"""
-WARNING:
-sys.__stdout__ was closed!
-I've tried to reopen it, but bear in mind that things may not work normally
-from now.  In particular, readline support may have broken.
-"""
-
-                # Now, check mismatch of objects
-                if sys.stdin is not sys.__stdin__:
-                    sys.stdin = sys.__stdin__
-                    fixed_in_out_err = 1
-                    print >> sys.__stderr__,"""
-WARNING:
-sys.stdin has been reset to sys.__stdin__.
-There seemed to be a problem with your sys.stdin.
-"""
-                if sys.stdout is not sys.__stdout__:
-                    sys.stdout = sys.__stdout__
-                    fixed_in_out_err = 1
-                    print >> sys.__stderr__,"""
-WARNING:
-sys.stdout has been reset to sys.__stdout__.
-There seemed to be a problem with your sys.stdout.
-"""
-
-                if sys.stderr is not sys.__stderr__:
-                    sys.stderr = sys.__stderr__
-                    fixed_in_out_err = 1
-                    print >> sys.__stderr__,"""
-WARNING:
-sys.stderr has been reset to sys.__stderr__.
-There seemed to be a problem with your sys.stderr.
-"""
-                # If the problem wasn't a broken out/err, it's an IPython bug
-                # I wish we could ask the user whether to crash or not, but
-                # calling any function at this point messes up the stack.
-                if not fixed_in_out_err:
-                    raise
                 
         # We are off again...
         __builtin__.__dict__['__IPYTHON__active'] -= 1
 
     def excepthook(self, type, value, tb):
       """One more defense for GUI apps that call sys.excepthook.
 
       GUI frameworks like wxPython trap exceptions and call
       sys.excepthook themselves.  I guess this is a feature that
       enables them to keep running after exceptions that would
       otherwise kill their mainloop. This is a bother for IPython
       which excepts to catch all of the program exceptions with a try:
       except: statement.
 
       Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
       any app directly invokes sys.excepthook, it will look to the user like
       IPython crashed.  In order to work around this, we can disable the
       CrashHandler and replace it with this excepthook instead, which prints a
       regular traceback using our InteractiveTB.  In this fashion, apps which
       call sys.excepthook will generate a regular-looking exception from
       IPython, and the CrashHandler will only be triggered by real IPython
       crashes.
 
       This hook should be used sparingly, only in places which are not likely
       to be true IPython errors.
       """
       
       self.InteractiveTB(type, value, tb, tb_offset=0)
       if self.InteractiveTB.call_pdb and self.has_readline:
           self.readline.set_completer(self.Completer.complete)
 
     def call_alias(self,alias,rest=''):
         """Call an alias given its name and the rest of the line.
 
         This function MUST be given a proper alias, because it doesn't make
         any checks when looking up into the alias table.  The caller is
         responsible for invoking it only with a valid alias."""
 
         #print 'ALIAS: <%s>+<%s>' % (alias,rest) # dbg
         nargs,cmd = self.alias_table[alias]
         # Expand the %l special to be the user's input line
         if cmd.find('%l') >= 0:
             cmd = cmd.replace('%l',rest)
             rest = ''
         if nargs==0:
             # Simple, argument-less aliases
             cmd = '%s %s' % (cmd,rest)
         else:
             # Handle aliases with positional arguments
             args = rest.split(None,nargs)
             if len(args)< nargs:
                 error('Alias <%s> requires %s arguments, %s given.' %
                       (alias,nargs,len(args)))
                 return
             cmd = '%s %s' % (cmd % tuple(args[:nargs]),' '.join(args[nargs:]))
         # Now call the macro, evaluating in the user's namespace
         try:
             self.system(cmd)
         except:
             self.showtraceback()
 
     def runlines(self,lines):
         """Run a string of one or more lines of source.
 
         This method is capable of running a string containing multiple source
         lines, as if they had been entered at the IPython prompt.  Since it
         exposes IPython's processing machinery, the given strings can contain
         magic calls (%magic), special shell access (!cmd), etc."""
 
         # We must start with a clean buffer, in case this is run from an
         # interactive IPython session (via a magic, for example).
         self.resetbuffer()
         lines = lines.split('\n')
         more = 0
         for line in lines:
             # skip blank lines so we don't mess up the prompt counter, but do
             # NOT skip even a blank line if we are in a code block (more is
             # true)
             if line or more:
                 more = self.push((self.prefilter(line,more)))
                 # IPython's runsource returns None if there was an error
                 # compiling the code.  This allows us to stop processing right
                 # away, so the user gets the error message at the right place.
                 if more is None:
                     break
         # final newline in case the input didn't have it, so that the code
         # actually does get executed
         if more:
             self.push('\n')
 
     def runsource(self, source, filename="<input>", symbol="single"):
         """Compile and run some source in the interpreter.
 
         Arguments are as for compile_command().
 
         One several things can happen:
 
         1) The input is incorrect; compile_command() raised an
         exception (SyntaxError or OverflowError).  A syntax traceback
         will be printed by calling the showsyntaxerror() method.
 
         2) The input is incomplete, and more input is required;
         compile_command() returned None.  Nothing happens.
 
         3) The input is complete; compile_command() returned a code
         object.  The code is executed by calling self.runcode() (which
         also handles run-time exceptions, except for SystemExit).
 
         The return value is:
 
           - True in case 2
 
           - False in the other cases, unless an exception is raised, where
           None is returned instead.  This can be used by external callers to
           know whether to continue feeding input or not.
 
         The return value can be used to decide whether to use sys.ps1 or
         sys.ps2 to prompt the next line."""
         try:
             code = self.compile(source, filename, symbol)
         except (OverflowError, SyntaxError, ValueError):
             # Case 1
             self.showsyntaxerror(filename)
             return None
 
         if code is None:
             # Case 2
             return True
 
         # Case 3
         # We store the code source and object so that threaded shells and
         # custom exception handlers can access all this info if needed.
         self.code_to_run_src = source
         self.code_to_run = code
         # now actually execute the code object
         if self.runcode(code) == 0:
             return False
         else:
             return None
 
     def runcode(self,code_obj):
         """Execute a code object.
 
         When an exception occurs, self.showtraceback() is called to display a
         traceback.
 
         Return value: a flag indicating whether the code to be run completed
         successfully:
 
           - 0: successful execution.
           - 1: an error occurred.
         """
 
         # Set our own excepthook in case the user code tries to call it
         # directly, so that the IPython crash handler doesn't get triggered
         old_excepthook,sys.excepthook = sys.excepthook, self.excepthook
         outflag = 1  # happens in more places, so it's easier as default
         try:
             try:
                 exec code_obj in self.locals
             finally:
                 # Reset our crash handler in place
                 sys.excepthook = old_excepthook
         except SystemExit:
             self.resetbuffer()
             self.showtraceback()
             warn( __builtin__.exit,level=1)
         except self.custom_exceptions:
             etype,value,tb = sys.exc_info()
             self.CustomTB(etype,value,tb)
         except:
             self.showtraceback()
         else:
             outflag = 0
             if code.softspace(sys.stdout, 0):
                 print
         # Flush out code object which has been run (and source)
         self.code_to_run = None
         self.code_to_run_src = ''
         return outflag
 
     def raw_input(self, prompt=""):
         """Write a prompt and read a line.
 
         The returned line does not include the trailing newline.
         When the user enters the EOF key sequence, EOFError is raised.
 
         The base implementation uses the built-in function
         raw_input(); a subclass may replace this with a different
         implementation.
         """
         return self.prefilter(raw_input_original(prompt),
                               prompt==self.outputcache.prompt2)
         
     def split_user_input(self,line):
         """Split user input into pre-char, function part and rest."""
 
         lsplit = self.line_split.match(line)
         if lsplit is None:  # no regexp match returns None
             try:
                 iFun,theRest = line.split(None,1)
             except ValueError:
                 iFun,theRest = line,''
             pre = re.match('^(\s*)(.*)',line).groups()[0]
         else:
             pre,iFun,theRest = lsplit.groups()
 
         #print 'line:<%s>' % line # dbg
         #print 'pre <%s> iFun <%s> rest <%s>' % (pre,iFun.strip(),theRest) # dbg
         return pre,iFun.strip(),theRest
 
     def _prefilter(self, line, continue_prompt):
         """Calls different preprocessors, depending on the form of line."""
 
         # All handlers *must* return a value, even if it's blank ('').
 
         # Lines are NOT logged here. Handlers should process the line as
         # needed, update the cache AND log it (so that the input cache array
         # stays synced).
 
         # This function is _very_ delicate, and since it's also the one which
         # determines IPython's response to user input, it must be as efficient
         # as possible.  For this reason it has _many_ returns in it, trying
         # always to exit as quickly as it can figure out what it needs to do.
 
         # This function is the main responsible for maintaining IPython's
         # behavior respectful of Python's semantics.  So be _very_ careful if
         # making changes to anything here.
 
         #.....................................................................
         # Code begins
 
         #if line.startswith('%crash'): raise RuntimeError,'Crash now!'  # dbg
 
         # save the line away in case we crash, so the post-mortem handler can
         # record it
         self._last_input_line = line
 
         #print '***line: <%s>' % line # dbg
 
         # the input history needs to track even empty lines
         if not line.strip():
             if not continue_prompt:
                 self.outputcache.prompt_count -= 1
             return self.handle_normal('',continue_prompt)
 
         # print '***cont',continue_prompt  # dbg
         # special handlers are only allowed for single line statements
         if continue_prompt and not self.rc.multi_line_specials:
             return self.handle_normal(line,continue_prompt)
 
         # For the rest, we need the structure of the input
         pre,iFun,theRest = self.split_user_input(line)
         #print 'pre <%s> iFun <%s> rest <%s>' % (pre,iFun,theRest)  # dbg
 
         # First check for explicit escapes in the last/first character
         handler = None
         if line[-1] == self.ESC_HELP:
             handler = self.esc_handlers.get(line[-1])  # the ? can be at the end
         if handler is None:
             # look at the first character of iFun, NOT of line, so we skip
             # leading whitespace in multiline input
             handler = self.esc_handlers.get(iFun[0:1])
         if handler is not None:
             return handler(line,continue_prompt,pre,iFun,theRest)
         # Emacs ipython-mode tags certain input lines
         if line.endswith('# PYTHON-MODE'):
             return self.handle_emacs(line,continue_prompt)
 
         # Next, check if we can automatically execute this thing
 
         # Allow ! in multi-line statements if multi_line_specials is on:
         if continue_prompt and self.rc.multi_line_specials and \
                iFun.startswith(self.ESC_SHELL):
             return self.handle_shell_escape(line,continue_prompt,
                                             pre=pre,iFun=iFun,
                                             theRest=theRest)
 
         # Let's try to find if the input line is a magic fn
         oinfo = None
         if hasattr(self,'magic_'+iFun):
             oinfo = self._ofind(iFun) # FIXME - _ofind is part of Magic
             if oinfo['ismagic']:
                 # Be careful not to call magics when a variable assignment is
                 # being made (ls='hi', for example)
                 if self.rc.automagic and \
                        (len(theRest)==0 or theRest[0] not in '!=()<>,') and \
                        (self.rc.multi_line_specials or not continue_prompt):
                     return self.handle_magic(line,continue_prompt,
                                              pre,iFun,theRest)
                 else:
                     return self.handle_normal(line,continue_prompt)
 
         # If the rest of the line begins with an (in)equality, assginment or
         # function call, we should not call _ofind but simply execute it.
         # This avoids spurious geattr() accesses on objects upon assignment.
         #
         # It also allows users to assign to either alias or magic names true
         # python variables (the magic/alias systems always take second seat to
         # true python code).
         if theRest and theRest[0] in '!=()':
             return self.handle_normal(line,continue_prompt)
 
         if oinfo is None:
             oinfo = self._ofind(iFun) # FIXME - _ofind is part of Magic
         
         if not oinfo['found']:
             return self.handle_normal(line,continue_prompt)
         else:
             #print 'iFun <%s> rest <%s>' % (iFun,theRest) # dbg
             if oinfo['isalias']:
                 return self.handle_alias(line,continue_prompt,
                                              pre,iFun,theRest)
 
             if self.rc.autocall and \
                    not self.re_exclude_auto.match(theRest) and \
                    self.re_fun_name.match(iFun) and \
                    callable(oinfo['obj']) :
                 #print 'going auto'  # dbg
                 return self.handle_auto(line,continue_prompt,pre,iFun,theRest)
             else:
                 #print 'was callable?', callable(oinfo['obj'])  # dbg
                 return self.handle_normal(line,continue_prompt)
 
         # If we get here, we have a normal Python line. Log and return.
         return self.handle_normal(line,continue_prompt)
 
     def _prefilter_dumb(self, line, continue_prompt):
         """simple prefilter function, for debugging"""
         return self.handle_normal(line,continue_prompt)
 
     # Set the default prefilter() function (this can be user-overridden)
     prefilter = _prefilter
 
     def handle_normal(self,line,continue_prompt=None,
                       pre=None,iFun=None,theRest=None):
         """Handle normal input lines. Use as a template for handlers."""
 
         self.log(line,continue_prompt)
         self.update_cache(line)
         return line
 
     def handle_alias(self,line,continue_prompt=None,
                      pre=None,iFun=None,theRest=None):
         """Handle alias input lines. """
 
         theRest = esc_quotes(theRest)
         line_out = "%s%s.call_alias('%s','%s')" % (pre,self.name,iFun,theRest)
         self.log(line_out,continue_prompt)
         self.update_cache(line_out)
         return line_out
 
     def handle_shell_escape(self, line, continue_prompt=None,
                             pre=None,iFun=None,theRest=None):
         """Execute the line in a shell, empty return value"""
 
         # Example of a special handler. Others follow a similar pattern.
         if continue_prompt:  # multi-line statements
             if iFun.startswith('!!'):
                 print 'SyntaxError: !! is not allowed in multiline statements'
                 return pre
             else:
                 cmd = ("%s %s" % (iFun[1:],theRest)).replace('"','\\"')
                 line_out = '%s%s.system("%s")' % (pre,self.name,cmd)
         else: # single-line input
             if line.startswith('!!'):
                 # rewrite iFun/theRest to properly hold the call to %sx and
                 # the actual command to be executed, so handle_magic can work
                 # correctly
                 theRest = '%s %s' % (iFun[2:],theRest)
                 iFun = 'sx'
                 return self.handle_magic('%ssx %s' % (self.ESC_MAGIC,line[2:]),
                                          continue_prompt,pre,iFun,theRest)
             else:
                 cmd = esc_quotes(line[1:])
                 line_out = '%s.system("%s")' % (self.name,cmd)
         # update cache/log and return
         self.log(line_out,continue_prompt)
         self.update_cache(line_out)   # readline cache gets normal line
         return line_out
 
     def handle_magic(self, line, continue_prompt=None,
                      pre=None,iFun=None,theRest=None):
         """Execute magic functions.
 
         Also log them with a prepended # so the log is clean Python."""
 
         cmd = '%sipmagic("%s")' % (pre,esc_quotes('%s %s' % (iFun,theRest)))
         self.log(cmd,continue_prompt)
         self.update_cache(line)
         #print 'in handle_magic, cmd=<%s>' % cmd  # dbg
         return cmd
 
     def handle_auto(self, line, continue_prompt=None,
                     pre=None,iFun=None,theRest=None):
         """Hande lines which can be auto-executed, quoting if requested."""
 
         #print 'pre <%s> iFun <%s> rest <%s>' % (pre,iFun,theRest)  # dbg
         
         # This should only be active for single-line input!
         if continue_prompt:
             return line
 
         if pre == self.ESC_QUOTE:
             # Auto-quote splitting on whitespace
             newcmd = '%s("%s")\n' % (iFun,'", "'.join(theRest.split()) )
         elif pre == self.ESC_QUOTE2:
             # Auto-quote whole string
             newcmd = '%s("%s")\n' % (iFun,theRest)
         else:
             # Auto-paren
             if theRest[0:1] in ('=','['):
                 # Don't autocall in these cases.  They can be either
                 # rebindings of an existing callable's name, or item access
                 # for an object which is BOTH callable and implements
                 # __getitem__.
                 return '%s %s\n' % (iFun,theRest)
             if theRest.endswith(';'):
                 newcmd = '%s(%s);\n' % (iFun.rstrip(),theRest[:-1])
             else:
                 newcmd = '%s(%s)\n' % (iFun.rstrip(),theRest)
 
         print >>Term.cout, self.outputcache.prompt1.auto_rewrite() + newcmd,
         # log what is now valid Python, not the actual user input (without the
         # final newline)
         self.log(newcmd.strip(),continue_prompt)
         return newcmd
 
     def handle_help(self, line, continue_prompt=None,
                     pre=None,iFun=None,theRest=None):
         """Try to get some help for the object.
 
         obj? or ?obj   -> basic information.
         obj?? or ??obj -> more details.
         """
 
         # We need to make sure that we don't process lines which would be
         # otherwise valid python, such as "x=1 # what?"
         try:
             code.compile_command(line)
         except SyntaxError:
             # We should only handle as help stuff which is NOT valid syntax
             if line[0]==self.ESC_HELP:
                 line = line[1:]
             elif line[-1]==self.ESC_HELP:
                 line = line[:-1]
             self.log('#?'+line)
             self.update_cache(line)
             if line:
                 self.magic_pinfo(line)
             else:
                 page(self.usage,screen_lines=self.rc.screen_length)
             return '' # Empty string is needed here!
         except:
             # Pass any other exceptions through to the normal handler
             return self.handle_normal(line,continue_prompt)
         else:
             # If the code compiles ok, we should handle it normally
             return self.handle_normal(line,continue_prompt)
 
     def handle_emacs(self,line,continue_prompt=None,
                     pre=None,iFun=None,theRest=None):
         """Handle input lines marked by python-mode."""
 
         # Currently, nothing is done.  Later more functionality can be added
         # here if needed.
 
         # The input cache shouldn't be updated
 
         return line
 
     def write(self,data):
         """Write a string to the default output"""
         Term.cout.write(data)
 
     def write_err(self,data):
         """Write a string to the default error output"""
         Term.cerr.write(data)
 
     def safe_execfile(self,fname,*where,**kw):
         fname = os.path.expanduser(fname)
 
         # find things also in current directory
         dname = os.path.dirname(fname)
         if not sys.path.count(dname):
             sys.path.append(dname)
 
         try:
             xfile = open(fname)
         except:
             print >> Term.cerr, \
                   'Could not open file <%s> for safe execution.' % fname
             return None
 
         kw.setdefault('islog',0)
         kw.setdefault('quiet',1)
         kw.setdefault('exit_ignore',0)
         first = xfile.readline()
         _LOGHEAD = str(self.LOGHEAD).split('\n',1)[0].strip()
         xfile.close()
         # line by line execution
         if first.startswith(_LOGHEAD) or kw['islog']:
             print 'Loading log file <%s> one line at a time...' % fname
             if kw['quiet']:
                 stdout_save = sys.stdout
                 sys.stdout = StringIO.StringIO()
             try:
                 globs,locs = where[0:2]
             except:
                 try:
                     globs = locs = where[0]
                 except:
                     globs = locs = globals()
             badblocks = []
 
             # we also need to identify indented blocks of code when replaying
             # logs and put them together before passing them to an exec
             # statement. This takes a bit of regexp and look-ahead work in the
             # file. It's easiest if we swallow the whole thing in memory
             # first, and manually walk through the lines list moving the
             # counter ourselves.
             indent_re = re.compile('\s+\S')
             xfile = open(fname)
             filelines = xfile.readlines()
             xfile.close()
             nlines = len(filelines)
             lnum = 0
             while lnum < nlines:
                 line = filelines[lnum]
                 lnum += 1
                 # don't re-insert logger status info into cache
                 if line.startswith('#log#'):
                     continue
                 elif line.startswith('#%s'% self.ESC_MAGIC):
                     self.update_cache(line[1:])
                     line = magic2python(line)
                 elif line.startswith('#!'):
                     self.update_cache(line[1:])
                 else:
                     # build a block of code (maybe a single line) for execution
                     block = line
                     try:
                         next = filelines[lnum] # lnum has already incremented
                     except:
                         next = None
                     while next and indent_re.match(next):
                         block += next
                         lnum += 1
                         try:
                             next = filelines[lnum]
                         except:
                             next = None
                     # now execute the block of one or more lines
                     try:
                         exec block in globs,locs
                         self.update_cache(block.rstrip())
                     except SystemExit:
                         pass
                     except:
                         badblocks.append(block.rstrip())
             if kw['quiet']:  # restore stdout
                 sys.stdout.close()
                 sys.stdout = stdout_save
             print 'Finished replaying log file <%s>' % fname
             if badblocks:
                 print >> sys.stderr, \
                       '\nThe following lines/blocks in file <%s> reported errors:' \
                       % fname
                 for badline in badblocks:
                     print >> sys.stderr, badline
         else:  # regular file execution
             try:
                 execfile(fname,*where)
             except SyntaxError:
                 etype, evalue = sys.exc_info()[0:2]
                 self.SyntaxTB(etype,evalue,[])
                 warn('Failure executing file: <%s>' % fname)
             except SystemExit,status:
                 if not kw['exit_ignore']:
                     self.InteractiveTB()
                     warn('Failure executing file: <%s>' % fname)
             except:
                 self.InteractiveTB()
                 warn('Failure executing file: <%s>' % fname)
 
 #************************* end of file <iplib.py> *****************************

IPython/ipmaker.py

@@ -1,736 +1,725 b''
 # -*- coding: utf-8 -*-
 """
 IPython -- An enhanced Interactive Python
 
 Requires Python 2.1 or better.
 
 This file contains the main make_IPython() starter function.
 
-$Id: ipmaker.py 582 2005-05-13 21:20:00Z fperez $"""
+$Id: ipmaker.py 638 2005-07-18 03:01:41Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2004 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Fernando']
 __license__ = Release.license
 __version__ = Release.version
 
 credits._Printer__data = """
 Python: %s
 
 IPython: Fernando Perez, Janko Hauser, Nathan Gray, and many users.
 See http://ipython.scipy.org for more information.""" \
 % credits._Printer__data
 
 copyright._Printer__data += """
 
 Copyright (c) 2001-2004 Fernando Perez, Janko Hauser, Nathan Gray.
 All Rights Reserved."""
 
 #****************************************************************************
 # Required modules
 
 # From the standard library
 import __main__, __builtin__
 import os,sys,types,re
 from pprint import pprint,pformat
 
 # Our own
 from IPython import DPyGetOpt
 from IPython.Struct import Struct
 from IPython.OutputTrap import OutputTrap
 from IPython.ConfigLoader import ConfigLoader
 from IPython.iplib import InteractiveShell,qw_lol,import_fail_info
 from IPython.usage import cmd_line_usage,interactive_usage
 from IPython.Prompts import CachedOutput
 from IPython.genutils import *
 
 #-----------------------------------------------------------------------------
 def make_IPython(argv=None,user_ns=None,debug=1,rc_override=None,
                  shell_class=InteractiveShell,embedded=False,**kw):
     """This is a dump of IPython into a single function.
 
     Later it will have to be broken up in a sensible manner.
 
     Arguments:
 
     - argv: a list similar to sys.argv[1:].  It should NOT contain the desired
     script name, b/c DPyGetOpt strips the first argument only for the real
     sys.argv.
 
     - user_ns: a dict to be used as the user's namespace."""
 
     #----------------------------------------------------------------------
     # Defaults and initialization
     
     # For developer debugging, deactivates crash handler and uses pdb.
     DEVDEBUG = False
 
     if argv is None:
         argv = sys.argv
 
     # __IP is the main global that lives throughout and represents the whole
     # application. If the user redefines it, all bets are off as to what
     # happens.
 
     # __IP is the name of he global which the caller will have accessible as
     # __IP.name. We set its name via the first parameter passed to
     # InteractiveShell:
 
     IP = shell_class('__IP',user_ns=user_ns,**kw)
 
     # Put 'help' in the user namespace
-    try:
         from site import _Helper
-    except ImportError:
-        # Use the _Helper class from Python 2.2 for older Python versions
-        class _Helper:
-            def __repr__(self):
-                return "Type help() for interactive help, " \
-                       "or help(object) for help about object."
-            def __call__(self, *args, **kwds):
-                import pydoc
-                return pydoc.help(*args, **kwds)
-    else:
         IP.user_ns['help'] = _Helper()
 
     if DEVDEBUG:
         # For developer debugging only (global flag)
         from IPython import ultraTB
         sys.excepthook = ultraTB.VerboseTB(call_pdb=1)
     else:
         # IPython itself shouldn't crash. This will produce a detailed
         # post-mortem if it does
         from IPython import CrashHandler
         sys.excepthook = CrashHandler.CrashHandler(IP)
 
     IP.BANNER_PARTS = ['Python %s\n'
                          'Type "copyright", "credits" or "license" '
                          'for more information.\n'
                          % (sys.version.split('\n')[0],),
                          "IPython %s -- An enhanced Interactive Python."
                          % (__version__,),
 """?       -> Introduction to IPython's features.
 %magic  -> Information about IPython's 'magic' % functions.
 help    -> Python's own help system.
 object? -> Details about 'object'. ?object also works, ?? prints more.
 """ ]
 
     IP.usage = interactive_usage
 
     # Platform-dependent suffix and directory names
     if os.name == 'posix':
         rc_suffix = ''
         ipdir_def = '.ipython'
     else:
         rc_suffix = '.ini'
         ipdir_def = '_ipython'
 
     # default directory for configuration
     ipythondir = os.path.abspath(os.environ.get('IPYTHONDIR',
                                  os.path.join(IP.home_dir,ipdir_def)))
 
     # we need the directory where IPython itself is installed
     import IPython
     IPython_dir = os.path.dirname(IPython.__file__)
     del IPython
     
     #-------------------------------------------------------------------------
     # Command line handling
 
     # Valid command line options (uses DPyGetOpt syntax, like Perl's
     # GetOpt::Long)
 
     # Any key not listed here gets deleted even if in the file (like session
     # or profile). That's deliberate, to maintain the rc namespace clean.
 
     # Each set of options appears twice: under _conv only the names are
     # listed, indicating which type they must be converted to when reading the
     # ipythonrc file. And under DPyGetOpt they are listed with the regular
     # DPyGetOpt syntax (=s,=i,:f,etc).
 
     # Make sure there's a space before each end of line (they get auto-joined!)
     cmdline_opts = ('autocall! autoindent! automagic! banner! cache_size|cs=i '
                     'c=s classic|cl color_info! colors=s confirm_exit! '
                     'debug! deep_reload! editor=s log|l messages! nosep pdb! '
                     'pprint! prompt_in1|pi1=s prompt_in2|pi2=s prompt_out|po=s '
                     'quick screen_length|sl=i prompts_pad_left=i '
                     'logfile|lf=s logplay|lp=s profile|p=s '
                     'readline! readline_merge_completions! '
                     'readline_omit__names! '
                     'rcfile=s separate_in|si=s separate_out|so=s '
                     'separate_out2|so2=s xmode=s '
                     'magic_docstrings system_verbose! '
                     'multi_line_specials!')
 
     # Options that can *only* appear at the cmd line (not in rcfiles).
     
     # The "ignore" option is a kludge so that Emacs buffers don't crash, since
     # the 'C-c !' command in emacs automatically appends a -i option at the end.
     cmdline_only = ('help ignore|i ipythondir=s Version upgrade '
                     'gthread! qthread! wthread! pylab! tk!')
 
     # Build the actual name list to be used by DPyGetOpt
     opts_names = qw(cmdline_opts) + qw(cmdline_only)
 
     # Set sensible command line defaults.
     # This should have everything from  cmdline_opts and cmdline_only
     opts_def = Struct(autocall = 1,
                       autoindent=0,
                       automagic = 1,
                       banner = 1,
                       cache_size = 1000,
                       c = '',
                       classic = 0,
                       colors = 'NoColor',
                       color_info = 0,
                       confirm_exit = 1,
                       debug = 0,
                       deep_reload = 0,
                       editor = '0',
                       help = 0,
                       ignore = 0,
                       ipythondir = ipythondir,
                       log = 0,
                       logfile = '',
                       logplay = '',
                       multi_line_specials = 1,
                       messages = 1,
                       nosep = 0,
                       pdb = 0,
                       pprint = 0,
                       profile = '',
                       prompt_in1 = 'In [\\#]:',
                       prompt_in2 = '   .\\D.:',
                       prompt_out = 'Out[\\#]:',
                       prompts_pad_left = 1,
                       quick = 0,
                       readline = 1,
                       readline_merge_completions = 1,
                       readline_omit__names = 0,
                       rcfile = 'ipythonrc' + rc_suffix,
                       screen_length = 0,
                       separate_in = '\n',
                       separate_out = '\n',
                       separate_out2 = '',
                       system_verbose = 0,
                       gthread = 0,
                       qthread = 0,
                       wthread = 0,
                       pylab = 0,
                       tk = 0,
                       upgrade = 0,
                       Version = 0,
                       xmode = 'Verbose',
                       magic_docstrings = 0,  # undocumented, for doc generation
                       )
     
     # Things that will *only* appear in rcfiles (not at the command line).
     # Make sure there's a space before each end of line (they get auto-joined!)
     rcfile_opts = { qwflat: 'include import_mod import_all execfile ',
                     qw_lol: 'import_some ',
                     # for things with embedded whitespace:
                     list_strings:'execute alias readline_parse_and_bind ',
                     # Regular strings need no conversion:
                     None:'readline_remove_delims ',
                     }
     # Default values for these
     rc_def = Struct(include = [],
                     import_mod = [], 
                     import_all = [],
                     import_some = [[]],
                     execute = [],
                     execfile = [],
                     alias = [],
                     readline_parse_and_bind = [],
                     readline_remove_delims = '',
                     )
 
     # Build the type conversion dictionary from the above tables:
     typeconv = rcfile_opts.copy()
     typeconv.update(optstr2types(cmdline_opts))
 
     # FIXME: the None key appears in both, put that back together by hand. Ugly!
     typeconv[None] += ' ' + rcfile_opts[None]
 
     # Remove quotes at ends of all strings (used to protect spaces)
     typeconv[unquote_ends] = typeconv[None]
     del typeconv[None]
 
     # Build the list we'll use to make all config decisions with defaults:
     opts_all = opts_def.copy()
     opts_all.update(rc_def)
 
     # Build conflict resolver for recursive loading of config files:
     # - preserve means the outermost file maintains the value, it is not
     # overwritten if an included file has the same key.
     # - add_flip applies + to the two values, so it better make sense to add
     # those types of keys. But it flips them first so that things loaded
     # deeper in the inclusion chain have lower precedence.
     conflict = {'preserve': ' '.join([ typeconv[int],
                                        typeconv[unquote_ends] ]),
                 'add_flip': ' '.join([ typeconv[qwflat],
                                        typeconv[qw_lol],
                                        typeconv[list_strings] ])
                 }
 
     # Now actually process the command line
     getopt = DPyGetOpt.DPyGetOpt()
     getopt.setIgnoreCase(0)
 
     getopt.parseConfiguration(opts_names)
 
     try:
         getopt.processArguments(argv)
     except:
         print cmd_line_usage
         warn('\nError in Arguments: ' + `sys.exc_value`)
         sys.exit()
 
     # convert the options dict to a struct for much lighter syntax later
     opts = Struct(getopt.optionValues)
     args = getopt.freeValues
 
     # this is the struct (which has default values at this point) with which
     # we make all decisions:
     opts_all.update(opts)
 
     # Options that force an immediate exit
     if opts_all.help:
         page(cmd_line_usage)
         sys.exit()
 
     if opts_all.Version:
         print __version__
         sys.exit()
 
     if opts_all.magic_docstrings:
         IP.magic_magic('-latex')
         sys.exit()
 
     # Create user config directory if it doesn't exist. This must be done
     # *after* getting the cmd line options.
     if not os.path.isdir(opts_all.ipythondir):
         IP.user_setup(opts_all.ipythondir,rc_suffix,'install')
 
     # upgrade user config files while preserving a copy of the originals
     if opts_all.upgrade:
         IP.user_setup(opts_all.ipythondir,rc_suffix,'upgrade')
 
     # check mutually exclusive options in the *original* command line
     mutex_opts(opts,[qw('log logfile'),qw('rcfile profile'),
                      qw('classic profile'),qw('classic rcfile')])
 
     # default logfilename used when -log is called.
     IP.LOGDEF = 'ipython.log'
 
     #---------------------------------------------------------------------------
     # Log replay
     
     # if -logplay, we need to 'become' the other session. That basically means
     # replacing the current command line environment with that of the old
     # session and moving on.
 
     # this is needed so that later we know we're in session reload mode, as
     # opts_all will get overwritten:
     load_logplay = 0
 
     if opts_all.logplay:
         load_logplay = opts_all.logplay 
         opts_debug_save = opts_all.debug
         try:
             logplay = open(opts_all.logplay)
         except IOError:
             if opts_all.debug: IP.InteractiveTB()
             warn('Could not open logplay file '+`opts_all.logplay`)
             # restore state as if nothing had happened and move on, but make
             # sure that later we don't try to actually load the session file
             logplay = None
             load_logplay = 0
             del opts_all.logplay
         else:
             try: 
                 logplay.readline()
                 logplay.readline();
                 # this reloads that session's command line
                 cmd = logplay.readline()[6:] 
                 exec cmd
                 # restore the true debug flag given so that the process of
                 # session loading itself can be monitored.
                 opts.debug = opts_debug_save
                 # save the logplay flag so later we don't overwrite the log
                 opts.logplay = load_logplay
                 # now we must update our own structure with defaults
                 opts_all.update(opts)
                 # now load args
                 cmd = logplay.readline()[6:] 
                 exec cmd
                 logplay.close()
             except:
                 logplay.close()
                 if opts_all.debug: IP.InteractiveTB()
                 warn("Logplay file lacking full configuration information.\n"
                      "I'll try to read it, but some things may not work.")
 
     #-------------------------------------------------------------------------
     # set up output traps: catch all output from files, being run, modules
     # loaded, etc. Then give it to the user in a clean form at the end.
     
     msg_out = 'Output messages. '
     msg_err = 'Error messages. '
     msg_sep = '\n'
     msg = Struct(config    = OutputTrap('Configuration Loader',msg_out,
                                         msg_err,msg_sep,debug,
                                         quiet_out=1),
                  user_exec = OutputTrap('User File Execution',msg_out,
                                         msg_err,msg_sep,debug),
                  logplay   = OutputTrap('Log Loader',msg_out,
                                         msg_err,msg_sep,debug),
                  summary = ''
                  )
 
     #-------------------------------------------------------------------------
     # Process user ipythonrc-type configuration files
 
     # turn on output trapping and log to msg.config
     # remember that with debug on, trapping is actually disabled
     msg.config.trap_all()
 
     # look for rcfile in current or default directory
     try:
         opts_all.rcfile = filefind(opts_all.rcfile,opts_all.ipythondir)
     except IOError:
         if opts_all.debug:  IP.InteractiveTB()
         warn('Configuration file %s not found. Ignoring request.'
              % (opts_all.rcfile) )
 
     # 'profiles' are a shorthand notation for config filenames
     if opts_all.profile:
         try:
             opts_all.rcfile = filefind('ipythonrc-' + opts_all.profile
                                        + rc_suffix,
                                        opts_all.ipythondir)
         except IOError:
            if opts_all.debug:  IP.InteractiveTB()
            opts.profile = ''  # remove profile from options if invalid
            warn('Profile configuration file %s not found. Ignoring request.'
                 % (opts_all.profile) )
 
     # load the config file
     rcfiledata = None
     if opts_all.quick:
         print 'Launching IPython in quick mode. No config file read.'
     elif opts_all.classic:
         print 'Launching IPython in classic mode. No config file read.'
     elif opts_all.rcfile:
         try:
             cfg_loader = ConfigLoader(conflict)
             rcfiledata = cfg_loader.load(opts_all.rcfile,typeconv,
                                          'include',opts_all.ipythondir,
                                          purge = 1,
                                          unique = conflict['preserve'])
         except:
             IP.InteractiveTB()
             warn('Problems loading configuration file '+
                  `opts_all.rcfile`+
                  '\nStarting with default -bare bones- configuration.')
     else:
         warn('No valid configuration file found in either currrent directory\n'+
              'or in the IPython config. directory: '+`opts_all.ipythondir`+
              '\nProceeding with internal defaults.')
 
     #------------------------------------------------------------------------
     # Set exception handlers in mode requested by user.
     otrap = OutputTrap(trap_out=1)  # trap messages from magic_xmode
     IP.magic_xmode(opts_all.xmode)
     otrap.release_out()
 
     #------------------------------------------------------------------------
     # Execute user config
 
     # first, create a valid config structure with the right precedence order:
     # defaults < rcfile < command line
     IP.rc = rc_def.copy()
     IP.rc.update(opts_def)
     if rcfiledata:
         # now we can update 
         IP.rc.update(rcfiledata)
     IP.rc.update(opts)
     IP.rc.update(rc_override)
 
     # Store the original cmd line for reference:
     IP.rc.opts = opts
     IP.rc.args = args
 
     # create a *runtime* Struct like rc for holding parameters which may be
     # created and/or modified by runtime user extensions.
     IP.runtime_rc = Struct()
 
     # from this point on, all config should be handled through IP.rc,
     # opts* shouldn't be used anymore.
 
     # add personal .ipython dir to sys.path so that users can put things in
     # there for customization
     sys.path.append(IP.rc.ipythondir)
     sys.path.insert(0, '') # add . to sys.path. Fix from Prabhu Ramachandran
     
     # update IP.rc with some special things that need manual
     # tweaks. Basically options which affect other options. I guess this
     # should just be written so that options are fully orthogonal and we
     # wouldn't worry about this stuff!
 
     if IP.rc.classic:
         IP.rc.quick = 1
         IP.rc.cache_size = 0
         IP.rc.pprint = 0
         IP.rc.prompt_in1 = '>>> '
         IP.rc.prompt_in2 = '... '
         IP.rc.prompt_out = ''
         IP.rc.separate_in = IP.rc.separate_out = IP.rc.separate_out2 = '0'
         IP.rc.colors = 'NoColor'
         IP.rc.xmode = 'Plain'
 
     # configure readline
     # Define the history file for saving commands in between sessions
     if IP.rc.profile:
         histfname = 'history-%s' % IP.rc.profile
     else:
         histfname = 'history'
     IP.histfile = os.path.join(opts_all.ipythondir,histfname)
     # Load readline proper
     if IP.rc.readline:
         IP.init_readline()
 
     # update exception handlers with rc file status
     otrap.trap_out()  # I don't want these messages ever.
     IP.magic_xmode(IP.rc.xmode)
     otrap.release_out()
 
     # activate logging if requested and not reloading a log
     if IP.rc.logplay:
         IP.magic_logstart(IP.rc.logplay + ' append')
     elif  IP.rc.logfile:
         IP.magic_logstart(IP.rc.logfile)
     elif IP.rc.log:
         IP.magic_logstart()
 
     # find user editor so that it we don't have to look it up constantly
     if IP.rc.editor.strip()=='0':
         try:
             ed = os.environ['EDITOR']
         except KeyError:
             if os.name == 'posix':
                 ed = 'vi'  # the only one guaranteed to be there!
             else:
                 ed = 'notepad' # same in Windows!
         IP.rc.editor = ed
 
     # Recursive reload
     try:
         from IPython import deep_reload
         if IP.rc.deep_reload:
             __builtin__.reload = deep_reload.reload
         else:
             __builtin__.dreload = deep_reload.reload
         del deep_reload
     except ImportError:
         pass
 
     # Save the current state of our namespace so that the interactive shell
     # can later know which variables have been created by us from config files
     # and loading. This way, loading a file (in any way) is treated just like
     # defining things on the command line, and %who works as expected.
 
     # DON'T do anything that affects the namespace beyond this point!
     IP.internal_ns = __main__.__dict__.copy()
 
     #IP.internal_ns.update(locals()) # so our stuff doesn't show up in %who
 
     # Now run through the different sections of the users's config
     if IP.rc.debug:
         print 'Trying to execute the following configuration structure:'
         print '(Things listed first are deeper in the inclusion tree and get'
         print 'loaded first).\n'
         pprint(IP.rc.__dict__)
         
     for mod in IP.rc.import_mod:
         try:
             exec 'import '+mod in IP.user_ns
         except :
             IP.InteractiveTB()
             import_fail_info(mod)
 
     for mod_fn in IP.rc.import_some:
         if mod_fn == []: break
         mod,fn = mod_fn[0],','.join(mod_fn[1:])
         try:
             exec 'from '+mod+' import '+fn in IP.user_ns
         except :
             IP.InteractiveTB()
             import_fail_info(mod,fn)
 
     for mod in IP.rc.import_all:
         try:
             exec 'from '+mod+' import *' in IP.user_ns
         except :
             IP.InteractiveTB()
             import_fail_info(mod)
 
     for code in IP.rc.execute:
         try:
             exec code in IP.user_ns
         except:
             IP.InteractiveTB()
             warn('Failure executing code: ' + `code`)
 
     # Execute the files the user wants in ipythonrc
     for file in IP.rc.execfile:
         try:
             file = filefind(file,sys.path+[IPython_dir])
         except IOError:
             warn(itpl('File $file not found. Skipping it.'))
         else:
             IP.safe_execfile(os.path.expanduser(file),IP.user_ns)
 
     # Load user aliases
     for alias in IP.rc.alias:
         IP.magic_alias(alias)
 
     # release stdout and stderr and save config log into a global summary
     msg.config.release_all()
     if IP.rc.messages:
         msg.summary += msg.config.summary_all()
 
     #------------------------------------------------------------------------
     # Setup interactive session
 
     # Now we should be fully configured. We can then execute files or load
     # things only needed for interactive use. Then we'll open the shell.
 
     # Take a snapshot of the user namespace before opening the shell. That way
     # we'll be able to identify which things were interactively defined and
     # which were defined through config files.
     IP.user_config_ns = IP.user_ns.copy()
 
     # Force reading a file as if it were a session log. Slower but safer.
     if load_logplay:
         print 'Replaying log...'
         try:
             if IP.rc.debug:
                 logplay_quiet = 0
             else:
                  logplay_quiet = 1
 
             msg.logplay.trap_all()
             IP.safe_execfile(load_logplay,IP.user_ns,
                              islog = 1, quiet = logplay_quiet)
             msg.logplay.release_all()
             if IP.rc.messages:
                 msg.summary += msg.logplay.summary_all()
         except:
             warn('Problems replaying logfile %s.' % load_logplay)
             IP.InteractiveTB()
 
     # Load remaining files in command line
     msg.user_exec.trap_all()
 
     # Do NOT execute files named in the command line as scripts to be loaded
     # by embedded instances.  Doing so has the potential for an infinite
     # recursion if there are exceptions thrown in the process.
 
     # XXX FIXME: the execution of user files should be moved out to after
     # ipython is fully initialized, just as if they were run via %run at the
     # ipython prompt.  This would also give them the benefit of ipython's
     # nice tracebacks.
     
     if not embedded and IP.rc.args:
         name_save = IP.user_ns['__name__']
         IP.user_ns['__name__'] = '__main__'
         try:
             # Set our own excepthook in case the user code tries to call it
             # directly. This prevents triggering the IPython crash handler.
             old_excepthook,sys.excepthook = sys.excepthook, IP.excepthook
             for run in args:
                 IP.safe_execfile(run,IP.user_ns)
         finally:
             # Reset our crash handler in place
             sys.excepthook = old_excepthook
             
         IP.user_ns['__name__'] = name_save
         
     msg.user_exec.release_all()
     if IP.rc.messages:
         msg.summary += msg.user_exec.summary_all()
 
     # since we can't specify a null string on the cmd line, 0 is the equivalent:
     if IP.rc.nosep:
         IP.rc.separate_in = IP.rc.separate_out = IP.rc.separate_out2 = '0'
     if IP.rc.separate_in == '0': IP.rc.separate_in = ''
     if IP.rc.separate_out == '0': IP.rc.separate_out = ''
     if IP.rc.separate_out2 == '0': IP.rc.separate_out2 = ''
     IP.rc.separate_in = IP.rc.separate_in.replace('\\n','\n')
     IP.rc.separate_out = IP.rc.separate_out.replace('\\n','\n')
     IP.rc.separate_out2 = IP.rc.separate_out2.replace('\\n','\n')
 
     # Determine how many lines at the bottom of the screen are needed for
     # showing prompts, so we can know wheter long strings are to be printed or
     # paged:
     num_lines_bot = IP.rc.separate_in.count('\n')+1
     IP.rc.screen_length = IP.rc.screen_length - num_lines_bot
     # Initialize cache, set in/out prompts and printing system
     IP.outputcache = CachedOutput(IP.rc.cache_size,
                                   IP.rc.pprint,
                                   input_sep = IP.rc.separate_in,
                                   output_sep = IP.rc.separate_out,
                                   output_sep2 = IP.rc.separate_out2,
                                   ps1 = IP.rc.prompt_in1,
                                   ps2 = IP.rc.prompt_in2,
                                   ps_out = IP.rc.prompt_out,
                                   user_ns = IP.user_ns,
                                   input_hist = IP.input_hist,
                                   pad_left = IP.rc.prompts_pad_left)
 
     # Set user colors (don't do it in the constructor above so that it doesn't
     # crash if colors option is invalid)
     IP.magic_colors(IP.rc.colors)
     
     # user may have over-ridden the default print hook:
     try:
         IP.outputcache.__class__.display = IP.hooks.display
     except AttributeError:
         pass
 
     # Set calling of pdb on exceptions
     IP.InteractiveTB.call_pdb = IP.rc.pdb
     
     # I don't like assigning globally to sys, because it means when embedding
     # instances, each embedded instance overrides the previous choice. But
     # sys.displayhook seems to be called internally by exec, so I don't see a
     # way around it.
     sys.displayhook = IP.outputcache
 
     # we need to know globally if we're caching i/o or not
     IP.do_full_cache = IP.outputcache.do_full_cache
     
     # configure startup banner
     if IP.rc.c:  # regular python doesn't print the banner with -c
         IP.rc.banner = 0
     if IP.rc.banner:
         IP.BANNER = '\n'.join(IP.BANNER_PARTS)
     else:
         IP.BANNER = ''
 
     if IP.rc.profile: IP.BANNER += '\nIPython profile: '+IP.rc.profile+'\n'
 
     # add message log (possibly empty)
     IP.BANNER += msg.summary
 
     IP.post_config_initialization()
 
     return IP
 #************************ end of file <ipmaker.py> **************************