Merge in all development done in bzr since February 16 2008.

At that time, a clean bzr branch was started from the SVN tree, but
without SVN history.  That SVN history has now been used as the basis
of this branch, and the development done on the history-less BZR
branch has been added and is the content of this merge.  

This branch will be the new official main line of development in
Launchpad (equivalent to the old SVN trunk).

.checkeol

@@ -0,0 +1,2 b''
+[LF]
+*.py

IPython/Extensions/ipy_autoreload.py

@@ -0,0 +1,244 b''
+"""
+IPython extension: autoreload modules before executing the next line
+
+Try:: 
+
+    %autoreload?
+
+for documentation.
+"""
+
+# Pauli Virtanen <[e-mail]>, 2008.
+# Thomas Heller, 2000.
+#
+# This IPython module is written by Pauli Virtanen, based on the autoreload
+# code by Thomas Heller.
+
+#------------------------------------------------------------------------------
+# Autoreload functionality
+#------------------------------------------------------------------------------
+
+import time, os, threading, sys, types, imp, inspect, traceback, atexit
+
+def _get_compiled_ext():
+    """Official way to get the extension of compiled files (.pyc or .pyo)"""
+    for ext, mode, typ in imp.get_suffixes():
+        if typ == imp.PY_COMPILED:
+            return ext
+
+PY_COMPILED_EXT = _get_compiled_ext()
+
+class ModuleReloader(object):
+    skipped = {}
+    """Modules that failed to reload: {module: mtime-on-failed-reload, ...}"""
+    
+    modules = {}
+    """Modules specially marked as autoreloadable."""
+
+    skip_modules = {}
+    """Modules specially marked as not autoreloadable."""
+
+    check_all = True
+    """Autoreload all modules, not just those listed in 'modules'"""
+    
+    def check(self, check_all=False):
+        """Check whether some modules need to be reloaded."""
+        
+        if check_all or self.check_all:
+            modules = sys.modules.keys()
+        else:
+            modules = self.modules.keys()
+        
+        for modname in modules:
+            m = sys.modules.get(modname, None)
+
+            if modname in self.skip_modules:
+                continue
+            
+            if not hasattr(m, '__file__'):
+                continue
+            
+            if m.__name__ == '__main__':
+                # we cannot reload(__main__)
+                continue
+            
+            filename = m.__file__
+            dirname = os.path.dirname(filename)
+            path, ext = os.path.splitext(filename)
+            
+            if ext.lower() == '.py':
+                ext = PY_COMPILED_EXT
+                filename = os.path.join(dirname, path + PY_COMPILED_EXT)
+            
+            if ext != PY_COMPILED_EXT:
+                continue
+            
+            try:
+                pymtime = os.stat(filename[:-1]).st_mtime
+                if pymtime <= os.stat(filename).st_mtime:
+                    continue
+                if self.skipped.get(filename[:-1], None) == pymtime:
+                    continue
+            except OSError:
+                continue
+            
+            try:
+                superreload(m)
+                if filename[:-1] in self.skipped:
+                    del self.skipped[filename[:-1]]
+            except:
+                self.skipped[filename[:-1]] = pymtime
+
+def update_function(old, new, attrnames):
+    for name in attrnames:
+        setattr(old, name, getattr(new, name))
+
+def superreload(module, reload=reload):
+    """Enhanced version of the builtin reload function.
+    
+    superreload replaces the class dictionary of every top-level
+    class in the module with the new one automatically,
+    as well as every function's code object.
+    
+    """
+
+    module = reload(module)
+    
+    # iterate over all objects and update them
+    count = 0
+    for name, new_obj in module.__dict__.items():
+        key = (module.__name__, name)
+        if _old_objects.has_key(key):
+            for old_obj in _old_objects[key]:
+                if type(new_obj) == types.ClassType:
+                    old_obj.__dict__.update(new_obj.__dict__)
+                    count += 1
+                elif type(new_obj) == types.FunctionType:
+                    update_function(old_obj,
+                           new_obj,
+                           "func_code func_defaults func_doc".split())
+                    count += 1
+                elif type(new_obj) == types.MethodType:
+                    update_function(old_obj.im_func,
+                           new_obj.im_func,
+                           "func_code func_defaults func_doc".split())
+                    count += 1
+    
+    return module
+
+reloader = ModuleReloader()
+
+#------------------------------------------------------------------------------
+# IPython monkey-patching
+#------------------------------------------------------------------------------
+
+import IPython.iplib
+
+autoreload_enabled = False
+
+def runcode_hook(self):
+    if not autoreload_enabled:
+        raise IPython.ipapi.TryNext
+    try:
+        reloader.check()
+    except:
+        pass
+
+
+def enable_autoreload():
+    global autoreload_enabled
+    autoreload_enabled = True
+    
+
+def disable_autoreload():
+    global autoreload_enabled
+    autoreload_enabled = False
+    
+#------------------------------------------------------------------------------
+# IPython connectivity
+#------------------------------------------------------------------------------
+
+import IPython.ipapi
+ip = IPython.ipapi.get()
+
+def autoreload_f(self, parameter_s=''):
+    r""" %autoreload => Reload modules automatically
+
+    %autoreload
+    Reload all modules (except thoses excluded by %aimport) automatically now.
+
+    %autoreload 1
+    Reload all modules imported with %aimport every time before executing
+    the Python code typed.
+
+    %autoreload 2
+    Reload all modules (except thoses excluded by %aimport) every time
+    before executing the Python code typed.
+
+    Reloading Python modules in a reliable way is in general
+    difficult, and unexpected things may occur. Some of the common
+    caveats relevant for 'autoreload' are:
+
+    - Modules are not reloaded in any specific order, and no dependency
+      analysis is done. For example, modules with 'from xxx import foo'
+      retain old versions of 'foo' when 'xxx' is autoreloaded.
+    - Functions or objects imported from the autoreloaded module to
+      the interactive namespace are not updated.
+    - C extension modules cannot be reloaded, and so cannot be
+      autoreloaded.
+    """
+    if parameter_s == '':
+        reloader.check(True)
+    elif parameter_s == '0':
+        disable_autoreload()
+    elif parameter_s == '1':
+        reloader.check_all = False
+        enable_autoreload()
+    elif parameter_s == '2':
+        reloader.check_all = True
+        enable_autoreload()
+
+def aimport_f(self, parameter_s=''):
+    """%aimport => Import modules for automatic reloading.
+
+    %aimport
+    List modules to automatically import and not to import.
+
+    %aimport foo
+    Import module 'foo' and mark it to be autoreloaded for %autoreload 1
+
+    %aimport -foo
+    Mark module 'foo' to not be autoreloaded for %autoreload 1
+
+    """
+    
+    modname = parameter_s
+    if not modname:
+        to_reload = reloader.modules.keys()
+        to_reload.sort()
+        to_skip = reloader.skip_modules.keys()
+        to_skip.sort()
+        if reloader.check_all:
+            print "Modules to reload:\nall-expect-skipped"
+        else:
+            print "Modules to reload:\n%s" % ' '.join(to_reload)
+        print "\nModules to skip:\n%s" % ' '.join(to_skip)
+    elif modname.startswith('-'):
+        modname = modname[1:]
+        try: del reloader.modules[modname]
+        except KeyError: pass
+        reloader.skip_modules[modname] = True
+    else:
+        try: del reloader.skip_modules[modname]
+        except KeyError: pass
+        reloader.modules[modname] = True
+
+        mod = __import__(modname)
+        ip.to_user_ns({modname: mod})
+
+def init():
+    ip.expose_magic('autoreload', autoreload_f)
+    ip.expose_magic('aimport', aimport_f)
+    ip.set_hook('pre_runcode_hook', runcode_hook)
+    
+init()
  No newline at end of file

IPython/Extensions/ipy_bzr.py

@@ -0,0 +1,343 b''
+""" Extension for bzr command tab completer. Supports comlpeting commands and options
+
+Unlike the core IPython, you should note that this extension is under GPL, not BSD.
+
+Based on "shell" bzr plugin by Aaron Bentley, license is below. The IPython additions
+are at the bottom of the file, the rest is left untouched.
+
+Must be loaded with ip.load('ipy_bzr') 
+
+""" 
+
+# Copyright (C) 2004, 2005 Aaron Bentley
+# <[e-mail]>
+#
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software
+#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+import cmd
+from itertools import chain
+import os
+import shlex
+import stat
+import string
+import sys
+
+from bzrlib import osutils
+from bzrlib.branch import Branch
+from bzrlib.config import config_dir, ensure_config_dir_exists
+from bzrlib.commands import get_cmd_object, get_all_cmds, get_alias
+from bzrlib.errors import BzrError
+from bzrlib.workingtree import WorkingTree
+import bzrlib.plugin
+
+
+SHELL_BLACKLIST = set(['rm', 'ls'])
+COMPLETION_BLACKLIST = set(['shell'])
+
+
+class BlackListedCommand(BzrError):
+    def __init__(self, command):
+        BzrError.__init__(self, "The command %s is blacklisted for shell use" %
+                          command)
+
+
+class CompletionContext(object):
+    def __init__(self, text, command=None, prev_opt=None, arg_pos=None):
+        self.text = text
+        self.command = command
+        self.prev_opt = prev_opt
+        self.arg_pos = None
+
+    def get_completions(self):
+        try:
+            return self.get_completions_or_raise()
+        except Exception, e:
+            print e, type(e)
+            return []
+
+    def get_option_completions(self):
+        try:
+            command_obj = get_cmd_object(self.command)
+        except BzrError:
+            return []
+        opts = [o+" " for o in iter_opt_completions(command_obj)]
+        return list(filter_completions(opts, self.text))
+
+    def get_completions_or_raise(self):
+        if self.command is None:
+            if '/' in self.text:
+                iter = iter_executables(self.text)
+            else:
+                iter = (c+" " for c in iter_command_names() if
+                        c not in COMPLETION_BLACKLIST)
+            return list(filter_completions(iter, self.text))
+        if self.prev_opt is None:
+            completions = self.get_option_completions()
+            if self.command == "cd":
+                iter = iter_dir_completions(self.text)
+                completions.extend(list(filter_completions(iter, self.text)))
+            else:
+                iter = iter_file_completions(self.text)
+                completions.extend(filter_completions(iter, self.text))
+            return completions
+
+
+class PromptCmd(cmd.Cmd):
+
+    def __init__(self):
+        cmd.Cmd.__init__(self)
+        self.prompt = "bzr> "
+        try:
+            self.tree = WorkingTree.open_containing('.')[0]
+        except:
+            self.tree = None
+        self.set_title()
+        self.set_prompt()
+        self.identchars += '-'
+        ensure_config_dir_exists()
+        self.history_file = osutils.pathjoin(config_dir(), 'shell-history')
+        readline.set_completer_delims(string.whitespace)
+        if os.access(self.history_file, os.R_OK) and \
+            os.path.isfile(self.history_file):
+            readline.read_history_file(self.history_file)
+        self.cwd = os.getcwd()
+
+    def write_history(self):
+        readline.write_history_file(self.history_file)
+
+    def do_quit(self, args):
+        self.write_history()
+        raise StopIteration
+
+    def do_exit(self, args):
+        self.do_quit(args)
+
+    def do_EOF(self, args):
+        print
+        self.do_quit(args)
+
+    def postcmd(self, line, bar):
+        self.set_title()
+        self.set_prompt()
+
+    def set_prompt(self):
+        if self.tree is not None:
+            try:
+                prompt_data = (self.tree.branch.nick, self.tree.branch.revno(),
+                               self.tree.relpath('.'))
+                prompt = " %s:%d/%s" % prompt_data
+            except:
+                prompt = ""
+        else:
+            prompt = ""
+        self.prompt = "bzr%s> " % prompt
+
+    def set_title(self, command=None):
+        try:
+            b = Branch.open_containing('.')[0]
+            version = "%s:%d" % (b.nick, b.revno())
+        except:
+            version = "[no version]"
+        if command is None:
+            command = ""
+        sys.stdout.write(terminal.term_title("bzr %s %s" % (command, version)))
+
+    def do_cd(self, line):
+        if line == "":
+            line = "~"
+        line = os.path.expanduser(line)
+        if os.path.isabs(line):
+            newcwd = line
+        else:
+            newcwd = self.cwd+'/'+line
+        newcwd = os.path.normpath(newcwd)
+        try:
+            os.chdir(newcwd)
+            self.cwd = newcwd
+        except Exception, e:
+            print e
+        try:
+            self.tree = WorkingTree.open_containing(".")[0]
+        except:
+            self.tree = None
+
+    def do_help(self, line):
+        self.default("help "+line)
+
+    def default(self, line):
+        args = shlex.split(line)
+        alias_args = get_alias(args[0])
+        if alias_args is not None:
+            args[0] = alias_args.pop(0)
+
+        commandname = args.pop(0)
+        for char in ('|', '<', '>'):
+            commandname = commandname.split(char)[0]
+        if commandname[-1] in ('|', '<', '>'):
+            commandname = commandname[:-1]
+        try:
+            if commandname in SHELL_BLACKLIST:
+                raise BlackListedCommand(commandname)
+            cmd_obj = get_cmd_object(commandname)
+        except (BlackListedCommand, BzrError):
+            return os.system(line)
+
+        try:
+            if too_complicated(line):
+                return os.system("bzr "+line)
+            else:
+                return (cmd_obj.run_argv_aliases(args, alias_args) or 0)
+        except BzrError, e:
+            print e
+        except KeyboardInterrupt, e:
+            print "Interrupted"
+        except Exception, e:
+#            print "Unhandled error:\n%s" % errors.exception_str(e)
+            print "Unhandled error:\n%s" % (e)
+
+
+    def completenames(self, text, line, begidx, endidx):
+        return CompletionContext(text).get_completions()
+
+    def completedefault(self, text, line, begidx, endidx):
+        """Perform completion for native commands.
+
+        :param text: The text to complete
+        :type text: str
+        :param line: The entire line to complete
+        :type line: str
+        :param begidx: The start of the text in the line
+        :type begidx: int
+        :param endidx: The end of the text in the line
+        :type endidx: int
+        """
+        (cmd, args, foo) = self.parseline(line)
+        if cmd == "bzr":
+            cmd = None
+        return CompletionContext(text, command=cmd).get_completions()
+
+
+def run_shell():
+    try:
+        prompt = PromptCmd()
+        try:
+            prompt.cmdloop()
+        finally:
+            prompt.write_history()
+    except StopIteration:
+        pass
+
+
+def iter_opt_completions(command_obj):
+    for option_name, option in command_obj.options().items():
+        yield "--" + option_name
+        short_name = option.short_name()
+        if short_name:
+            yield "-" + short_name
+
+
+def iter_file_completions(arg, only_dirs = False):
+    """Generate an iterator that iterates through filename completions.
+
+    :param arg: The filename fragment to match
+    :type arg: str
+    :param only_dirs: If true, match only directories
+    :type only_dirs: bool
+    """
+    cwd = os.getcwd()
+    if cwd != "/":
+        extras = [".", ".."]
+    else:
+        extras = []
+    (dir, file) = os.path.split(arg)
+    if dir != "":
+        listingdir = os.path.expanduser(dir)
+    else:
+        listingdir = cwd
+    for file in chain(os.listdir(listingdir), extras):
+        if dir != "":
+            userfile = dir+'/'+file
+        else:
+            userfile = file
+        if userfile.startswith(arg):
+            if os.path.isdir(listingdir+'/'+file):
+                userfile+='/'
+                yield userfile
+            elif not only_dirs:
+                yield userfile + ' '
+
+
+def iter_dir_completions(arg):
+    """Generate an iterator that iterates through directory name completions.
+
+    :param arg: The directory name fragment to match
+    :type arg: str
+    """
+    return iter_file_completions(arg, True)
+
+
+def iter_command_names(hidden=False):
+    for real_cmd_name, cmd_class in get_all_cmds():
+        if not hidden and cmd_class.hidden:
+            continue
+        for name in [real_cmd_name] + cmd_class.aliases:
+            # Don't complete on aliases that are prefixes of the canonical name
+            if name == real_cmd_name or not real_cmd_name.startswith(name):
+                yield name
+
+
+def iter_executables(path):
+    dirname, partial = os.path.split(path)
+    for filename in os.listdir(dirname):
+        if not filename.startswith(partial):
+            continue
+        fullpath = os.path.join(dirname, filename)
+        mode=os.lstat(fullpath)[stat.ST_MODE]
+        if stat.S_ISREG(mode) and 0111 & mode:
+            yield fullpath + ' '
+
+
+def filter_completions(iter, arg):
+    return (c for c in iter if c.startswith(arg))
+
+
+def iter_munged_completions(iter, arg, text):
+    for completion in iter:
+        completion = str(completion)
+        if completion.startswith(arg):
+            yield completion[len(arg)-len(text):]+" "
+
+
+def too_complicated(line):
+    for char in '|<>*?':
+        if char in line:
+            return True
+    return False
+
+    
+### IPython mods start
+
+def init_ipython(ip):
+    def bzr_completer(self,ev):
+        #print "bzr complete"
+        tup =  ev.line.split(None,2)
+        if len(tup) > 2:
+            cmd = tup[1]
+        else:
+            cmd = None
+                  
+        return CompletionContext(ev.symbol, command = cmd).get_completions()
+    bzrlib.plugin.load_plugins()
+    ip.set_hook('complete_command', bzr_completer, str_key = 'bzr')    

IPython/Extensions/ipy_greedycompleter.py

@@ -0,0 +1,75 b''
+""" Greedy completer extension for IPython
+
+Normal tab completer refuses to evaluate nonsafe stuff. This will evaluate
+everything, so you need to consider the consequences of pressing tab 
+yourself!
+
+Note that this extension simplifies readline interaction by setting
+only whitespace as completer delimiter. If this works well, we will
+do the same in default completer.
+
+"""
+from IPython import generics,ipapi
+from IPython.genutils import dir2
+
+def attr_matches(self, text):
+    """Compute matches when text contains a dot.
+
+    MONKEYPATCHED VERSION (ipy_greedycompleter.py)
+    
+    Assuming the text is of the form NAME.NAME....[NAME], and is
+    evaluatable in self.namespace or self.global_namespace, it will be
+    evaluated and its attributes (as revealed by dir()) are used as
+    possible completions.  (For class instances, class members are are
+    also considered.)
+
+    WARNING: this can still invoke arbitrary C code, if an object
+    with a __getattr__ hook is evaluated.
+
+    """
+    import re
+
+    force_complete = 1
+    # Another option, seems to work great. Catches things like ''.<tab>
+    m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text)
+
+    if m:
+        expr, attr = m.group(1, 3)
+    else:
+        # force match - eval anything that ends with colon
+        if not force_complete:
+            return []
+                
+        m2 = re.match(r"(.+)\.(\w*)$", self.lbuf)
+        if not m2:
+            return []
+        expr, attr = m2.group(1,2)
+        
+            
+    try:        
+        obj = eval(expr, self.namespace)
+    except:
+        try:
+            obj = eval(expr, self.global_namespace)
+        except:
+            return []
+
+    words = dir2(obj)
+    
+    try:
+        words = generics.complete_object(obj, words)
+    except ipapi.TryNext:
+        pass
+    # Build match list to return
+    n = len(attr)
+    res = ["%s.%s" % (expr, w) for w in words if w[:n] == attr ]
+    return res
+
+def main():
+    import readline
+    readline.set_completer_delims(" \n\t")
+    # monkeypatch - the code will be folded to normal completer later on
+    import IPython.completer
+    IPython.completer.Completer.attr_matches = attr_matches
+
+main()
  No newline at end of file

IPython/Extensions/ipy_jot.py

@@ -0,0 +1,311 b''
+# -*- coding: utf-8 -*-
+"""
+%jot magic for lightweight persistence.
+
+Stores variables in Struct with some notes in PicleShare database
+
+
+"""
+
+from datetime import datetime 
+import IPython.ipapi
+ip = IPython.ipapi.get()
+
+import pickleshare
+
+import inspect,pickle,os,sys,textwrap
+from IPython.FakeModule import FakeModule
+from IPython.ipstruct import Struct 
+
+
+def refresh_variables(ip, key=None):
+    db = ip.db
+    if key is None:
+        keys = db.keys('jot/*')
+    else:
+        keys = db.keys('jot/'+key)
+    for key in keys:
+        # strip autorestore
+        justkey = os.path.basename(key)
+        print "Restoring from", justkey, "..."
+        try:
+            obj = db[key]
+        except KeyError:
+            print "Unable to restore variable '%s', ignoring (use %%jot -d to forget!)" % justkey
+            print "The error was:",sys.exc_info()[0]
+        else:
+            #print "restored",justkey,"=",obj #dbg
+            try:
+                origname = obj.name
+            except:
+                ip.user_ns[justkey] = obj
+                print "Restored", justkey
+            else:
+                ip.user_ns[origname] = obj['val']
+                print "Restored", origname 
+
+def read_variables(ip, key=None):
+    db = ip.db
+    if key is None:
+        return None
+    else:
+        keys = db.keys('jot/'+key)
+    for key in keys:
+        # strip autorestore
+        justkey = os.path.basename(key)
+        print "restoring from ", justkey 
+        try:
+            obj = db[key]
+        except KeyError:
+            print "Unable to read variable '%s', ignoring (use %%jot -d to forget!)" % justkey
+            print "The error was:",sys.exc_info()[0]
+        else:
+            return obj
+
+
+def detail_variables(ip, key=None):
+    db, get = ip.db, ip.db.get
+ 
+    if key is None:
+        keys = db.keys('jot/*')
+    else:
+        keys = db.keys('jot/'+key)
+    if keys:
+        size = max(map(len,keys))
+    else:
+        size = 0
+
+    fmthead = '%-'+str(size)+'s [%s]'
+    fmtbody = 'Comment:\n %s'
+    fmtdata = 'Data:\n %s, %s'
+    for key in keys:
+        v = get(key,'<unavailable>')
+        justkey = os.path.basename(key)
+        try: 
+            print fmthead % (justkey, datetime.ctime(v.get('time','<unavailable>')))
+            print fmtbody % (v.get('comment','<unavailable>'))
+            d = v.get('val','unavailable')
+            print fmtdata % (repr(type(d)), '')
+            print repr(d)[0:200]
+            print
+            print
+        except AttributeError:
+            print fmt % (justkey, '<unavailable>', '<unavailable>', repr(v)[:50])
+
+
+def intm(n):
+    try:
+        return int(n)
+    except:
+        return 0
+
+def jot_obj(self, obj, name, comment=''):
+    """
+    write obj data to the note database, with whatever that should be noted.
+    """
+    had = self.db.keys('jot/'+name+'*')
+    # if it the same name but a later version, we stupidly add a number to the 
+    # so the name doesn't collide. Any better idea?
+    suffix = ''
+    if len(had)>0:
+        pre = os.path.commonprefix(had)
+        suf = [n.split(pre)[1] for n in had]
+        versions = map(intm, suf)
+        suffix = str(max(versions)+1)
+
+    uname = 'jot/'+name+suffix
+
+    # which one works better? 
+    #all = ip.IP.shadowhist.all()
+    all = ip.IP.shell.input_hist
+
+    # We may actually want to make snapshot of files that are run-ned.
+
+    # get the comment 
+    try:
+        comment = ip.IP.magic_edit('-x').strip()
+    except:
+        print "No comment is recorded."
+        comment = ''
+
+    self.db[uname] = Struct({'val':obj, 
+                'time'  :   datetime.now(), 
+                'hist'  :   all,
+                'name'  :   name,
+                'comment' : comment,})
+
+    print "Jotted down notes for '%s' (%s)" % (uname, obj.__class__.__name__)
+
+    
+
+def magic_jot(self, parameter_s=''):
+    """Lightweight persistence for python variables.
+
+    Example:
+    
+    ville@badger[~]|1> A = ['hello',10,'world']\\
+    ville@badger[~]|2> %jot A\\
+    ville@badger[~]|3> Exit
+    
+    (IPython session is closed and started again...)
+    
+    ville@badger:~$ ipython -p pysh\\
+    ville@badger[~]|1> print A
+    
+    ['hello', 10, 'world']
+    
+    Usage:
+    
+    %jot          - Show list of all variables and their current values\\
+    %jot -l       - Show list of all variables and their current values in detail\\
+    %jot -l <var> - Show one variable and its current values in detail\\
+    %jot <var>    - Store the *current* value of the variable to disk\\
+    %jot -d <var> - Remove the variable and its value from storage\\
+    %jot -z       - Remove all variables from storage (disabled)\\
+    %jot -r <var> - Refresh/Load variable from jot (delete current vals)\\
+    %jot foo >a.txt  - Store value of foo to new file a.txt\\
+    %jot foo >>a.txt - Append value of foo to file a.txt\\   
+    
+    It should be noted that if you change the value of a variable, you
+    need to %note it again if you want to persist the new value.
+    
+    Note also that the variables will need to be pickleable; most basic
+    python types can be safely %stored.
+    
+    """
+    
+    opts,argsl = self.parse_options(parameter_s,'drzl',mode='string')
+    args = argsl.split(None,1)
+    ip = self.getapi()
+    db = ip.db
+    # delete
+    if opts.has_key('d'):
+        try:
+            todel = args[0]
+        except IndexError:
+            error('You must provide the variable to forget')
+        else:
+            try:
+                del db['jot/' + todel]
+            except:
+                error("Can't delete variable '%s'" % todel)
+    # reset the whole database
+    elif opts.has_key('z'):
+        print "reseting the whole database has been disabled."
+        #for k in db.keys('autorestore/*'):
+        #    del db[k]
+
+    elif opts.has_key('r'):
+        try:
+            toret = args[0]
+        except:
+            print "restoring all the variables jotted down..."
+            refresh_variables(ip)
+        else:
+            refresh_variables(ip, toret)
+
+    elif opts.has_key('l'):
+        try:
+            tolist = args[0]
+        except:
+            print "List details for all the items."
+            detail_variables(ip)
+        else:
+            print "Details for", tolist, ":"
+            detail_variables(ip, tolist)
+    
+    # run without arguments -> list noted variables & notes 
+    elif not args:
+        vars = self.db.keys('jot/*')
+        vars.sort()            
+        if vars:
+            size = max(map(len,vars)) - 4 
+        else:
+            size = 0
+            
+        print 'Variables and their in-db values:'
+        fmt = '%-'+str(size)+'s [%s] -> %s'
+        get = db.get
+        for var in vars:
+            justkey = os.path.basename(var)
+            v = get(var,'<unavailable>')
+            try: 
+                print fmt % (justkey,\
+                    datetime.ctime(v.get('time','<unavailable>')),\
+                    v.get('comment','<unavailable>')[:70].replace('\n',' '),)
+            except AttributeError:
+                print fmt % (justkey, '<unavailable>', '<unavailable>', repr(v)[:50])
+
+    
+    # default action - store the variable
+    else:
+        # %store foo >file.txt or >>file.txt
+        if len(args) > 1 and args[1].startswith('>'):
+            fnam = os.path.expanduser(args[1].lstrip('>').lstrip())
+            if args[1].startswith('>>'):
+                fil = open(fnam,'a')
+            else:
+                fil = open(fnam,'w')
+            obj = ip.ev(args[0])
+            print "Writing '%s' (%s) to file '%s'." % (args[0],
+              obj.__class__.__name__, fnam)
+
+            
+            if not isinstance (obj,basestring):
+                from pprint import pprint
+                pprint(obj,fil)
+            else:
+                fil.write(obj)
+                if not obj.endswith('\n'):
+                    fil.write('\n')
+            
+            fil.close()
+            return
+        
+        # %note foo
+        try:
+            obj = ip.user_ns[args[0]]
+        except KeyError:
+            # this should not be alias, for aliases, use %store
+            print
+            print "Error: %s doesn't exist." % args[0]
+            print 
+            print "Use %note -r <var> to retrieve variables. This should not be used " +\
+                  "to store alias, for saving aliases, use %store" 
+            return
+        else:
+            if isinstance(inspect.getmodule(obj), FakeModule):
+                print textwrap.dedent("""\
+                Warning:%s is %s 
+                Proper storage of interactively declared classes (or instances
+                of those classes) is not possible! Only instances
+                of classes in real modules on file system can be %%store'd.
+                """ % (args[0], obj) ) 
+                return
+            #pickled = pickle.dumps(obj)
+            #self.db[ 'jot/' + args[0] ] = obj
+            jot_obj(self, obj, args[0])
+
+
+def magic_read(self, parameter_s=''):
+    """    
+    %read <var> - Load variable from data that is jotted down.\\
+    
+    """
+    
+    opts,argsl = self.parse_options(parameter_s,'drzl',mode='string')
+    args = argsl.split(None,1)
+    ip = self.getapi()
+    db = ip.db
+    #if opts.has_key('r'):
+    try:
+        toret = args[0]
+    except:
+        print "which record do you want to read out?"
+        return
+    else:
+        return read_variables(ip, toret)
+
+
+ip.expose_magic('jot',magic_jot)
+ip.expose_magic('read',magic_read)

IPython/Extensions/ipy_lookfor.py

@@ -0,0 +1,234 b''
+"""
+IPython extension: %lookfor command for searching docstrings
+
+"""
+# Pauli Virtanen <[e-mail]>, 2008.
+
+import re, inspect, pkgutil, pydoc
+
+#------------------------------------------------------------------------------
+# Lookfor functionality
+#------------------------------------------------------------------------------
+
+# Cache for lookfor: {id(module): {name: (docstring, kind, index), ...}...}
+# where kind: "func", "class", "module", "object"
+# and index: index in breadth-first namespace traversal
+_lookfor_caches = {}
+
+# regexp whose match indicates that the string may contain a function signature
+_function_signature_re = re.compile(r"[a-z_]+\(.*[,=].*\)", re.I)
+
+def lookfor(what, modules=None, import_modules=True, regenerate=False):
+    """
+    Search for objects whose documentation contains all given words.
+    Shows a summary of matching objects, sorted roughly by relevance.
+
+    Parameters
+    ----------
+    what : str
+        String containing words to look for.
+
+    module : str, module
+        Module whose docstrings to go through.
+    import_modules : bool
+        Whether to import sub-modules in packages.
+        Will import only modules in __all__
+    regenerate: bool
+        Re-generate the docstring cache
+
+    """
+    # Cache
+    cache = {}
+    for module in modules:
+        try:
+            c = _lookfor_generate_cache(module, import_modules, regenerate)
+            cache.update(c)
+        except ImportError:
+            pass
+    
+    # Search
+    # XXX: maybe using a real stemming search engine would be better?
+    found = []
+    whats = str(what).lower().split()
+    if not whats: return
+
+    for name, (docstring, kind, index) in cache.iteritems():
+        if kind in ('module', 'object'): 
+            # don't show modules or objects
+            continue
+        ok = True
+        doc = docstring.lower()
+        for w in whats:
+            if w not in doc:
+                ok = False
+                break
+        if ok:
+            found.append(name)
+
+    # Relevance sort
+    # XXX: this is full Harrison-Stetson heuristics now,
+    # XXX: it probably could be improved
+
+    kind_relevance = {'func': 1000, 'class': 1000, 
+                      'module': -1000, 'object': -1000}
+
+    def relevance(name, docstr, kind, index):
+        r = 0
+        # do the keywords occur within the start of the docstring?
+        first_doc = "\n".join(docstr.lower().strip().split("\n")[:3])
+        r += sum([200 for w in whats if w in first_doc])
+        # do the keywords occur in the function name?
+        r += sum([30 for w in whats if w in name])
+        # is the full name long?
+        r += -len(name) * 5
+        # is the object of bad type?
+        r += kind_relevance.get(kind, -1000)
+        # is the object deep in namespace hierarchy?
+        r += -name.count('.') * 10
+        r += max(-index / 100, -100)
+        return r
+
+    def relevance_sort(a, b):
+        dr = relevance(b, *cache[b]) - relevance(a, *cache[a])
+        if dr != 0: return dr
+        else: return cmp(a, b)
+    found.sort(relevance_sort)
+
+    # Pretty-print
+    s = "Search results for '%s'" % (' '.join(whats))
+    help_text = [s, "-"*len(s)]
+    for name in found:
+        doc, kind, ix = cache[name]
+
+        doclines = [line.strip() for line in doc.strip().split("\n")
+                    if line.strip()]
+
+        # find a suitable short description
+        try:
+            first_doc = doclines[0].strip()
+            if _function_signature_re.search(first_doc):
+                first_doc = doclines[1].strip()
+        except IndexError:
+            first_doc = ""
+        help_text.append("%s\n    %s" % (name, first_doc))
+
+    # Output
+    if len(help_text) > 10:
+        pager = pydoc.getpager()
+        pager("\n".join(help_text))
+    else:
+        print "\n".join(help_text)
+
+def _lookfor_generate_cache(module, import_modules, regenerate):
+    """
+    Generate docstring cache for given module.
+
+    Parameters
+    ----------
+    module : str, None, module
+        Module for which to generate docstring cache
+    import_modules : bool
+        Whether to import sub-modules in packages.
+        Will import only modules in __all__
+    regenerate: bool
+        Re-generate the docstring cache
+
+    Returns
+    -------
+    cache : dict {obj_full_name: (docstring, kind, index), ...}
+        Docstring cache for the module, either cached one (regenerate=False)
+        or newly generated.
+        
+    """
+    global _lookfor_caches
+
+    if module is None:
+        module = "numpy"
+
+    if isinstance(module, str):
+        module = __import__(module)
+
+    if id(module) in _lookfor_caches and not regenerate:
+        return _lookfor_caches[id(module)]
+
+    # walk items and collect docstrings
+    cache = {}
+    _lookfor_caches[id(module)] = cache
+    seen = {}
+    index = 0
+    stack = [(module.__name__, module)]
+    while stack:
+        name, item = stack.pop(0)
+        if id(item) in seen: continue
+        seen[id(item)] = True
+
+        index += 1
+        kind = "object"
+       
+        if inspect.ismodule(item):
+            kind = "module"
+            try:
+                _all = item.__all__
+            except AttributeError:
+                _all = None
+            # import sub-packages
+            if import_modules and hasattr(item, '__path__'):
+                for m in pkgutil.iter_modules(item.__path__):
+                    if _all is not None and m[1] not in _all:
+                        continue
+                    try:
+                        __import__("%s.%s" % (name, m[1]))
+                    except ImportError:
+                        continue
+            for n, v in inspect.getmembers(item):
+                if _all is not None and n not in _all:
+                    continue
+                stack.append(("%s.%s" % (name, n), v))
+        elif inspect.isclass(item):
+            kind = "class"
+            for n, v in inspect.getmembers(item):
+                stack.append(("%s.%s" % (name, n), v))
+        elif callable(item):
+            kind = "func"
+        
+        doc = inspect.getdoc(item)
+        if doc is not None:
+            cache[name] = (doc, kind, index)
+
+    return cache
+
+#------------------------------------------------------------------------------
+# IPython connectivity
+#------------------------------------------------------------------------------
+
+import IPython.ipapi
+ip = IPython.ipapi.get()
+
+_lookfor_modules = ['numpy', 'scipy']
+
+def lookfor_f(self, arg=''):
+    r"""
+    Search for objects whose documentation contains all given words.
+    Shows a summary of matching objects, sorted roughly by relevance.
+
+    Usage
+    -----
+    %lookfor +numpy  some words
+    Search module 'numpy'
+
+    %lookfor_modules numpy scipy
+    Set default modules whose docstrings to search
+
+    """
+    lookfor(arg, modules=_lookfor_modules)
+
+def lookfor_modules_f(self, arg=''):
+    global _lookfor_modules
+    if not arg:
+        print "Modules included in %lookfor search:", _lookfor_modules
+    else:
+        _lookfor_modules = arg.split()
+
+ip.expose_magic('lookfor', lookfor_f)
+ip.expose_magic('lookfor_modules', lookfor_modules_f)
+

IPython/Extensions/ipy_vimserver.py

@@ -0,0 +1,239 b''
+""" Integration with gvim, by Erich Heine
+
+Provides a %vim magic command, and reuses the same vim session. Uses 
+unix domain sockets for communication between vim and IPython. ipy.vim is 
+available in doc/examples of the IPython distribution.
+
+Slightly touched up email announcement (and description how to use it) by 
+Erich Heine is here:
+
+Ive recently been playing with ipython, and like it quite a bit. I did
+however discover a bit of frustration, namely with editor interaction.
+I am a gvim user, and using the command edit on a new file causes
+ipython to try and run that file as soon as the text editor opens
+up. The -x command of course fixes this, but its still a bit annoying,
+switching windows to do a run file, then back to the text
+editor. Being a heavy tab user in gvim, another annoyance is not being
+able to specify weather a new tab is how I choose to open the file.
+
+Not being one to shirk my open source duties (and seeing this as a
+good excuse to poke around ipython internals), Ive created a script
+for having gvim and ipython work very nicely together. Ive attached
+both to this email (hoping of course that the mailing list allows such
+things).
+
+There are 2 files: 
+
+ipy_vimserver.py -- this file contains the ipython stuff
+ipy.vim -- this file contains the gvim stuff
+
+In combination they allow for a few functionalities:
+
+#1. the vim magic command. This is a fancy wrapper around the edit
+magic, that allows for a new option, -t, which opens the text in a new
+gvim tab.  Otherwise it works the same as edit -x. (it internally
+calls edit -x). This magic command also juggles vim server management,
+so when it is called when there is not a gvim running, it creates a
+new gvim instance, named after the ipython session name.  Once such a
+gvim instance is running, it will be used for subsequent uses of the
+vim command.
+
+#2. ipython - gvim interaction. Once a file has been opened with the
+vim magic (and a session set up, see below), pressing the F5 key in
+vim will cause the calling ipython instance to execute run
+filename.py. (if you typo like I do, this is very useful)
+
+#3. ipython server - this is a thread wich listens on a unix domain
+socket, and runs commands sent to that socket.
+
+Note, this only works on POSIX systems, that allow for AF_UNIX type
+sockets. It has only been tested on linux (a fairly recent debian
+testing distro).
+
+To install it put, the ipserver.py in your favorite locaion for
+sourcing ipython scripts. I put the ipy.vim in
+~/.vim/after/ftplugin/python/.
+
+To use (this can be scripted im sure, but i usually have 2 or 3
+ipythons and corresponding gvims open):
+
+import ipy_vimserver
+ipy_vimserver.setup('sessionname')
+
+(Editors note - you can probably add these to your ipy_user_conf.py)
+
+Then use ipython as you normally would, until you need to edit
+something. Instead of edit, use the vim magic.  Thats it!
+
+"""
+
+import IPython.ipapi
+#import ipythonhooks
+import socket, select
+import os, threading, subprocess
+import re
+
+ERRCONDS = select.POLLHUP|select.POLLERR
+SERVER = None
+ip = IPython.ipapi.get()
+
+# this listens to a unix domain socket in a separate thread, so that comms
+# between a vim instance and ipython can happen in a fun and productive way
+class IpyServer(threading.Thread):
+    def __init__(self, sname):
+        super(IpyServer, self).__init__()
+        self.keep_running = True
+        self.__sname = sname
+        self.socket = socket.socket(socket.AF_UNIX)
+        self.poller = select.poll()
+        self.current_conns = dict()
+        self.setDaemon(True)
+
+    def listen(self):
+        self.socket.bind(self.__sname)
+        self.socket.listen(1)
+
+    def __handle_error(self, socket):
+        if socket == self.socket.fileno():
+            self.keep_running = False
+            for a in self.current_conns.values():
+                a.close()
+            return False
+        else:
+            y = self.current_conns[socket]
+            del self.current_conns[socket]
+        y.close()
+        self.poller.unregister(socket)
+
+    def serve_me(self):
+        self.listen()
+        self.poller.register(self.socket,select.POLLIN|ERRCONDS)
+
+        while self.keep_running:
+            try:
+                avail = self.poller.poll(1)
+            except:
+                continue
+
+            if not avail: continue
+
+            for sock, conds in avail:
+                if conds & (ERRCONDS):
+                    if self.__handle_error(sock): continue
+                    else: break
+
+                if sock == self.socket.fileno():
+                    y = self.socket.accept()[0]
+                    self.poller.register(y, select.POLLIN|ERRCONDS)
+                    self.current_conns[y.fileno()] = y
+                else: y = self.current_conns.get(sock)
+
+                self.handle_request(y)
+
+        os.remove(self.__sname)
+
+    run = serve_me
+
+    def stop(self):
+        self.keep_running = False
+
+    def handle_request(self,sock):
+        sock.settimeout(1)
+        while self.keep_running:
+            try:
+                x = sock.recv(4096)
+            except socket.timeout:
+                pass
+            else:
+                break
+        self.do_it(x)
+
+    def do_it(self, data):
+        data = data.split('\n')
+        cmds = list()
+        for line in data:
+            cmds.append(line)
+        ip.runlines(cmds)
+
+
+# try to help ensure that the unix domain socket is cleaned up proper
+def shutdown_server(self):
+    if SERVER:
+        SERVER.stop()
+        SERVER.join(3)
+    raise IPython.ipapi.TryNext
+
+ip.set_hook('shutdown_hook', shutdown_server, 10)
+
+# this fun function exists to make setup easier for all, and makes the
+# vimhook function ready for instance specific communication
+def setup(sessionname='',socketdir=os.path.expanduser('~/.ipython/')):
+    global SERVER
+
+    if sessionname:
+        session = sessionname
+    elif os.environ.get('IPY_SESSION'):
+        session = os.environ.get('IPY_SESSION')
+    else:
+        session = 'IPYS'
+    vimhook.vimserver=session
+    vimhook.ipyserver = os.path.join(socketdir, session)
+    if not SERVER:
+        SERVER = IpyServer(vimhook.ipyserver)
+        SERVER.start()
+
+
+
+# calls gvim, with all ops happening on the correct gvim instance for this
+# ipython instance. it then calls edit -x (since gvim will return right away)
+# things of note: it sets up a special environment, so that the ipy.vim script
+# can connect back to the ipython instance and do fun things, like run the file
+def vimhook(self, fname, line):
+    env = os.environ.copy()
+    vserver = vimhook.vimserver.upper()
+    check = subprocess.Popen('gvim --serverlist', stdout = subprocess.PIPE,
+        shell=True)
+    check.wait()
+    cval = [l for l in check.stdout.readlines() if vserver in l]
+
+    if cval:
+        vimargs = '--remote%s' % (vimhook.extras,)
+    else:
+        vimargs = ''
+    vimhook.extras = ''
+
+    env['IPY_SESSION'] = vimhook.vimserver
+    env['IPY_SERVER'] = vimhook.ipyserver
+
+    if line is None: line = ''
+    else: line = '+' + line
+    vim_cmd = 'gvim --servername %s %s %s %s' % (vimhook.vimserver, vimargs,
+        line, fname)
+    subprocess.call(vim_cmd, env=env, shell=True)
+
+
+#default values to keep it sane...
+vimhook.vimserver = ''
+vimhook.ipyserver = ''
+
+ip.set_hook('editor',vimhook)
+
+# this is set up so more vim specific commands can be added, instead of just
+# the current -t. all thats required is a compiled regex, a call to do_arg(pat)
+# and the logic to deal with the new feature
+newtab = re.compile(r'-t(?:\s|$)')
+def vim(self, argstr):
+    def do_arg(pat, rarg):
+        x = len(pat.findall(argstr))
+        if x:
+            a = pat.sub('',argstr)
+            return rarg, a
+        else: return '', argstr
+
+    t, argstr = do_arg(newtab, '-tab')
+    vimhook.extras = t
+    argstr = 'edit -x ' + argstr
+    ip.magic(argstr)
+
+ip.expose_magic('vim', vim)
+

IPython/Extensions/ipy_winpdb.py

@@ -0,0 +1,73 b''
+""" Debug a script (like %run -d) in IPython process, Using WinPdb
+
+Usage:
+
+%wdb test.py
+    run test.py, with a winpdb breakpoint at start of the file 
+    
+%wdb pass
+    Change the password (e.g. if you have forgotten the old one)
+"""
+
+import os
+
+import IPython.ipapi
+import rpdb2
+
+ip = IPython.ipapi.get()
+
+rpdb_started = False
+
+def wdb_f(self, arg):
+    """ Debug a script (like %run -d) in IPython process, Using WinPdb
+    
+    Usage:
+    
+    %wdb test.py
+        run test.py, with a winpdb breakpoint at start of the file 
+        
+    %wdb pass
+        Change the password (e.g. if you have forgotten the old one)
+        
+    Note that after the script has been run, you need to do "Go" (f5) 
+    in WinPdb to resume normal IPython operation.
+    """
+
+    global rpdb_started
+    if not arg.strip():
+        print __doc__
+        return
+        
+    if arg.strip() == 'pass':
+        passwd = raw_input('Enter new winpdb session password: ')
+        ip.db['winpdb_pass'] = passwd
+        print "Winpdb password changed"
+        if rpdb_started:
+            print "You need to restart IPython to use the new password"
+        return 
+    
+    path = os.path.abspath(arg)
+    if not os.path.isfile(path):
+        raise IPython.ipapi.UsageError("%%wdb: file %s does not exist" % path)
+    if not rpdb_started:
+        passwd = ip.db.get('winpdb_pass', None)
+        if passwd is None:
+            import textwrap
+            print textwrap.dedent("""\
+            Winpdb sessions need a password that you use for attaching the external
+            winpdb session. IPython will remember this. You can change the password later 
+            by '%wpdb pass'
+            """)
+            passwd = raw_input('Enter new winpdb session password: ')
+            ip.db['winpdb_pass'] = passwd
+            
+        print "Starting rpdb2 in IPython process"
+        rpdb2.start_embedded_debugger(passwd, timeout = 0)
+        rpdb_started = True
+        
+    rpdb2.set_temp_breakpoint(path)
+    print 'It is time to attach with WinPdb (launch WinPdb if needed, File -> Attach)'
+    ip.magic('%run ' + arg)
+    
+
+ip.expose_magic('wdb', wdb_f)

IPython/gui/wx/ipshell_nonblocking.py

@@ -0,0 +1,460 b''
+#!/usr/bin/python
+# -*- coding: iso-8859-15 -*-
+'''
+Provides IPython remote instance.
+
+@author: Laurent Dufrechou
+laurent.dufrechou _at_ gmail.com
+@license: BSD
+
+All rights reserved. This program and the accompanying materials are made
+available under the terms of the BSD which accompanies this distribution, and
+is available at U{http://www.opensource.org/licenses/bsd-license.php}
+'''
+
+__version__ = 0.9
+__author__  = "Laurent Dufrechou"
+__email__   = "laurent.dufrechou _at_ gmail.com"
+__license__ = "BSD"
+
+import re
+import sys
+import os
+import locale
+import time
+import pydoc,__builtin__,site
+from thread_ex import ThreadEx
+from StringIO import StringIO
+
+try:
+        import IPython
+except Exception,e:
+        raise "Error importing IPython (%s)" % str(e)
+
+##############################################################################
+class _Helper(object):
+    """Redefine the built-in 'help'.
+    This is a wrapper around pydoc.help (with a twist).
+    """
+
+    def __init__(self,pager):
+        self._pager = pager
+
+    def __repr__(self):
+        return "Type help() for interactive help, " \
+               "or help(object) for help about object."
+    
+    def __call__(self, *args, **kwds):
+        class DummyWriter(object):
+            def __init__(self,pager):
+                self._pager = pager
+                
+            def write(self,data):
+                self._pager(data)
+
+        import pydoc
+        pydoc.help.output = DummyWriter(self._pager)
+        pydoc.help.interact = lambda :1
+        
+        return pydoc.help(*args, **kwds)
+
+  
+##############################################################################
+class _CodeExecutor(ThreadEx):
+    
+    def __init__(self, instance, after):
+        ThreadEx.__init__(self)
+        self.instance = instance
+        self._afterExecute=after
+
+    def run(self):
+        try:
+            self.instance._doc_text = None
+            self.instance._help_text = None
+            self.instance._execute()
+            # used for uper class to generate event after execution
+            self._afterExecute() 
+            
+        except KeyboardInterrupt:
+            pass
+ 
+
+##############################################################################
+class NonBlockingIPShell(object):
+    '''
+    Create an IPython instance, running the commands in a separate,
+    non-blocking thread. 
+    This allows embedding in any GUI without blockage.
+
+    Note: The ThreadEx class supports asynchroneous function call
+          via raise_exc()
+    '''
+
+    def __init__(self,argv=[],user_ns={},user_global_ns=None,
+                 cin=None, cout=None, cerr=None,
+                 ask_exit_handler=None):
+        '''
+        @param argv: Command line options for IPython
+        @type argv: list
+        @param user_ns: User namespace.
+        @type user_ns: dictionary
+        @param user_global_ns: User global namespace.
+        @type user_global_ns: dictionary.
+        @param cin: Console standard input.
+        @type cin: IO stream
+        @param cout: Console standard output.
+        @type cout: IO stream
+        @param cerr: Console standard error.
+        @type cerr: IO stream
+        @param exit_handler: Replacement for builtin exit() function
+        @type exit_handler: function
+        @param time_loop: Define the sleep time between two thread's loop
+        @type int
+        '''
+        #ipython0 initialisation
+        self.initIpython0(argv, user_ns, user_global_ns,
+                          cin, cout, cerr,
+                          ask_exit_handler)
+        
+        #vars used by _execute
+        self._iter_more = 0
+        self._history_level = 0
+        self._complete_sep =  re.compile('[\s\{\}\[\]\(\)\=]')
+        self._prompt = str(self._IP.outputcache.prompt1).strip()
+
+        #thread working vars
+        self._line_to_execute = ''
+
+        #vars that will be checked by GUI loop to handle thread states...
+        #will be replaced later by PostEvent GUI funtions...
+        self._doc_text = None
+        self._help_text = None
+        self._add_button = None
+
+    def initIpython0(self, argv=[], user_ns={}, user_global_ns=None,
+                     cin=None, cout=None, cerr=None,
+                     ask_exit_handler=None):
+        #first we redefine in/out/error functions of IPython 
+        if cin:
+            IPython.Shell.Term.cin = cin
+        if cout:
+            IPython.Shell.Term.cout = cout
+        if cerr:
+            IPython.Shell.Term.cerr = cerr
+        
+        # This is to get rid of the blockage that accurs during
+        # IPython.Shell.InteractiveShell.user_setup()
+        IPython.iplib.raw_input = lambda x: None
+
+        self._term = IPython.genutils.IOTerm(cin=cin, cout=cout, cerr=cerr)
+
+        excepthook = sys.excepthook
+
+        self._IP = IPython.Shell.make_IPython(
+                                            argv,user_ns=user_ns,
+                                            user_global_ns=user_global_ns,
+                                            embedded=True,
+                                            shell_class=IPython.Shell.InteractiveShell)
+
+        #we replace IPython default encoding by wx locale encoding
+        loc = locale.getpreferredencoding()
+        if loc:
+                self._IP.stdin_encoding = loc
+        #we replace the ipython default pager by our pager
+        self._IP.set_hook('show_in_pager',self._pager)
+        
+        #we replace the ipython default shell command caller by our shell handler
+        self._IP.set_hook('shell_hook',self._shell)
+        
+        #we replace the ipython default input command caller by our method
+        IPython.iplib.raw_input_original = self._raw_input
+        #we replace the ipython default exit command by our method
+        self._IP.exit = ask_exit_handler
+        #we replace the help command
+        self._IP.user_ns['help'] = _Helper(self._pager_help)
+
+        #we disable cpase magic... until we found a way to use it properly.
+        #import IPython.ipapi
+        ip = IPython.ipapi.get()
+        def bypassMagic(self, arg):
+            print '%this magic is currently disabled.'
+        ip.expose_magic('cpaste', bypassMagic)
+        
+        sys.excepthook = excepthook
+
+    #----------------------- Thread management section ----------------------    
+    def doExecute(self,line):
+        """
+        Tell the thread to process the 'line' command
+        """
+
+        self._line_to_execute = line
+        #we launch the ipython line execution in a thread to make it interruptible
+        self.ce = _CodeExecutor(self,self._afterExecute)
+        self.ce.start()
+        
+    #----------------------- IPython management section ----------------------    
+    def getDocText(self):
+        """
+        Returns the output of the processing that need to be paged (if any)
+
+        @return: The std output string.
+        @rtype: string
+        """
+        return self._doc_text
+        
+    def getHelpText(self):
+        """
+        Returns the output of the processing that need to be paged via help pager(if any)
+
+        @return: The std output string.
+        @rtype: string
+        """
+        return self._help_text
+
+    def getBanner(self):
+        """
+        Returns the IPython banner for useful info on IPython instance
+
+        @return: The banner string.
+        @rtype: string
+        """
+        return self._IP.BANNER
+    
+    def getPromptCount(self):
+        """
+        Returns the prompt number.
+        Each time a user execute a line in the IPython shell the prompt count is increased
+
+        @return: The prompt number
+        @rtype: int
+        """
+        return self._IP.outputcache.prompt_count
+
+    def getPrompt(self):
+        """
+        Returns current prompt inside IPython instance
+        (Can be In [...]: ot ...:)
+
+        @return: The current prompt.
+        @rtype: string
+        """
+        return self._prompt
+
+    def getIndentation(self):
+        """
+        Returns the current indentation level
+        Usefull to put the caret at the good start position if we want to do autoindentation.
+
+        @return: The indentation level.
+        @rtype: int
+        """
+        return self._IP.indent_current_nsp
+        
+    def updateNamespace(self, ns_dict):
+        '''
+        Add the current dictionary to the shell namespace.
+
+        @param ns_dict: A dictionary of symbol-values.
+        @type ns_dict: dictionary
+        '''
+        self._IP.user_ns.update(ns_dict)
+
+    def complete(self, line):
+        '''
+        Returns an auto completed line and/or posibilities for completion.
+
+        @param line: Given line so far.
+        @type line: string
+
+        @return: Line completed as for as possible,
+        and possible further completions.
+        @rtype: tuple
+        '''
+        split_line = self._complete_sep.split(line)
+        possibilities = self._IP.complete(split_line[-1])
+        if possibilities:
+
+            def _commonPrefix(str1, str2):
+                '''
+                Reduction function. returns common prefix of two given strings.
+
+                @param str1: First string.
+                @type str1: string
+                @param str2: Second string
+                @type str2: string
+
+                @return: Common prefix to both strings.
+                @rtype: string
+                '''
+                for i in range(len(str1)):
+                    if not str2.startswith(str1[:i+1]):
+                        return str1[:i]
+                return str1
+            common_prefix = reduce(_commonPrefix, possibilities)
+            completed = line[:-len(split_line[-1])]+common_prefix
+        else:
+            completed = line
+        return completed, possibilities
+
+    def historyBack(self):
+        '''
+        Provides one history command back.
+
+        @return: The command string.
+        @rtype: string
+        '''
+        history = ''
+        #the below while loop is used to suppress empty history lines
+        while((history == '' or history == '\n') and self._history_level >0):
+                if self._history_level>=1:
+                        self._history_level -= 1
+                history = self._getHistory()            
+        return history
+
+    def historyForward(self):
+        '''
+        Provides one history command forward.
+
+        @return: The command string.
+        @rtype: string
+        '''
+        history = ''
+        #the below while loop is used to suppress empty history lines
+        while((history == '' or history == '\n') and self._history_level <= self._getHistoryMaxIndex()):
+                if self._history_level < self._getHistoryMaxIndex():
+                        self._history_level += 1
+                        history = self._getHistory()
+                else:
+                        if self._history_level == self._getHistoryMaxIndex():
+                                history = self._getHistory()
+                                self._history_level += 1
+                        else:
+                                history = ''
+        return history
+
+    def initHistoryIndex(self):
+        '''
+        set history to last command entered
+        '''
+        self._history_level = self._getHistoryMaxIndex()+1
+
+    #----------------------- IPython PRIVATE management section --------------
+    def _afterExecute(self):
+        '''
+        Can be redefined to generate post event after excution is done
+        '''
+        pass
+
+    #def _askExit(self):
+    #    '''
+    #    Can be redefined to generate post event to exit the Ipython shell
+    #    '''
+    #    pass
+
+    def _getHistoryMaxIndex(self):
+        '''
+        returns the max length of the history buffer
+
+        @return: history length
+        @rtype: int
+        '''
+        return len(self._IP.input_hist_raw)-1
+        
+    def _getHistory(self):
+        '''
+        Get's the command string of the current history level.
+
+        @return: Historic command stri
+        @rtype: string
+        '''
+        rv = self._IP.input_hist_raw[self._history_level].strip('\n')
+        return rv
+
+    def _pager_help(self,text):
+        '''
+        This function is used as a callback replacment to IPython help pager function
+
+        It puts the 'text' value inside the self._help_text string that can be retrived via getHelpText
+        function.
+        '''
+        if self._help_text == None:
+            self._help_text = text
+        else:
+            self._help_text += text
+    
+    def _pager(self,IP,text):
+        '''
+        This function is used as a callback replacment to IPython pager function
+
+        It puts the 'text' value inside the self._doc_text string that can be retrived via getDocText
+        function.
+        '''
+        self._doc_text = text
+    
+    def _raw_input(self, prompt=''):
+        '''
+        Custom raw_input() replacement. Get's current line from console buffer.
+
+        @param prompt: Prompt to print. Here for compatability as replacement.
+        @type prompt: string
+
+        @return: The current command line text.
+        @rtype: string
+        '''
+        return self._line_to_execute
+
+    def _execute(self):
+        '''
+        Executes the current line provided by the shell object.
+        '''
+        orig_stdout = sys.stdout
+        sys.stdout = IPython.Shell.Term.cout
+                
+        try:
+            line = self._IP.raw_input(None, self._iter_more)
+            if self._IP.autoindent:
+                self._IP.readline_startup_hook(None)
+
+        except KeyboardInterrupt:
+            self._IP.write('\nKeyboardInterrupt\n')
+            self._IP.resetbuffer()
+            # keep cache in sync with the prompt counter:
+            self._IP.outputcache.prompt_count -= 1
+
+            if self._IP.autoindent:
+                self._IP.indent_current_nsp = 0
+            self._iter_more = 0
+        except:
+            self._IP.showtraceback()
+        else:
+            self._iter_more = self._IP.push(line)
+            if (self._IP.SyntaxTB.last_syntax_error and
+                    self._IP.rc.autoedit_syntax):
+                self._IP.edit_syntax_error()
+        if self._iter_more:
+            self._prompt = str(self._IP.outputcache.prompt2).strip()
+            if self._IP.autoindent:
+                self._IP.readline_startup_hook(self._IP.pre_readline)
+        else:
+            self._prompt = str(self._IP.outputcache.prompt1).strip()
+            self._IP.indent_current_nsp = 0 #we set indentation to 0
+        sys.stdout = orig_stdout
+    
+    def _shell(self, ip, cmd):
+        '''
+        Replacement method to allow shell commands without them blocking.
+
+        @param ip: Ipython instance, same as self._IP
+        @type cmd: Ipython instance
+        @param cmd: Shell command to execute.
+        @type cmd: string
+        '''
+        stdin, stdout = os.popen4(cmd)
+        result = stdout.read().decode('cp437').encode(locale.getpreferredencoding())
+        #we use print command because the shell command is called inside IPython instance and thus is
+        #redirected to thread cout
+        #"\x01\x1b[1;36m\x02" <-- add colour to the text...
+        print "\x01\x1b[1;36m\x02"+result
+        stdout.close()
+        stdin.close()

IPython/Extensions/igrid_help.html

@@ -1,86 +1,86 b''
-<?xml version='1.0' encoding='iso-8859-1'?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html>
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
-<link rel="stylesheet" href="igrid_help.css" type="text/css" />
-<title>igrid help</title>
-</head>
-<body>
-<h1>igrid help</h1>
-
-
-<h2>Commands</h2>
-
-
-<h3>pick (P)</h3>
-<p>Pick the whole row (object is available as "_")</p>
-
-<h3>pickattr (Shift-P)</h3>
-<p>Pick the attribute under the cursor</p>
-
-<h3>pickallattrs (Shift-C)</h3>
-<p>Pick' the complete column under the cursor (i.e. the attribute under the
-cursor) from all currently fetched objects. These attributes will be returned
-as a list.</p>
-
-<h3>enter (E)</h3>
-<p>Enter the object under the cursor. (what this mean depends on the object
-itself, i.e. how it implements iteration). This opens a new browser 'level'.</p>
-
-<h3>enterattr (Shift-E)</h3>
-<p>Enter the attribute under the cursor.</p>
-
-<h3>detail (D)</h3>
-<p>Show a detail view of the object under the cursor. This shows the name,
-type, doc string and value of the object attributes (and it might show more
-attributes than in the list view, depending on the object).</p>
-
-<h3>detailattr (Shift-D)</h3>
-<p>Show a detail view of the attribute under the cursor.</p>
-
-<h3>pickrows (M)</h3>
-<p>Pick multiple selected rows (M)</p>
-
-<h3>pickrowsattr (CTRL-M)</h3>
-<p>From multiple selected rows pick the cells matching the attribute the cursor is in (CTRL-M)</p>
-
-<h3>find (CTRL-F)</h3>
-<p>Find text</p>
-
-<h3>find_next (F3)</h3>
-<p>Find next occurrence of the searchtext</p>
-
-<h3>find_previous (Shift-F3)</h3>
-<p>Find previous occurrence of the searchtext </p>
-
-<h3>sortattrasc (V)</h3>
-<p>Sort the objects (in ascending order) using the attribute under the cursor as the sort key.</p>
-
-<h3>sortattrdesc (Shift-V)</h3>
-<p>Sort the objects (in descending order) using the attribute under the cursor as the sort key.</p>
-
-<h3>leave (Backspace, DEL, X)</h3>
-<p>Close current tab (and all the tabs to the right of the current one).</h3>
-
-<h3>quit (ESC,Q)</h3>
-<p>Quit igrid and return to the IPython prompt.</p>
-
-
-<h2>Navigation</h2>
-
-
-<h3>Jump to the last column of the current row (END, CTRL-E, CTRL-Right)</h3>
-
-<h3>Jump to the first column of the current row (HOME, CTRL-A, CTRL-Left)</h3>
-
-<h3>Move the cursor one column to the left (&lt;)</h3>
-
-<h3>Move the cursor one column to the right (&gt;)</h3>
-
-<h3>Jump to the first row in the current column (CTRL-Up)</h3>
-
-<h3>Jump to the last row in the current column (CTRL-Down)</h3>
-
-</body>
-</html>
+<?xml version='1.0' encoding='iso-8859-1'?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+<link rel="stylesheet" href="igrid_help.css" type="text/css" />
+<title>igrid help</title>
+</head>
+<body>
+<h1>igrid help</h1>
+
+
+<h2>Commands</h2>
+
+
+<h3>pick (P)</h3>
+<p>Pick the whole row (object is available as "_")</p>
+
+<h3>pickattr (Shift-P)</h3>
+<p>Pick the attribute under the cursor</p>
+
+<h3>pickallattrs (Shift-C)</h3>
+<p>Pick' the complete column under the cursor (i.e. the attribute under the
+cursor) from all currently fetched objects. These attributes will be returned
+as a list.</p>
+
+<h3>enter (E)</h3>
+<p>Enter the object under the cursor. (what this mean depends on the object
+itself, i.e. how it implements iteration). This opens a new browser 'level'.</p>
+
+<h3>enterattr (Shift-E)</h3>
+<p>Enter the attribute under the cursor.</p>
+
+<h3>detail (D)</h3>
+<p>Show a detail view of the object under the cursor. This shows the name,
+type, doc string and value of the object attributes (and it might show more
+attributes than in the list view, depending on the object).</p>
+
+<h3>detailattr (Shift-D)</h3>
+<p>Show a detail view of the attribute under the cursor.</p>
+
+<h3>pickrows (M)</h3>
+<p>Pick multiple selected rows (M)</p>
+
+<h3>pickrowsattr (CTRL-M)</h3>
+<p>From multiple selected rows pick the cells matching the attribute the cursor is in (CTRL-M)</p>
+
+<h3>find (CTRL-F)</h3>
+<p>Find text</p>
+
+<h3>find_next (F3)</h3>
+<p>Find next occurrence of the searchtext</p>
+
+<h3>find_previous (Shift-F3)</h3>
+<p>Find previous occurrence of the searchtext </p>
+
+<h3>sortattrasc (V)</h3>
+<p>Sort the objects (in ascending order) using the attribute under the cursor as the sort key.</p>
+
+<h3>sortattrdesc (Shift-V)</h3>
+<p>Sort the objects (in descending order) using the attribute under the cursor as the sort key.</p>
+
+<h3>leave (Backspace, DEL, X)</h3>
+<p>Close current tab (and all the tabs to the right of the current one).</h3>
+
+<h3>quit (ESC,Q)</h3>
+<p>Quit igrid and return to the IPython prompt.</p>
+
+
+<h2>Navigation</h2>
+
+
+<h3>Jump to the last column of the current row (END, CTRL-E, CTRL-Right)</h3>
+
+<h3>Jump to the first column of the current row (HOME, CTRL-A, CTRL-Left)</h3>
+
+<h3>Move the cursor one column to the left (&lt;)</h3>
+
+<h3>Move the cursor one column to the right (&gt;)</h3>
+
+<h3>Jump to the first row in the current column (CTRL-Up)</h3>
+
+<h3>Jump to the last row in the current column (CTRL-Down)</h3>
+
+</body>
+</html>

IPython/Extensions/ipy_app_completers.py

@@ -1,17 +1,19 b''
 """ Install various IPython completers
 
 IPython extension that installs the completers related to external apps.
 
 The actual implementations are in Extensions/ipy_completers.py
 
 """
 import IPython.ipapi
 
 ip = IPython.ipapi.get()
 
 from ipy_completers import *
 
 ip.set_hook('complete_command', apt_completer, re_key = '.*apt-get')
 ip.set_hook('complete_command', svn_completer, str_key = 'svn')
 ip.set_hook('complete_command', hg_completer, str_key = 'hg')
-ip.set_hook('complete_command', bzr_completer, str_key = 'bzr')
+
+# the old bzr completer is deprecated, we recommend ipy_bzr
+#ip.set_hook('complete_command', bzr_completer, str_key = 'bzr')

IPython/Extensions/ipy_completers.py

@@ -1,359 +1,386 b''
-#!/usr/bin/env python
 
 """ Implementations for various useful completers
 
 See Extensions/ipy_stock_completers.py on examples of how to enable a completer,
 but the basic idea is to do:
 
 ip.set_hook('complete_command', svn_completer, str_key = 'svn')
 
 """
 import IPython.ipapi
 import glob,os,shlex,sys
 import inspect
 from time import time
+from zipimport import zipimporter
 ip = IPython.ipapi.get()
 
 try:
     set
 except:
     from sets import Set as set
 
 TIMEOUT_STORAGE = 3 #Time in seconds after which the rootmodules will be stored
 TIMEOUT_GIVEUP = 20 #Time in seconds after which we give up
 
 def quick_completer(cmd, completions):
     """ Easily create a trivial completer for a command.
 
     Takes either a list of completions, or all completions in string
     (that will be split on whitespace)
     
     Example::
     
         [d:\ipython]|1> import ipy_completers                                       
         [d:\ipython]|2> ipy_completers.quick_completer('foo', ['bar','baz'])        
         [d:\ipython]|3> foo b<TAB>
         bar baz                                                                     
         [d:\ipython]|3> foo ba
     """
     if isinstance(completions, basestring):
         
         completions = completions.split()
     def do_complete(self,event):
         return completions
     
     ip.set_hook('complete_command',do_complete, str_key = cmd)
     
 def getRootModules():
     """
     Returns a list containing the names of all the modules available in the
     folders of the pythonpath.
     """
     modules = []
     if ip.db.has_key('rootmodules'):
         return ip.db['rootmodules']
     t = time()
     store = False
     for path in sys.path:
         modules += moduleList(path)        
         if time() - t >= TIMEOUT_STORAGE and not store:
             store = True
             print "\nCaching the list of root modules, please wait!" 
             print "(This will only be done once - type '%rehashx' to " + \
             "reset cache!)"
             print
         if time() - t > TIMEOUT_GIVEUP:
             print "This is taking too long, we give up."
             print
             ip.db['rootmodules'] = []
             return []
     
     modules += sys.builtin_module_names
       
     modules = list(set(modules))
     if '__init__' in modules:
         modules.remove('__init__')
     modules = list(set(modules))
     if store:
         ip.db['rootmodules'] = modules
     return modules
 
 def moduleList(path):
     """
     Return the list containing the names of the modules available in the given
     folder.
     """
 
     if os.path.isdir(path):
         folder_list = os.listdir(path)
+    elif path.endswith('.egg'):
+        try:
+            folder_list = [f for f in zipimporter(path)._files]
+        except:
+            folder_list = []
     else:
         folder_list = []
     #folder_list = glob.glob(os.path.join(path,'*'))
     folder_list = [p for p in folder_list  \
        if os.path.exists(os.path.join(path, p,'__init__.py'))\
            or p[-3:] in ('.py','.so')\
-           or p[-4:] in ('.pyc','.pyo')]
+           or p[-4:] in ('.pyc','.pyo','.pyd')]
 
     folder_list = [os.path.basename(p).split('.')[0] for p in folder_list]
     return folder_list
 
 def moduleCompletion(line):
     """
     Returns a list containing the completion possibilities for an import line.
     The line looks like this :
     'import xml.d'
     'from xml.dom import'
     """
     def tryImport(mod, only_modules=False):
         def isImportable(module, attr):
             if only_modules:
                 return inspect.ismodule(getattr(module, attr))
             else:
                 return not(attr[:2] == '__' and attr[-2:] == '__')
         try:
             m = __import__(mod)
         except:
             return []
         mods = mod.split('.')
         for module in mods[1:]:
             m = getattr(m,module)
         if (not hasattr(m, '__file__')) or (not only_modules) or\
            (hasattr(m, '__file__') and '__init__' in m.__file__):
             completion_list = [attr for attr in dir(m) if isImportable(m, attr)]
         completion_list.extend(getattr(m,'__all__',[]))
         if hasattr(m, '__file__') and '__init__' in m.__file__:
             completion_list.extend(moduleList(os.path.dirname(m.__file__)))
         completion_list = list(set(completion_list))
         if '__init__' in completion_list:
             completion_list.remove('__init__')
         return completion_list
 
     words = line.split(' ')
     if len(words) == 3 and words[0] == 'from':
         return ['import ']
     if len(words) < 3 and (words[0] in ['import','from']) :
         if len(words) == 1:
             return getRootModules()
         mod = words[1].split('.')
         if len(mod) < 2:
             return getRootModules()
         completion_list = tryImport('.'.join(mod[:-1]), True)
         completion_list = ['.'.join(mod[:-1] + [el]) for el in completion_list]
         return completion_list
     if len(words) >= 3 and words[0] == 'from':
         mod = words[1]
         return tryImport(mod)
 
 def vcs_completer(commands, event):
     """ utility to make writing typical version control app completers easier
 
     VCS command line apps typically have the format:
 
     [sudo ]PROGNAME [help] [command] file file...
 
     """
 
 
     cmd_param = event.line.split()
     if event.line.endswith(' '):
         cmd_param.append('')
 
     if cmd_param[0] == 'sudo':
         cmd_param = cmd_param[1:]
 
     if len(cmd_param) == 2 or 'help' in cmd_param:
         return commands.split()
 
     return ip.IP.Completer.file_matches(event.symbol)
 
 
 pkg_cache = None
 
 def module_completer(self,event):
     """ Give completions after user has typed 'import ...' or 'from ...'"""
 
     # This works in all versions of python.  While 2.5 has
     # pkgutil.walk_packages(), that particular routine is fairly dangerous,
     # since it imports *EVERYTHING* on sys.path.  That is: a) very slow b) full
     # of possibly problematic side effects.
     # This search the folders in the sys.path for available modules.
 
     return moduleCompletion(event.line)
 
 
 svn_commands = """\
 add blame praise annotate ann cat checkout co cleanup commit ci copy
 cp delete del remove rm diff di export help ? h import info list ls
 lock log merge mkdir move mv rename ren propdel pdel pd propedit pedit
 pe propget pget pg proplist plist pl propset pset ps resolved revert
 status stat st switch sw unlock update
 """
 
 def svn_completer(self,event):
     return vcs_completer(svn_commands, event)
 
 
 hg_commands = """
 add addremove annotate archive backout branch branches bundle cat
 clone commit copy diff export grep heads help identify import incoming
 init locate log manifest merge outgoing parents paths pull push
 qapplied qclone qcommit qdelete qdiff qfold qguard qheader qimport
 qinit qnew qnext qpop qprev qpush qrefresh qrename qrestore qsave
 qselect qseries qtop qunapplied recover remove rename revert rollback
 root serve showconfig status strip tag tags tip unbundle update verify
 version
 """
 
 def hg_completer(self,event):
     """ Completer for mercurial commands """
 
     return vcs_completer(hg_commands, event)
 
 
 
-bzr_commands = """
-add annotate bind branch break-lock bundle-revisions cat check
-checkout commit conflicts deleted diff export gannotate gbranch
-gcommit gdiff help ignore ignored info init init-repository inventory
-log merge missing mkdir mv nick pull push reconcile register-branch
-remerge remove renames resolve revert revno root serve sign-my-commits
-status testament unbind uncommit unknowns update upgrade version
-version-info visualise whoami
-"""
+__bzr_commands = None
+
+def bzr_commands():
+    global __bzr_commands
+    if __bzr_commands is not None:
+        return __bzr_commands
+    out = os.popen('bzr help commands')
+    __bzr_commands = [l.split()[0] for l in out]
+    return __bzr_commands                
 
 def bzr_completer(self,event):
     """ Completer for bazaar commands """
     cmd_param = event.line.split()
     if event.line.endswith(' '):
         cmd_param.append('')
 
     if len(cmd_param) > 2:
         cmd = cmd_param[1]
         param = cmd_param[-1]
         output_file = (param == '--output=')
         if cmd == 'help':
-            return bzr_commands.split()
+            return bzr_commands()
         elif cmd in ['bundle-revisions','conflicts',
                      'deleted','nick','register-branch',
                      'serve','unbind','upgrade','version',
                      'whoami'] and not output_file:
             return []
         else:
             # the rest are probably file names
             return ip.IP.Completer.file_matches(event.symbol)
 
-    return bzr_commands.split()
+    return bzr_commands()
 
 
 def shlex_split(x):
     """Helper function to split lines into segments."""
     #shlex.split raise exception if syntax error in sh syntax
     #for example if no closing " is found. This function keeps dropping
     #the last character of the line until shlex.split does not raise
     #exception. Adds end of the line to the result of shlex.split
     #example: %run "c:/python  -> ['%run','"c:/python']
     endofline=[]
     while x!="":
         try:
             comps=shlex.split(x)
             if len(endofline)>=1:
                 comps.append("".join(endofline))
             return comps
         except ValueError:
             endofline=[x[-1:]]+endofline
             x=x[:-1]
     return ["".join(endofline)]
 
 def runlistpy(self, event):
     comps = shlex_split(event.line)
     relpath = (len(comps) > 1 and comps[-1] or '').strip("'\"")
 
     #print "\nev=",event  # dbg
     #print "rp=",relpath  # dbg
     #print 'comps=',comps  # dbg
 
     lglob = glob.glob
     isdir = os.path.isdir
     if relpath.startswith('~'):
         relpath = os.path.expanduser(relpath)
     dirs = [f.replace('\\','/') + "/" for f in lglob(relpath+'*')
             if isdir(f)]
 
     # Find if the user has already typed the first filename, after which we
     # should complete on all files, since after the first one other files may
     # be arguments to the input script.
     #filter(
     if filter(lambda f: f.endswith('.py') or f.endswith('.ipy') or
               f.endswith('.pyw'),comps):
         pys =  [f.replace('\\','/') for f in lglob('*')]
     else:
         pys =  [f.replace('\\','/')
                 for f in lglob(relpath+'*.py') + lglob(relpath+'*.ipy') +
                 lglob(relpath + '*.pyw')]
     return dirs + pys
 
 
 def cd_completer(self, event):
     relpath = event.symbol
     #print event # dbg
     if '-b' in event.line:
         # return only bookmark completions
         bkms = self.db.get('bookmarks',{})
         return bkms.keys()
 
     
     if event.symbol == '-':
         width_dh = str(len(str(len(ip.user_ns['_dh']) + 1)))
         # jump in directory history by number
         fmt = '-%0' + width_dh +'d [%s]'
         ents = [ fmt % (i,s) for i,s in enumerate(ip.user_ns['_dh'])]
         if len(ents) > 1:
             return ents
         return []
 
     if relpath.startswith('~'):
         relpath = os.path.expanduser(relpath).replace('\\','/')
     found = []
     for d in [f.replace('\\','/') + '/' for f in glob.glob(relpath+'*')
               if os.path.isdir(f)]:
         if ' ' in d:
             # we don't want to deal with any of that, complex code
             # for this is elsewhere
             raise IPython.ipapi.TryNext
         found.append( d )
 
     if not found:
         if os.path.isdir(relpath):
             return [relpath]
         raise IPython.ipapi.TryNext
-    return found
+
+
+    def single_dir_expand(matches):
+        "Recursively expand match lists containing a single dir."
+        
+        if len(matches) == 1 and os.path.isdir(matches[0]):
+            # Takes care of links to directories also.  Use '/'
+            # explicitly, even under Windows, so that name completions
+            # don't end up escaped.
+            d = matches[0]
+            if d[-1] in ['/','\\']:
+                d = d[:-1]
+
+            subdirs = [p for p in os.listdir(d) if os.path.isdir( d + '/' + p)]
+            if subdirs:
+                matches = [ (d + '/' + p) for p in subdirs ]
+                return single_dir_expand(matches)
+            else:
+                return matches
+        else:
+            return matches
+
+    return single_dir_expand(found)
 
 def apt_get_packages(prefix):
     out = os.popen('apt-cache pkgnames')
     for p in out:
         if p.startswith(prefix):
             yield p.rstrip()
     
     
 apt_commands = """\
 update upgrade install remove purge source build-dep dist-upgrade
 dselect-upgrade clean autoclean check"""
 
 def apt_completer(self, event):
     """ Completer for apt-get (uses apt-cache internally)
 
     """
 
 
     cmd_param = event.line.split()
     if event.line.endswith(' '):
         cmd_param.append('')
 
     if cmd_param[0] == 'sudo':
         cmd_param = cmd_param[1:]
 
     if len(cmd_param) == 2 or 'help' in cmd_param:
         return apt_commands.split()
 
     return list(apt_get_packages(event.symbol))
 

IPython/Extensions/ipy_leo.py

@@ -1,248 +1,592 b''
-""" Leo plugin for IPython
-
-Example use:
-
-nodes.foo = "hello world"
-
-  -> create '@ipy foo' node with text "hello world"
-
-Access works also, and so does tab completion.
-   
-"""
-import IPython.ipapi
-import IPython.genutils
-import IPython.generics
-import re
-
-
-
-ip = IPython.ipapi.get()
-leo = ip.user_ns['leox']
-c,g = leo.c, leo.g
-
-# will probably be overwritten by user, but handy for experimentation early on
-ip.user_ns['c'] = c
-ip.user_ns['g'] = g
-
-
-from IPython.external.simplegeneric import generic 
-import pprint
-
-@generic
-def format_for_leo(obj):
-    """ Convert obj to string representiation (for editing in Leo)"""
-    return pprint.pformat(obj)
-
-@format_for_leo.when_type(list)
-def format_list(obj):
-    return "\n".join(str(s) for s in obj)
-
-nodename_re = r'(@ipy?[\w-]+)?\s?(\w+)'
-
-def all_cells():
-    d = {}
-    for p in c.allNodes_iter():
-        h = p.headString()
-        if h.startswith('@') and len(h.split()) == 1: 
-            continue
-        mo = re.match(nodename_re, h)
-        if not mo:
-            continue
-        d[mo.group(2)] = p.copy()
-    return d    
-    
-
-class TrivialLeoWorkbook:
-    """ class to find cells with simple syntax
-    
-    """
-    def __getattr__(self, key):
-        cells = all_cells()
-        p = cells[key]
-        body = p.bodyString()
-        return eval_body(body)
-    def __setattr__(self,key,val):
-        cells = all_cells()
-        p = cells.get(key,None)
-        if p is None:
-            add_var(key,val)
-        else:
-            c.setBodyString(p,format_for_leo(val))
-    def __str__(self):
-        return "<TrivialLeoWorkbook>"
-    __repr__ = __str__
-
-ip.user_ns['nodes'] = TrivialLeoWorkbook()            
-
-
-class LeoNode(object):
-    def __init__(self,p):
-        self.p = p.copy()
-
-    def get_h(self): return self.p.headString()
-    def set_h(self,val):
-        print "set head",val
-        c.beginUpdate() 
-        try:
-            c.setHeadString(self.p,val)
-        finally:
-            c.endUpdate()
-        
-    h = property( get_h, set_h)  
-
-    def get_b(self): return self.p.bodyString()
-    def set_b(self,val):
-        print "set body",val
-        c.beginUpdate()
-        try: 
-            c.setBodyString(self.p, val)
-        finally:
-            c.endUpdate()
-    
-    b = property(get_b, set_b)
-    
-    def set_val(self, val):
-        self.b = pprint.pformat(val)
-        
-    v = property(lambda self: ip.ev(self.b.strip()), set_val)
-    
-    def set_l(self,val):
-        self.b = '\n'.join(val )
-    l = property(lambda self : IPython.genutils.SList(self.b.splitlines()), 
-                 set_l)
-    
-    def __iter__(self):
-        return (LeoNode(p) for p in self.p.children_iter())
-     
-
-class LeoWorkbook:
-    """ class for 'advanced' node access """
-    def __getattr__(self, key):
-        if key.startswith('_') or key == 'trait_names':
-            raise AttributeError
-        cells = all_cells()
-        p = cells.get(key, None)
-        if p is None:
-            p = add_var(key,None)
-
-        return LeoNode(p)
-
-    def __str__(self):
-        return "<LeoWorkbook>"
-    __repr__ = __str__
-ip.user_ns['wb'] = LeoWorkbook()
-
-
-_dummyval = object()
[e-mail]_object.when_type(LeoWorkbook)
-def workbook_complete(obj, prev):
-    return all_cells().keys()
-    
-
-def add_var(varname, value = _dummyval):
-    c.beginUpdate()
-    try:
-        
-        nodename = '@ipy-var ' + varname
-        p2 = g.findNodeAnywhere(c,nodename)
-        if not c.positionExists(p2):
-            p2 = c.currentPosition().insertAfter()
-            c.setHeadString(p2,'@ipy ' + varname)
-            
-        c.setCurrentPosition(p2)
-        if value is _dummyval:
-            val = ip.user_ns[varname]
-        else:
-            val = value
-        if val is not None:
-            formatted = format_for_leo(val)
-            c.setBodyString(p2,formatted)
-        return p2
-    finally:
-        c.endUpdate()
-
-def add_file(self,fname):
-    p2 = c.currentPosition().insertAfter()
-
-def push_script(p):
-    c.beginUpdate()
-    try:
-        ohist = ip.IP.output_hist 
-        hstart = len(ip.IP.input_hist)
-        script = g.getScript(c,p,useSelectedText=False,forcePythonSentinels=False,useSentinels=False)
-        
-        script = g.splitLines(script + '\n')
-        script = ''.join(z for z in script if z.strip())
-        
-        ip.runlines(script)
-        
-        has_output = False
-        for idx in range(hstart,len(ip.IP.input_hist)):
-            val = ohist.get(idx,None)
-            if val is None:
-                continue
-            has_output = True
-            inp = ip.IP.input_hist[idx]
-            if inp.strip():
-                g.es('In: %s' % (inp[:40], ),  tabName = 'IPython')
-                
-            g.es('<%d> %s' % (idx, pprint.pformat(ohist[idx],width = 40)), tabName = 'IPython')
-        
-        if not has_output:
-            g.es('ipy run: %s' %( p.headString(),), tabName = 'IPython')
-    finally:
-        c.endUpdate()
-    
-    
-def eval_body(body):
-    try:
-        val = ip.ev(body)
-    except:
-        # just use stringlist if it's not completely legal python expression
-        val = IPython.genutils.SList(body.splitlines())
-    return val 
-    
-def push_variable(p,varname):
-    body = p.bodyString()
-    val = eval_body(body.strip())
-    ip.user_ns[varname] = val
-    g.es('ipy var: %s' % (varname,), tabName = "IPython")
-    
-def push_from_leo(p):    
-    tup = p.headString().split(None,1)
-    # @ipy foo is variable foo
-    if len(tup) == 2 and tup[0] == '@ipy':
-        varname = tup[1]
-        push_variable(p,varname)
-        return
-
-    push_script(p)
-    return
-    
-    
-ip.user_ns['leox'].push = push_from_leo    
-    
-def leo_f(self,s):
-    """ open file(s) in Leo
-    
-    Takes an mglob pattern, e.g. '%leo *.cpp' or %leo 'rec:*.cpp'  
-    """
-    import os
-    from IPython.external import mglob
-    
-    files = mglob.expand(s)
-    c.beginUpdate()
-    try:
-        for fname in files:
-            p = g.findNodeAnywhere(c,'@auto ' + fname)
-            if not p:
-                p = c.currentPosition().insertAfter()
-            
-            p.setHeadString('@auto ' + fname)
-            if os.path.isfile(fname):
-                c.setBodyString(p,open(fname).read())
-            c.selectPosition(p)
-    finally:
-        c.endUpdate()
-
-ip.expose_magic('leo',leo_f)
+""" ILeo - Leo plugin for IPython
+
+   
+"""
+import IPython.ipapi
+import IPython.genutils
+import IPython.generics
+from IPython.hooks import CommandChainDispatcher
+import re
+import UserDict
+from IPython.ipapi import TryNext 
+import IPython.macro
+import IPython.Shell
+
+def init_ipython(ipy):
+    """ This will be run by _ip.load('ipy_leo') 
+    
+    Leo still needs to run update_commander() after this.
+    
+    """
+    global ip
+    ip = ipy
+    IPython.Shell.hijack_tk()
+    ip.set_hook('complete_command', mb_completer, str_key = '%mb')
+    ip.expose_magic('mb',mb_f)
+    ip.expose_magic('lee',lee_f)
+    ip.expose_magic('leoref',leoref_f)
+    expose_ileo_push(push_cl_node,100)
+    # this should be the LAST one that will be executed, and it will never raise TryNext
+    expose_ileo_push(push_ipython_script, 1000)
+    expose_ileo_push(push_plain_python, 100)
+    expose_ileo_push(push_ev_node, 100)
+    global wb
+    wb = LeoWorkbook()
+    ip.user_ns['wb'] = wb 
+    
+    show_welcome()
+
+
+def update_commander(new_leox):
+    """ Set the Leo commander to use
+    
+    This will be run every time Leo does ipython-launch; basically,
+    when the user switches the document he is focusing on, he should do
+    ipython-launch to tell ILeo what document the commands apply to.
+    
+    """
+    
+    global c,g
+    c,g = new_leox.c, new_leox.g
+    print "Set Leo Commander:",c.frame.getTitle()
+    
+    # will probably be overwritten by user, but handy for experimentation early on
+    ip.user_ns['c'] = c
+    ip.user_ns['g'] = g
+    ip.user_ns['_leo'] = new_leox
+    
+    new_leox.push = push_position_from_leo
+    run_leo_startup_node()
+
+from IPython.external.simplegeneric import generic 
+import pprint
+
+def es(s):    
+    g.es(s, tabName = 'IPython')
+    pass
+
+@generic
+def format_for_leo(obj):
+    """ Convert obj to string representiation (for editing in Leo)"""
+    return pprint.pformat(obj)
+
+@format_for_leo.when_type(list)
+def format_list(obj):
+    return "\n".join(str(s) for s in obj)
+  
+
+attribute_re = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*$')
+def valid_attribute(s):
+    return attribute_re.match(s)    
+
+_rootnode = None
+def rootnode():
+    """ Get ileo root node (@ipy-root) 
+    
+    if node has become invalid or has not been set, return None
+    
+    Note that the root is the *first* @ipy-root item found    
+    """
+    global _rootnode
+    if _rootnode is None:
+        return None
+    if c.positionExists(_rootnode.p):
+        return _rootnode
+    _rootnode = None
+    return None  
+
+def all_cells():
+    global _rootnode
+    d = {}
+    r = rootnode() 
+    if r is not None:
+        nodes = r.p.children_iter()
+    else:
+        nodes = c.allNodes_iter()
+
+    for p in nodes:
+        h = p.headString()
+        if h.strip() == '@ipy-root':
+            # update root node (found it for the first time)
+            _rootnode = LeoNode(p)            
+            # the next recursive call will use the children of new root
+            return all_cells()
+        
+        if h.startswith('@a '):
+            d[h.lstrip('@a ').strip()] = p.parent().copy()
+        elif not valid_attribute(h):
+            continue 
+        d[h] = p.copy()
+    return d    
+
+def eval_node(n):
+    body = n.b    
+    if not body.startswith('@cl'):
+        # plain python repr node, just eval it
+        return ip.ev(n.b)
+    # @cl nodes deserve special treatment - first eval the first line (minus cl), then use it to call the rest of body
+    first, rest = body.split('\n',1)
+    tup = first.split(None, 1)
+    # @cl alone SPECIAL USE-> dump var to user_ns
+    if len(tup) == 1:
+        val = ip.ev(rest)
+        ip.user_ns[n.h] = val
+        es("%s = %s" % (n.h, repr(val)[:20]  )) 
+        return val
+
+    cl, hd = tup 
+
+    xformer = ip.ev(hd.strip())
+    es('Transform w/ %s' % repr(xformer))
+    return xformer(rest, n)
+
+class LeoNode(object, UserDict.DictMixin):
+    """ Node in Leo outline
+    
+    Most important attributes (getters/setters available:
+     .v     - evaluate node, can also be alligned 
+     .b, .h - body string, headline string
+     .l     - value as string list
+    
+    Also supports iteration, 
+    
+    setitem / getitem (indexing):  
+     wb.foo['key'] = 12
+     assert wb.foo['key'].v == 12
+    
+    Note the asymmetry on setitem and getitem! Also other
+    dict methods are available. 
+    
+    .ipush() - run push-to-ipython
+
+    Minibuffer command access (tab completion works):
+    
+     mb save-to-file
+    
+    """
+    def __init__(self,p):
+        self.p = p.copy()
+
+    def __str__(self):
+        return "<LeoNode %s>" % str(self.p)
+    
+    __repr__ = __str__
+    
+    def __get_h(self): return self.p.headString()
+    def __set_h(self,val):
+        print "set head",val
+        c.beginUpdate() 
+        try:
+            c.setHeadString(self.p,val)
+        finally:
+            c.endUpdate()
+        
+    h = property( __get_h, __set_h, doc = "Node headline string")  
+
+    def __get_b(self): return self.p.bodyString()
+    def __set_b(self,val):
+        print "set body",val
+        c.beginUpdate()
+        try: 
+            c.setBodyString(self.p, val)
+        finally:
+            c.endUpdate()
+    
+    b = property(__get_b, __set_b, doc = "Nody body string")
+    
+    def __set_val(self, val):        
+        self.b = format_for_leo(val)
+        
+    v = property(lambda self: eval_node(self), __set_val, doc = "Node evaluated value")
+    
+    def __set_l(self,val):
+        self.b = '\n'.join(val )
+    l = property(lambda self : IPython.genutils.SList(self.b.splitlines()), 
+                 __set_l, doc = "Node value as string list")
+    
+    def __iter__(self):
+        """ Iterate through nodes direct children """
+        
+        return (LeoNode(p) for p in self.p.children_iter())
+
+    def __children(self):
+        d = {}
+        for child in self:
+            head = child.h
+            tup = head.split(None,1)
+            if len(tup) > 1 and tup[0] == '@k':
+                d[tup[1]] = child
+                continue
+            
+            if not valid_attribute(head):
+                d[head] = child
+                continue
+        return d
+    def keys(self):
+        d = self.__children()
+        return d.keys()
+    def __getitem__(self, key):
+        """ wb.foo['Some stuff'] Return a child node with headline 'Some stuff'
+        
+        If key is a valid python name (e.g. 'foo'), look for headline '@k foo' as well
+        """  
+        key = str(key)
+        d = self.__children()
+        return d[key]
+    def __setitem__(self, key, val):
+        """ You can do wb.foo['My Stuff'] = 12 to create children 
+        
+        This will create 'My Stuff' as a child of foo (if it does not exist), and 
+        do .v = 12 assignment.
+        
+        Exception:
+        
+        wb.foo['bar'] = 12
+        
+        will create a child with headline '@k bar', because bar is a valid python name
+        and we don't want to crowd the WorkBook namespace with (possibly numerous) entries
+        """
+        key = str(key)
+        d = self.__children()
+        if key in d:
+            d[key].v = val
+            return
+        
+        if not valid_attribute(key):
+            head = key
+        else:
+            head = '@k ' + key
+        p = c.createLastChildNode(self.p, head, '')
+        LeoNode(p).v = val
+        
+    def ipush(self):
+        """ Does push-to-ipython on the node """
+        push_from_leo(self)
+        
+    def go(self):
+        """ Set node as current node (to quickly see it in Outline) """
+        c.beginUpdate()
+        try:
+            c.setCurrentPosition(self.p)
+        finally:
+            c.endUpdate()  
+        
+    def script(self):
+        """ Method to get the 'tangled' contents of the node
+        
+        (parse @others, << section >> references etc.)
+        """
+        return g.getScript(c,self.p,useSelectedText=False,useSentinels=False)
+    
+    def __get_uA(self):
+        p = self.p
+        # Create the uA if necessary.
+        if not hasattr(p.v.t,'unknownAttributes'):
+            p.v.t.unknownAttributes = {}        
+        
+        d = p.v.t.unknownAttributes.setdefault('ipython', {})
+        return d        
+    
+    uA = property(__get_uA, doc = "Access persistent unknownAttributes of node")
+        
+
+class LeoWorkbook:
+    """ class for 'advanced' node access 
+    
+    Has attributes for all "discoverable" nodes. Node is discoverable if it 
+    either
+    
+    - has a valid python name (Foo, bar_12)
+    - is a parent of an anchor node (if it has a child '@a foo', it is visible as foo)
+    
+    """
+    def __getattr__(self, key):
+        if key.startswith('_') or key == 'trait_names' or not valid_attribute(key):
+            raise AttributeError
+        cells = all_cells()
+        p = cells.get(key, None)
+        if p is None:
+            return add_var(key)
+
+        return LeoNode(p)
+
+    def __str__(self):
+        return "<LeoWorkbook>"
+    def __setattr__(self,key, val):
+        raise AttributeError("Direct assignment to workbook denied, try wb.%s.v = %s" % (key,val))
+        
+    __repr__ = __str__
+    
+    def __iter__(self):
+        """ Iterate all (even non-exposed) nodes """
+        cells = all_cells()
+        return (LeoNode(p) for p in c.allNodes_iter())
+    
+    current = property(lambda self: LeoNode(c.currentPosition()), doc = "Currently selected node")
+    
+    def match_h(self, regex):
+        cmp = re.compile(regex)
+        for node in self:
+            if re.match(cmp, node.h, re.IGNORECASE):
+                yield node
+        return
+
[e-mail]_object.when_type(LeoWorkbook)
+def workbook_complete(obj, prev):
+    return all_cells().keys() + [s for s in prev if not s.startswith('_')]
+    
+
+def add_var(varname):
+    c.beginUpdate()
+    r = rootnode()
+    try:
+        if r is None:
+            p2 = g.findNodeAnywhere(c,varname)
+        else:
+            p2 = g.findNodeInChildren(c, r.p, varname)
+        if p2:
+            return LeoNode(p2)
+
+        if r is not None:
+            p2 = r.p.insertAsLastChild()
+        
+        else:
+            p2 =  c.currentPosition().insertAfter()
+        
+        c.setHeadString(p2,varname)
+        return LeoNode(p2)
+    finally:
+        c.endUpdate()
+
+def add_file(self,fname):
+    p2 = c.currentPosition().insertAfter()
+
+push_from_leo = CommandChainDispatcher()
+
+def expose_ileo_push(f, prio = 0):
+    push_from_leo.add(f, prio)
+
+def push_ipython_script(node):
+    """ Execute the node body in IPython, as if it was entered in interactive prompt """
+    c.beginUpdate()
+    try:
+        ohist = ip.IP.output_hist 
+        hstart = len(ip.IP.input_hist)
+        script = node.script()
+                
+        ip.user_ns['_p'] = node
+        ip.runlines(script)
+        ip.user_ns.pop('_p',None)
+        
+        has_output = False
+        for idx in range(hstart,len(ip.IP.input_hist)):
+            val = ohist.get(idx,None)
+            if val is None:
+                continue
+            has_output = True
+            inp = ip.IP.input_hist[idx]
+            if inp.strip():
+                es('In: %s' % (inp[:40], ))
+                
+            es('<%d> %s' % (idx, pprint.pformat(ohist[idx],width = 40)))
+        
+        if not has_output:
+            es('ipy run: %s (%d LL)' %( node.h,len(script)))
+    finally:
+        c.endUpdate()
+
+    
+def eval_body(body):
+    try:
+        val = ip.ev(body)
+    except:
+        # just use stringlist if it's not completely legal python expression
+        val = IPython.genutils.SList(body.splitlines())
+    return val 
+    
+def push_plain_python(node):
+    if not node.h.endswith('P'):
+        raise TryNext
+    script = node.script()
+    lines = script.count('\n')
+    try:
+        exec script in ip.user_ns
+    except:
+        print " -- Exception in script:\n"+script + "\n --"
+        raise
+    es('ipy plain: %s (%d LL)' % (node.h,lines))
+    
+
+def push_cl_node(node):
+    """ If node starts with @cl, eval it
+    
+    The result is put as last child of @ipy-results node, if it exists
+    """
+    if not node.b.startswith('@cl'):
+        raise TryNext
+        
+    p2 = g.findNodeAnywhere(c,'@ipy-results')
+    val = node.v
+    if p2:
+        es("=> @ipy-results")
+        LeoNode(p2).v = val
+    es(val)
+
+def push_ev_node(node):
+    """ If headline starts with @ev, eval it and put result in body """
+    if not node.h.startswith('@ev '):
+        raise TryNext
+    expr = node.h.lstrip('@ev ')
+    es('ipy eval ' + expr)
+    res = ip.ev(expr)
+    node.v = res
+    
+    
+def push_position_from_leo(p):
+    try:
+        push_from_leo(LeoNode(p))
+    except AttributeError,e:
+        if e.args == ("Commands instance has no attribute 'frame'",):
+            es("Error: ILeo not associated with .leo document")
+            es("Press alt+shift+I to fix!")
+        else:
+            raise
+
+@generic
+def edit_object_in_leo(obj, varname):
+    """ Make it @cl node so it can be pushed back directly by alt+I """
+    node = add_var(varname)
+    formatted = format_for_leo(obj)
+    if not formatted.startswith('@cl'):
+        formatted = '@cl\n' + formatted
+    node.b = formatted 
+    node.go()
+    
+@edit_object_in_leo.when_type(IPython.macro.Macro)
+def edit_macro(obj,varname):
+    bod = '_ip.defmacro("""\\\n' + obj.value + '""")'
+    node = add_var('Macro_' + varname)
+    node.b = bod
+    node.go()
+
+def get_history(hstart = 0):
+    res = []
+    ohist = ip.IP.output_hist 
+
+    for idx in range(hstart, len(ip.IP.input_hist)):
+        val = ohist.get(idx,None)
+        has_output = True
+        inp = ip.IP.input_hist_raw[idx]
+        if inp.strip():
+            res.append('In [%d]: %s' % (idx, inp))
+        if val:
+            res.append(pprint.pformat(val))
+            res.append('\n')    
+    return ''.join(res)
+    
+    
+def lee_f(self,s):
+    """ Open file(s)/objects in Leo
+    
+    - %lee hist -> open full session history in leo
+    - Takes an object. l = [1,2,"hello"]; %lee l. Alt+I in leo pushes the object back
+    - Takes an mglob pattern, e.g. '%lee *.cpp' or %lee 'rec:*.cpp'
+    - Takes input history indices:  %lee 4 6-8 10 12-47
+    """
+    import os
+        
+    c.beginUpdate()
+    try:
+        if s == 'hist':
+            wb.ipython_history.b = get_history()
+            wb.ipython_history.go()
+            return
+        
+            
+        if s and s[0].isdigit():
+            # numbers; push input slices to leo
+            lines = self.extract_input_slices(s.strip().split(), True)
+            v = add_var('stored_ipython_input')
+            v.b = '\n'.join(lines)
+            return
+            
+        
+        # try editing the object directly
+        obj = ip.user_ns.get(s, None)
+        if obj is not None:
+            edit_object_in_leo(obj,s)
+            return
+     
+        
+        # if it's not object, it's a file name / mglob pattern
+        from IPython.external import mglob
+        
+        files = (os.path.abspath(f) for f in mglob.expand(s))
+        for fname in files:
+            p = g.findNodeAnywhere(c,'@auto ' + fname)
+            if not p:
+                p = c.currentPosition().insertAfter()
+            
+            p.setHeadString('@auto ' + fname)
+            if os.path.isfile(fname):
+                c.setBodyString(p,open(fname).read())
+            c.selectPosition(p)
+        print "Editing file(s), press ctrl+shift+w in Leo to write @auto nodes"
+    finally:
+        c.endUpdate()
+
+
+
+def leoref_f(self,s):
+    """ Quick reference for ILeo """
+    import textwrap
+    print textwrap.dedent("""\
+    %leoe file/object - open file / object in leo
+    wb.foo.v  - eval node foo (i.e. headstring is 'foo' or '@ipy foo')
+    wb.foo.v = 12 - assign to body of node foo
+    wb.foo.b - read or write the body of node foo
+    wb.foo.l - body of node foo as string list
+    
+    for el in wb.foo:
+      print el.v
+       
+    """
+    )
+
+
+
+def mb_f(self, arg):
+    """ Execute leo minibuffer commands 
+    
+    Example:
+     mb save-to-file
+    """
+    c.executeMinibufferCommand(arg)
+
+def mb_completer(self,event):
+    """ Custom completer for minibuffer """
+    cmd_param = event.line.split()
+    if event.line.endswith(' '):
+        cmd_param.append('')
+    if len(cmd_param) > 2:
+        return ip.IP.Completer.file_matches(event.symbol)
+    cmds = c.commandsDict.keys()
+    cmds.sort()
+    return cmds
+
+def show_welcome():
+    print "------------------"
+    print "Welcome to Leo-enabled IPython session!"
+    print "Try %leoref for quick reference."
+    import IPython.platutils
+    IPython.platutils.set_term_title('ILeo')
+    IPython.platutils.freeze_term_title()
+
+def run_leo_startup_node():
+    p = g.findNodeAnywhere(c,'@ipy-startup')
+    if p:
+        print "Running @ipy-startup nodes"
+        for n in LeoNode(p):
+            push_from_leo(n)
+            
+

IPython/Extensions/ipy_profile_sh.py

@@ -1,226 +1,251 b''
 """Shell mode for IPython.
 
 Start ipython in shell mode by invoking "ipython -p sh"
 
 (the old version, "ipython -p pysh" still works but this is the more "modern" 
 shell mode and is recommended for users who don't care about pysh-mode
 compatibility)
 """
 
 from IPython import ipapi
 import os,textwrap
 
 # The import below effectively obsoletes your old-style ipythonrc[.ini],
 # so consider yourself warned!
 
 import ipy_defaults
 
 def main():    
     ip = ipapi.get()
     o = ip.options
     # autocall to "full" mode (smart mode is default, I like full mode)
     
     o.autocall = 2
     
     # Jason Orendorff's path class is handy to have in user namespace
     # if you are doing shell-like stuff
     try:
         ip.ex("from IPython.external.path import path" )
     except ImportError:
         pass
     
     # beefed up %env is handy in shell mode
     import envpersist
     
     # To see where mycmd resides (in path/aliases), do %which mycmd 
     import ipy_which
     
     # tab completers for hg, svn, ...
     import ipy_app_completers
     
     # To make executables foo and bar in mybin usable without PATH change, do:
     # %rehashdir c:/mybin
     # %store foo
     # %store bar
     import ipy_rehashdir
-    import ipy_signals
+
+    # does not work without subprocess module!
+    #import ipy_signals
     
     ip.ex('import os')
     ip.ex("def up(): os.chdir('..')")
-        
+    ip.user_ns['LA'] = LastArgFinder() 
     # Nice prompt
     
     o.prompt_in1= r'\C_LightBlue[\C_LightCyan\Y2\C_LightBlue]\C_Green|\#> '
     o.prompt_in2= r'\C_Green|\C_LightGreen\D\C_Green> '
     o.prompt_out= '<\#> '
     
     from IPython import Release
 
     import sys
     # Non-chatty banner
     o.banner = "IPython %s   [on Py %s]\n" % (Release.version,sys.version.split(None,1)[0])
     
     
     ip.IP.default_option('cd','-q')
     ip.IP.default_option('macro', '-r')
     # If you only rarely want to execute the things you %edit...  
     #ip.IP.default_option('edit','-x')
     
 
     o.prompts_pad_left="1"
     # Remove all blank lines in between prompts, like a normal shell.
     o.separate_in="0"
     o.separate_out="0"
     o.separate_out2="0"
     
     # now alias all syscommands
     
     db = ip.db
     
     syscmds = db.get("syscmdlist",[] )
     if not syscmds:
         print textwrap.dedent("""
         System command list not initialized, probably the first run...
         running %rehashx to refresh the command list. Run %rehashx
         again to refresh command list (after installing new software etc.)
         """)
         ip.magic('rehashx')
         syscmds = db.get("syscmdlist")
     
     # lowcase aliases on win32 only
     if os.name == 'posix':
         mapper = lambda s:s
     else:
         def mapper(s): return s.lower()
         
     for cmd in syscmds:
         # print "sys",cmd #dbg
         noext, ext = os.path.splitext(cmd)
         key = mapper(noext)
         if key not in ip.IP.alias_table:
             ip.defalias(key, cmd)
 
     # mglob combines 'find', recursion, exclusion... '%mglob?' to learn more
     ip.load("IPython.external.mglob")    
 
     # win32 is crippled w/o cygwin, try to help it a little bit
     if sys.platform == 'win32':
         if 'cygwin' in os.environ['PATH'].lower():          
             # use the colors of cygwin ls (recommended)
             ip.defalias('d', 'ls -F --color=auto')
         else:
             # get icp, imv, imkdir, igrep, irm,...
             ip.load('ipy_fsops')
             
             # and the next best thing to real 'ls -F'
             ip.defalias('d','dir /w /og /on')
     
     extend_shell_behavior(ip)
 
+class LastArgFinder:
+    """ Allow $LA to work as "last argument of previous command", like $! in bash
+    
+    To call this in normal IPython code, do LA()
+    """
+    def __call__(self, hist_idx = None):
+        ip = ipapi.get()
+        if hist_idx is None:
+            return str(self)
+        return ip.IP.input_hist_raw[hist_idx].strip().split()[-1]
+    def __str__(self):
+        ip = ipapi.get()        
+        for cmd in reversed(ip.IP.input_hist_raw):
+            parts = cmd.strip().split()
+            if len(parts) < 2 or parts[-1] in ['$LA', 'LA()']:
+                continue
+            return parts[-1]
+        return ""
+
+
+
+
+
 # XXX You do not need to understand the next function!
 # This should probably be moved out of profile
 
 def extend_shell_behavior(ip):
 
     # Instead of making signature a global variable tie it to IPSHELL.
     # In future if it is required to distinguish between different
     # shells we can assign a signature per shell basis
     ip.IP.__sig__ = 0xa005
     # mark the IPSHELL with this signature
     ip.IP.user_ns['__builtins__'].__dict__['__sig__'] = ip.IP.__sig__
 
     from IPython.Itpl import ItplNS
     from IPython.genutils import shell
     # utility to expand user variables via Itpl
     # xxx do something sensible with depth?
     ip.IP.var_expand = lambda cmd, lvars=None, depth=2: \
         str(ItplNS(cmd, ip.IP.user_ns, get_locals()))
 
     def get_locals():
         """ Substituting a variable through Itpl deep inside the IPSHELL stack
             requires the knowledge of all the variables in scope upto the last
             IPSHELL frame. This routine simply merges all the local variables
             on the IPSHELL stack without worrying about their scope rules
         """
         import sys
         # note lambda expression constitues a function call
         # hence fno should be incremented by one
         getsig = lambda fno: sys._getframe(fno+1).f_globals \
                              ['__builtins__'].__dict__['__sig__']
         getlvars = lambda fno: sys._getframe(fno+1).f_locals
         # trackback until we enter the IPSHELL
         frame_no = 1
         sig = ip.IP.__sig__
         fsig = ~sig
         while fsig != sig :
             try:
                 fsig = getsig(frame_no)
             except (AttributeError, KeyError):
                 frame_no += 1
             except ValueError:
                 # stack is depleted
                 # call did not originate from IPSHELL
                 return {}
         first_frame = frame_no
         # walk further back until we exit from IPSHELL or deplete stack
         try:
             while(sig == getsig(frame_no+1)):
                 frame_no += 1
         except (AttributeError, KeyError, ValueError):
             pass
         # merge the locals from top down hence overriding
         # any re-definitions of variables, functions etc.
         lvars = {}
         for fno in range(frame_no, first_frame-1, -1):
             lvars.update(getlvars(fno))
         #print '\n'*5, first_frame, frame_no, '\n', lvars, '\n'*5 #dbg
         return lvars
 
     def _runlines(lines):
         """Run a string of one or more lines of source.
 
         This method is capable of running a string containing multiple source
         lines, as if they had been entered at the IPython prompt.  Since it
         exposes IPython's processing machinery, the given strings can contain
         magic calls (%magic), special shell access (!cmd), etc."""
 
         # We must start with a clean buffer, in case this is run from an
         # interactive IPython session (via a magic, for example).
         ip.IP.resetbuffer()
         lines = lines.split('\n')
         more = 0
         command = ''
         for line in lines:
             # skip blank lines so we don't mess up the prompt counter, but do
             # NOT skip even a blank line if we are in a code block (more is
             # true)
             # if command is not empty trim the line
             if command != '' :
                 line = line.strip()
             # add the broken line to the command
             if line and line[-1] == '\\' :
                 command += line[0:-1] + ' '
                 more = True
                 continue
             else :
                 # add the last (current) line to the command
                 command += line
                 if command or more:
                     # push to raw history, so hist line numbers stay in sync
                     ip.IP.input_hist_raw.append("# " + command + "\n")
                     
                     more = ip.IP.push(ip.IP.prefilter(command,more))
                     command = ''
                     # IPython's runsource returns None if there was an error
                     # compiling the code.  This allows us to stop processing right
                     # away, so the user gets the error message at the right place.
                     if more is None:
                         break
         # final newline in case the input didn't have it, so that the code
         # actually does get executed
         if more:
             ip.IP.push('\n')
 
     ip.IP.runlines = _runlines
 
 main()

IPython/Extensions/jobctrl.py

@@ -1,162 +1,240 b''
 """ Preliminary "job control" extensions for IPython 
 
 requires python 2.4 (or separate 'subprocess' module
 
 This provides 2 features, launching background jobs and killing foreground jobs from another IPython instance.
 
 Launching background jobs:
 
     Usage:
 
     [ipython]|2> import jobctrl
     [ipython]|3> &ls
              <3> <jobctrl.IpyPopen object at 0x00D87FD0>
     [ipython]|4> _3.go
     -----------> _3.go()
     ChangeLog
     IPython
     MANIFEST.in
     README
     README_Windows.txt
 
     ...
 
 Killing foreground tasks:
 
 Launch IPython instance, run a blocking command:
 
     [Q:/ipython]|1> import jobctrl
     [Q:/ipython]|2> cat
 
 Now launch a new IPython prompt and kill the process:
 
     IPython 0.8.3.svn.r2919   [on Py 2.5]
     [Q:/ipython]|1> import jobctrl
     [Q:/ipython]|2> %tasks
     6020: 'cat ' (Q:\ipython)
     [Q:/ipython]|3> %kill
     SUCCESS: The process with PID 6020 has been terminated.
     [Q:/ipython]|4>
 
 (you don't need to specify PID for %kill if only one task is running)
 """                     
 
-from subprocess import Popen,PIPE
+from subprocess import *
 import os,shlex,sys,time
+import threading,Queue
 
 from IPython import genutils
 
 import IPython.ipapi
 
 if os.name == 'nt':
     def kill_process(pid):
         os.system('taskkill /F /PID %d' % pid)
 else:
     def kill_process(pid):
         os.system('kill -9 %d' % pid)
     
     
 
 class IpyPopen(Popen):
     def go(self):
         print self.communicate()[0]
     def __repr__(self):
         return '<IPython job "%s" PID=%d>' % (self.line, self.pid)
 
     def kill(self):
         kill_process(self.pid)
                   
 def startjob(job):
     p = IpyPopen(shlex.split(job), stdout=PIPE, shell = False)
     p.line = job
     return p
 
+class AsyncJobQ(threading.Thread):
+    def __init__(self):
+        threading.Thread.__init__(self)
+        self.q = Queue.Queue()
+        self.output = []
+        self.stop = False
+    def run(self):
+        while 1:
+            cmd,cwd = self.q.get()
+            if self.stop:
+                self.output.append("** Discarding: '%s' - %s" % (cmd,cwd))
+                continue
+            self.output.append("** Task started: '%s' - %s" % (cmd,cwd))
+            
+            p = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT, cwd = cwd)
+            out = p.stdout.read()
+            self.output.append("** Task complete: '%s'\n" % cmd)
+            self.output.append(out)
+
+    def add(self,cmd):
+        self.q.put_nowait((cmd, os.getcwd()))
+        
+    def dumpoutput(self):
+        while self.output:
+            item = self.output.pop(0)
+            print item            
+
+_jobq = None
+
+def jobqueue_f(self, line):
+     
+    global _jobq
+    if not _jobq:
+        print "Starting jobqueue - do '&some_long_lasting_system_command' to enqueue"
+        _jobq = AsyncJobQ()
+        _jobq.setDaemon(True)
+        _jobq.start()
+        ip.jobq = _jobq.add
+        return
+    if line.strip() == 'stop':
+        print "Stopping and clearing jobqueue, %jobqueue start to start again"
+        _jobq.stop = True
+        return
+    if line.strip() == 'start':
+        _jobq.stop = False
+        return
+        
 def jobctrl_prefilter_f(self,line):    
     if line.startswith('&'):
         pre,fn,rest = self.split_user_input(line[1:])
         
         line = ip.IP.expand_aliases(fn,rest)
-        return '_ip.startjob(%s)' % genutils.make_quoted_expr(line)
+        if not _jobq:
+            return '_ip.startjob(%s)' % genutils.make_quoted_expr(line)
+        return '_ip.jobq(%s)' % genutils.make_quoted_expr(line)
 
     raise IPython.ipapi.TryNext
 
+def jobq_output_hook(self):
+    if not _jobq:
+        return
+    _jobq.dumpoutput()
+
+
 
 def job_list(ip):
     keys = ip.db.keys('tasks/*')
     ents = [ip.db[k] for k in keys]
     return ents
 
 def magic_tasks(self,line):
     """ Show a list of tasks.
 
     A 'task' is a process that has been started in IPython when 'jobctrl' extension is enabled.
     Tasks can be killed with %kill.
+    
+    '%tasks clear' clears the task list (from stale tasks)
     """
     ip = self.getapi()
+    if line.strip() == 'clear':
+        for k in ip.db.keys('tasks/*'):
+            print "Clearing",ip.db[k]
+            del ip.db[k]
+        return
+        
     ents = job_list(ip)
     if not ents:
         print "No tasks running"
     for pid,cmd,cwd,t in ents:
         dur = int(time.time()-t)
         print "%d: '%s' (%s) %d:%02d" % (pid,cmd,cwd, dur / 60,dur%60)
 
 def magic_kill(self,line):
     """ Kill a task
 
     Without args, either kill one task (if only one running) or show list (if many)
     With arg, assume it's the process id.
 
     %kill is typically (much) more powerful than trying to terminate a process with ctrl+C.
     """
     ip = self.getapi()
     jobs  = job_list(ip)
 
     if not line.strip():
         if len(jobs) == 1:
             kill_process(jobs[0][0])
         else:
             magic_tasks(self,line)
         return
         
     try:
         pid = int(line)
         kill_process(pid)
     except ValueError:
         magic_tasks(self,line)
         
 if sys.platform == 'win32':
-    shell_internal_commands = 'break chcp cls copy ctty date del erase dir md mkdir path prompt rd rmdir time 	type ver vol'.split()
+    shell_internal_commands = 'break chcp cls copy ctty date del erase dir md mkdir path prompt rd rmdir start time type ver vol'.split()
 else:
     # todo linux commands
     shell_internal_commands = [] 
 
 
 def jobctrl_shellcmd(ip,cmd):
     """ os.system replacement that stores process info to db['tasks/t1234'] """
+    cmd = cmd.strip()
     cmdname = cmd.split(None,1)[0]
-    if cmdname in shell_internal_commands:
+    if cmdname in shell_internal_commands or '|' in cmd or '>' in cmd or '<' in cmd:
         use_shell = True
     else:
         use_shell = False
 
-    p = Popen(cmd,shell = use_shell)
-    jobentry = 'tasks/t' + str(p.pid)
-    
+    jobentry = None
     try:
+        try:
+            p = Popen(cmd,shell = use_shell)
+        except WindowsError:
+            if use_shell:
+                # try with os.system
+                os.system(cmd)
+                return
+            else:
+                # have to go via shell, sucks
+                p = Popen(cmd,shell = True)
+            
+        jobentry = 'tasks/t' + str(p.pid)
         ip.db[jobentry] = (p.pid,cmd,os.getcwd(),time.time())
-        p.communicate()
+        p.communicate()        
+
     finally:
-        del ip.db[jobentry]
+        if jobentry:
+            del ip.db[jobentry]
 
 
 def install():
     global ip
     ip = IPython.ipapi.get()
     # needed to make startjob visible as _ip.startjob('blah')
     ip.startjob = startjob
     ip.set_hook('input_prefilter', jobctrl_prefilter_f)
     ip.set_hook('shell_hook', jobctrl_shellcmd)
     ip.expose_magic('kill',magic_kill)
     ip.expose_magic('tasks',magic_tasks)
-    
+    ip.expose_magic('jobqueue',jobqueue_f)
+    ip.set_hook('pre_prompt_hook', jobq_output_hook) 
 install()

IPython/Itpl.py

@@ -1,284 +1,292 b''
 # -*- coding: utf-8 -*-
 """String interpolation for Python (by Ka-Ping Yee, 14 Feb 2000).
 
 This module lets you quickly and conveniently interpolate values into
 strings (in the flavour of Perl or Tcl, but with less extraneous
 punctuation).  You get a bit more power than in the other languages,
 because this module allows subscripting, slicing, function calls,
 attribute lookup, or arbitrary expressions.  Variables and expressions
 are evaluated in the namespace of the caller.
 
 The itpl() function returns the result of interpolating a string, and
 printpl() prints out an interpolated string.  Here are some examples:
 
     from Itpl import printpl
     printpl("Here is a $string.")
     printpl("Here is a $module.member.")
     printpl("Here is an $object.member.")
     printpl("Here is a $functioncall(with, arguments).")
     printpl("Here is an ${arbitrary + expression}.")
     printpl("Here is an $array[3] member.")
     printpl("Here is a $dictionary['member'].")
 
 The filter() function filters a file object so that output through it
 is interpolated.  This lets you produce the illusion that Python knows
 how to do interpolation:
 
     import Itpl
     sys.stdout = Itpl.filter()
     f = "fancy"
     print "Isn't this $f?"
     print "Standard output has been replaced with a $sys.stdout object."
     sys.stdout = Itpl.unfilter()
     print "Okay, back $to $normal."
 
 Under the hood, the Itpl class represents a string that knows how to
 interpolate values.  An instance of the class parses the string once
 upon initialization; the evaluation and substitution can then be done
 each time the instance is evaluated with str(instance).  For example:
 
     from Itpl import Itpl
     s = Itpl("Here is $foo.")
     foo = 5
     print str(s)
     foo = "bar"
     print str(s)
 
 $Id: Itpl.py 2918 2007-12-31 14:34:47Z vivainio $
 """                   # ' -> close an open quote for stupid emacs
 
 #*****************************************************************************
 #
 # Copyright (c) 2001 Ka-Ping Yee <[e-mail]>
 #
 #
 # Published under the terms of the MIT license, hereby reproduced:
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to
 # deal in the Software without restriction, including without limitation the
 # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 # sell copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
 #
 #*****************************************************************************
 
 __author__  = 'Ka-Ping Yee <[e-mail]>'
 __license__ = 'MIT'
 
 import string
 import sys
 from tokenize import tokenprog
 from types import StringType
 
 class ItplError(ValueError):
     def __init__(self, text, pos):
         self.text = text
         self.pos = pos
     def __str__(self):
         return "unfinished expression in %s at char %d" % (
             repr(self.text), self.pos)
 
 def matchorfail(text, pos):
     match = tokenprog.match(text, pos)
     if match is None:
         raise ItplError(text, pos)
+
     return match, match.end()
 
+try:
+    itpl_encoding = sys.stdin.encoding or 'ascii'
+except AttributeError:
+    itpl_encoding = 'ascii'
+
+    
+
 class Itpl:
     """Class representing a string with interpolation abilities.
     
     Upon creation, an instance works out what parts of the format
     string are literal and what parts need to be evaluated.  The
     evaluation and substitution happens in the namespace of the
     caller when str(instance) is called."""
 
-    def __init__(self, format,codec=sys.stdin.encoding,encoding_errors='backslashreplace'):
+    def __init__(self, format,codec=itpl_encoding,encoding_errors='backslashreplace'):
         """The single mandatory argument to this constructor is a format
         string.
 
         The format string is parsed according to the following rules:
 
         1.  A dollar sign and a name, possibly followed by any of: 
               - an open-paren, and anything up to the matching paren 
               - an open-bracket, and anything up to the matching bracket 
               - a period and a name 
             any number of times, is evaluated as a Python expression.
 
         2.  A dollar sign immediately followed by an open-brace, and
             anything up to the matching close-brace, is evaluated as
             a Python expression.
 
         3.  Outside of the expressions described in the above two rules,
             two dollar signs in a row give you one literal dollar sign.
 
         Optional arguments:
 
         - codec('utf_8'): a string containing the name of a valid Python
         codec.
 
         - encoding_errors('backslashreplace'): a string with a valid error handling
         policy.  See the codecs module documentation for details.
 
         These are used to encode the format string if a call to str() fails on
         the expanded result."""
 
         if not isinstance(format,basestring):
             raise TypeError, "needs string initializer"
         self.format = format
         self.codec = codec
         self.encoding_errors = encoding_errors
         
         namechars = "abcdefghijklmnopqrstuvwxyz" \
             "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_";
         chunks = []
         pos = 0
 
         while 1:
             dollar = string.find(format, "$", pos)
             if dollar < 0: break
             nextchar = format[dollar+1]
 
             if nextchar == "{":
                 chunks.append((0, format[pos:dollar]))
                 pos, level = dollar+2, 1
                 while level:
                     match, pos = matchorfail(format, pos)
                     tstart, tend = match.regs[3]
                     token = format[tstart:tend]
                     if token == "{": level = level+1
                     elif token == "}": level = level-1
                 chunks.append((1, format[dollar+2:pos-1]))
 
             elif nextchar in namechars:
                 chunks.append((0, format[pos:dollar]))
                 match, pos = matchorfail(format, dollar+1)
                 while pos < len(format):
                     if format[pos] == "." and \
                         pos+1 < len(format) and format[pos+1] in namechars:
                         match, pos = matchorfail(format, pos+1)
                     elif format[pos] in "([":
                         pos, level = pos+1, 1
                         while level:
                             match, pos = matchorfail(format, pos)
                             tstart, tend = match.regs[3]
                             token = format[tstart:tend]
                             if token[0] in "([": level = level+1
                             elif token[0] in ")]": level = level-1
                     else: break
                 chunks.append((1, format[dollar+1:pos]))
 
             else:
                 chunks.append((0, format[pos:dollar+1]))
                 pos = dollar + 1 + (nextchar == "$")
 
         if pos < len(format): chunks.append((0, format[pos:]))
         self.chunks = chunks
 
     def __repr__(self):
         return "<Itpl %s >" % repr(self.format)
 
     def _str(self,glob,loc):
         """Evaluate to a string in the given globals/locals.
 
         The final output is built by calling str(), but if this fails, the
         result is encoded with the instance's codec and error handling policy,
         via a call to out.encode(self.codec,self.encoding_errors)"""
         result = []
         app = result.append
         for live, chunk in self.chunks:
             if live:
                 val = eval(chunk,glob,loc)
                 try:
                     app(str(val))
                 except UnicodeEncodeError:
                     app(unicode(val))
                 
             else: app(chunk)
         out = ''.join(result)
         try:
             return str(out)
         except UnicodeError:
             return out.encode(self.codec,self.encoding_errors)
 
     def __str__(self):
         """Evaluate and substitute the appropriate parts of the string."""
 
         # We need to skip enough frames to get to the actual caller outside of
         # Itpl.
         frame = sys._getframe(1)
         while frame.f_globals["__name__"] == __name__: frame = frame.f_back
         loc, glob = frame.f_locals, frame.f_globals
 
         return self._str(glob,loc)
     
 class ItplNS(Itpl):
     """Class representing a string with interpolation abilities.
 
     This inherits from Itpl, but at creation time a namespace is provided
     where the evaluation will occur.  The interpolation becomes a bit more
     efficient, as no traceback needs to be extracte.  It also allows the
     caller to supply a different namespace for the interpolation to occur than
     its own."""
     
     def __init__(self, format,globals,locals=None,
                  codec='utf_8',encoding_errors='backslashreplace'):
         """ItplNS(format,globals[,locals]) -> interpolating string instance.
 
         This constructor, besides a format string, takes a globals dictionary
         and optionally a locals (which defaults to globals if not provided).
 
         For further details, see the Itpl constructor."""
 
         if locals is None:
             locals = globals
         self.globals = globals
         self.locals = locals
         Itpl.__init__(self,format,codec,encoding_errors)
         
     def __str__(self):
         """Evaluate and substitute the appropriate parts of the string."""
         return self._str(self.globals,self.locals)
 
     def __repr__(self):
         return "<ItplNS %s >" % repr(self.format)
 
 # utilities for fast printing
 def itpl(text): return str(Itpl(text))
 def printpl(text): print itpl(text)
 # versions with namespace
 def itplns(text,globals,locals=None): return str(ItplNS(text,globals,locals))
 def printplns(text,globals,locals=None): print itplns(text,globals,locals)
 
 class ItplFile:
     """A file object that filters each write() through an interpolator."""
     def __init__(self, file): self.file = file
     def __repr__(self): return "<interpolated " + repr(self.file) + ">"
     def __getattr__(self, attr): return getattr(self.file, attr)
     def write(self, text): self.file.write(str(Itpl(text)))
 
 def filter(file=sys.stdout):
     """Return an ItplFile that filters writes to the given file object.
     
     'file = filter(file)' replaces 'file' with a filtered object that
     has a write() method.  When called with no argument, this creates
     a filter to sys.stdout."""
     return ItplFile(file)
 
 def unfilter(ifile=None):
     """Return the original file that corresponds to the given ItplFile.
     
     'file = unfilter(file)' undoes the effect of 'file = filter(file)'.
     'sys.stdout = unfilter()' undoes the effect of 'sys.stdout = filter()'."""
     return ifile and ifile.file or sys.stdout.file

IPython/Magic.py

@@ -1,3283 +1,3319 b''
 # -*- coding: utf-8 -*-
 """Magic functions for InteractiveShell.
 
 $Id: Magic.py 2996 2008-01-30 06:31:39Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001 Janko Hauser <[e-mail]> and
 #       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 #****************************************************************************
 # Modules and globals
 
 from IPython import Release
 __author__  = '%s <%s>\n%s <%s>' % \
               ( Release.authors['Janko'] + Release.authors['Fernando'] )
 __license__ = Release.license
 
 # Python standard modules
 import __builtin__
 import bdb
 import inspect
 import os
 import pdb
 import pydoc
 import sys
 import re
 import tempfile
 import time
 import cPickle as pickle
 import textwrap
 from cStringIO import StringIO
 from getopt import getopt,GetoptError
 from pprint import pprint, pformat
 from sets import Set
 
 # cProfile was added in Python2.5
 try:
     import cProfile as profile
     import pstats
 except ImportError:
     # profile isn't bundled by default in Debian for license reasons
     try:
         import profile,pstats
     except ImportError:
         profile = pstats = None
 
 # Homebrewed
 import IPython
 from IPython import Debugger, OInspect, wildcard
 from IPython.FakeModule import FakeModule
 from IPython.Itpl import Itpl, itpl, printpl,itplns
 from IPython.PyColorize import Parser
 from IPython.ipstruct import Struct
 from IPython.macro import Macro
 from IPython.genutils import *
 from IPython import platutils
 import IPython.generics
 import IPython.ipapi
 from IPython.ipapi import UsageError
 #***************************************************************************
 # Utility functions
 def on_off(tag):
     """Return an ON/OFF string for a 1/0 input. Simple utility function."""
     return ['OFF','ON'][tag]
 
 class Bunch: pass
 
 def compress_dhist(dh):
     head, tail = dh[:-10], dh[-10:]
 
     newhead = []
     done = Set()
     for h in head:
         if h in done:
             continue
         newhead.append(h)
         done.add(h)
 
     return newhead + tail        
         
 
 #***************************************************************************
 # Main class implementing Magic functionality
 class Magic:
     """Magic functions for InteractiveShell.
 
     Shell functions which can be reached as %function_name. All magic
     functions should accept a string, which they can parse for their own
     needs. This can make some functions easier to type, eg `%cd ../`
     vs. `%cd("../")`
 
     ALL definitions MUST begin with the prefix magic_. The user won't need it
     at the command line, but it is is needed in the definition. """
 
     # class globals
     auto_status = ['Automagic is OFF, % prefix IS needed for magic functions.',
                    'Automagic is ON, % prefix NOT needed for magic functions.']
 
     #......................................................................
     # some utility functions
 
     def __init__(self,shell):
         
         self.options_table = {}
         if profile is None:
             self.magic_prun = self.profile_missing_notice
         self.shell = shell
 
         # namespace for holding state we may need
         self._magic_state = Bunch()
 
     def profile_missing_notice(self, *args, **kwargs):
         error("""\
-The profile module could not be found.  If you are a Debian user,
-it has been removed from the standard Debian package because of its non-free
-license. To use profiling, please install"python2.3-profiler" from non-free.""")
+The profile module could not be found. It has been removed from the standard
+python packages because of its non-free license. To use profiling, install the
+python-profiler package from non-free.""")
 
     def default_option(self,fn,optstr):
         """Make an entry in the options_table for fn, with value optstr"""
 
         if fn not in self.lsmagic():
             error("%s is not a magic function" % fn)
         self.options_table[fn] = optstr
 
     def lsmagic(self):
         """Return a list of currently available magic functions.
 
         Gives a list of the bare names after mangling (['ls','cd', ...], not
         ['magic_ls','magic_cd',...]"""
 
         # FIXME. This needs a cleanup, in the way the magics list is built.
         
         # magics in class definition
         class_magic = lambda fn: fn.startswith('magic_') and \
                       callable(Magic.__dict__[fn])
         # in instance namespace (run-time user additions)
         inst_magic =  lambda fn: fn.startswith('magic_') and \
                      callable(self.__dict__[fn])
         # and bound magics by user (so they can access self):
         inst_bound_magic =  lambda fn: fn.startswith('magic_') and \
                            callable(self.__class__.__dict__[fn])
         magics = filter(class_magic,Magic.__dict__.keys()) + \
                  filter(inst_magic,self.__dict__.keys()) + \
                  filter(inst_bound_magic,self.__class__.__dict__.keys())
         out = []
-        for fn in magics:
+        for fn in Set(magics):
             out.append(fn.replace('magic_','',1))
         out.sort()
         return out
     
     def extract_input_slices(self,slices,raw=False):
         """Return as a string a set of input history slices.
 
         Inputs:
 
           - slices: the set of slices is given as a list of strings (like
           ['1','4:8','9'], since this function is for use by magic functions
           which get their arguments as strings.
 
         Optional inputs:
 
           - raw(False): by default, the processed input is used.  If this is
           true, the raw input history is used instead.
 
         Note that slices can be called with two notations:
 
         N:M -> standard python form, means including items N...(M-1).
 
         N-M -> include items N..M (closed endpoint)."""
 
         if raw:
             hist = self.shell.input_hist_raw
         else:
             hist = self.shell.input_hist
 
         cmds = []
         for chunk in slices:
             if ':' in chunk:
                 ini,fin = map(int,chunk.split(':'))
             elif '-' in chunk:
                 ini,fin = map(int,chunk.split('-'))
                 fin += 1
             else:
                 ini = int(chunk)
                 fin = ini+1
             cmds.append(hist[ini:fin])
         return cmds
         
     def _ofind(self, oname, namespaces=None):
         """Find an object in the available namespaces.
 
         self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
 
         Has special code to detect magic functions.
         """
         
         oname = oname.strip()
 
         alias_ns = None
         if namespaces is None:
             # Namespaces to search in:
             # Put them in a list. The order is important so that we
             # find things in the same order that Python finds them.
             namespaces = [ ('Interactive', self.shell.user_ns),
                            ('IPython internal', self.shell.internal_ns),
                            ('Python builtin', __builtin__.__dict__),
                            ('Alias', self.shell.alias_table),
                            ]
             alias_ns = self.shell.alias_table
 
         # initialize results to 'null'
         found = 0; obj = None;  ospace = None;  ds = None;
         ismagic = 0; isalias = 0; parent = None
 
         # Look for the given name by splitting it in parts.  If the head is
         # found, then we look for all the remaining parts as members, and only
         # declare success if we can find them all.
         oname_parts = oname.split('.')
         oname_head, oname_rest = oname_parts[0],oname_parts[1:]
         for nsname,ns in namespaces:
             try:
                 obj = ns[oname_head]
             except KeyError:
                 continue
             else:
                 #print 'oname_rest:', oname_rest  # dbg
                 for part in oname_rest:
                     try:
                         parent = obj
                         obj = getattr(obj,part)
                     except:
                         # Blanket except b/c some badly implemented objects
                         # allow __getattr__ to raise exceptions other than
                         # AttributeError, which then crashes IPython.
                         break
                 else:
                     # If we finish the for loop (no break), we got all members
                     found = 1
                     ospace = nsname
                     if ns == alias_ns:
                         isalias = 1
                     break  # namespace loop
 
         # Try to see if it's magic
         if not found:
             if oname.startswith(self.shell.ESC_MAGIC):
                 oname = oname[1:]
             obj = getattr(self,'magic_'+oname,None)
             if obj is not None:
                 found = 1
                 ospace = 'IPython internal'
                 ismagic = 1
 
         # Last try: special-case some literals like '', [], {}, etc:
         if not found and oname_head in ["''",'""','[]','{}','()']:
             obj = eval(oname_head)
             found = 1
             ospace = 'Interactive'
             
         return {'found':found, 'obj':obj, 'namespace':ospace,
                 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
     
     def arg_err(self,func):
         """Print docstring if incorrect arguments were passed"""
         print 'Error in arguments:'
         print OInspect.getdoc(func)
 
     def format_latex(self,strng):
         """Format a string for latex inclusion."""
 
         # Characters that need to be escaped for latex:
         escape_re = re.compile(r'(%|_|\$|#|&)',re.MULTILINE)
         # Magic command names as headers:
         cmd_name_re = re.compile(r'^(%s.*?):' % self.shell.ESC_MAGIC,
                                  re.MULTILINE)
         # Magic commands 
         cmd_re = re.compile(r'(?P<cmd>%s.+?\b)(?!\}\}:)' % self.shell.ESC_MAGIC,
                             re.MULTILINE)
         # Paragraph continue
         par_re = re.compile(r'\\$',re.MULTILINE)
 
         # The "\n" symbol
         newline_re = re.compile(r'\\n')
 
         # Now build the string for output:
         #strng = cmd_name_re.sub(r'\n\\texttt{\\textsl{\\large \1}}:',strng)
         strng = cmd_name_re.sub(r'\n\\bigskip\n\\texttt{\\textbf{ \1}}:',
                                 strng)
         strng = cmd_re.sub(r'\\texttt{\g<cmd>}',strng)
         strng = par_re.sub(r'\\\\',strng)
         strng = escape_re.sub(r'\\\1',strng)
         strng = newline_re.sub(r'\\textbackslash{}n',strng)
         return strng
 
     def format_screen(self,strng):
         """Format a string for screen printing.
 
         This removes some latex-type format codes."""
         # Paragraph continue
         par_re = re.compile(r'\\$',re.MULTILINE)
         strng = par_re.sub('',strng)
         return strng
 
     def parse_options(self,arg_str,opt_str,*long_opts,**kw):
         """Parse options passed to an argument string.
 
         The interface is similar to that of getopt(), but it returns back a
         Struct with the options as keys and the stripped argument string still
         as a string.
 
         arg_str is quoted as a true sys.argv vector by using shlex.split.
         This allows us to easily expand variables, glob files, quote
         arguments, etc.
 
         Options:
           -mode: default 'string'. If given as 'list', the argument string is
           returned as a list (split on whitespace) instead of a string.
 
           -list_all: put all option values in lists. Normally only options
           appearing more than once are put in a list.
 
           -posix (True): whether to split the input line in POSIX mode or not,
           as per the conventions outlined in the shlex module from the
           standard library."""
  
         # inject default options at the beginning of the input line
         caller = sys._getframe(1).f_code.co_name.replace('magic_','')
         arg_str = '%s %s' % (self.options_table.get(caller,''),arg_str)
         
         mode = kw.get('mode','string')
         if mode not in ['string','list']:
             raise ValueError,'incorrect mode given: %s' % mode
         # Get options
         list_all = kw.get('list_all',0)
         posix = kw.get('posix',True)
 
         # Check if we have more than one argument to warrant extra processing:
         odict = {}  # Dictionary with options
         args = arg_str.split()
         if len(args) >= 1:
             # If the list of inputs only has 0 or 1 thing in it, there's no
             # need to look for options
             argv = arg_split(arg_str,posix)
             # Do regular option processing
             try:
                 opts,args = getopt(argv,opt_str,*long_opts)
             except GetoptError,e:
                 raise UsageError('%s ( allowed: "%s" %s)' % (e.msg,opt_str, 
                                         " ".join(long_opts)))
             for o,a in opts:
                 if o.startswith('--'):
                     o = o[2:]
                 else:
                     o = o[1:]
                 try:
                     odict[o].append(a)
                 except AttributeError:
                     odict[o] = [odict[o],a]
                 except KeyError:
                     if list_all:
                         odict[o] = [a]
                     else:
                         odict[o] = a
 
         # Prepare opts,args for return
         opts = Struct(odict)
         if mode == 'string':
             args = ' '.join(args)
 
         return opts,args
     
     #......................................................................
     # And now the actual magic functions
 
     # Functions for IPython shell work (vars,funcs, config, etc)
     def magic_lsmagic(self, parameter_s = ''):
         """List currently available magic functions."""
         mesc = self.shell.ESC_MAGIC
         print 'Available magic functions:\n'+mesc+\
               ('  '+mesc).join(self.lsmagic())
         print '\n' + Magic.auto_status[self.shell.rc.automagic]
         return None
         
     def magic_magic(self, parameter_s = ''):
-        """Print information about the magic function system."""
+        """Print information about the magic function system.
+        
+        Supported formats: -latex, -brief, -rest        
+        """
 
         mode = ''
         try:
             if parameter_s.split()[0] == '-latex':
                 mode = 'latex'
             if parameter_s.split()[0] == '-brief':
                 mode = 'brief'
+            if parameter_s.split()[0] == '-rest':
+                mode = 'rest'
+                rest_docs = []
         except:
             pass
 
         magic_docs = []
         for fname in self.lsmagic():
             mname = 'magic_' + fname
             for space in (Magic,self,self.__class__):
                 try:
                     fn = space.__dict__[mname]
                 except KeyError:
                     pass
                 else:
                     break
             if mode == 'brief':
                 # only first line
-                fndoc = fn.__doc__.split('\n',1)[0]
+                if fn.__doc__:                    
+                    fndoc = fn.__doc__.split('\n',1)[0]
+                else:
+                    fndoc = 'No documentation'
+            else:
+                fndoc = fn.__doc__.rstrip()
+                
+            if mode == 'rest':
+                rest_docs.append('**%s%s**::\n\n\t%s\n\n' %(self.shell.ESC_MAGIC,
+                                                    fname,fndoc))
+                
             else:
-                fndoc = fn.__doc__
+                magic_docs.append('%s%s:\n\t%s\n' %(self.shell.ESC_MAGIC,
+                                                    fname,fndoc))
                 
-            magic_docs.append('%s%s:\n\t%s\n' %(self.shell.ESC_MAGIC,
-                                                fname,fndoc))
         magic_docs = ''.join(magic_docs)
 
+        if mode == 'rest':
+            return "".join(rest_docs)
+        
         if mode == 'latex':
             print self.format_latex(magic_docs)
             return
         else:
             magic_docs = self.format_screen(magic_docs)
         if mode == 'brief':
             return magic_docs
         
         outmsg = """
 IPython's 'magic' functions
 ===========================
 
 The magic function system provides a series of functions which allow you to
 control the behavior of IPython itself, plus a lot of system-type
 features. All these functions are prefixed with a % character, but parameters
 are given without parentheses or quotes.
 
 NOTE: If you have 'automagic' enabled (via the command line option or with the
 %automagic function), you don't need to type in the % explicitly.  By default,
 IPython ships with automagic on, so you should only rarely need the % escape.
 
 Example: typing '%cd mydir' (without the quotes) changes you working directory
 to 'mydir', if it exists.
 
 You can define your own magic functions to extend the system. See the supplied
 ipythonrc and example-magic.py files for details (in your ipython
 configuration directory, typically $HOME/.ipython/).
 
 You can also define your own aliased names for magic functions. In your
 ipythonrc file, placing a line like:
 
   execute __IPYTHON__.magic_pf = __IPYTHON__.magic_profile
 
 will define %pf as a new name for %profile.
 
 You can also call magics in code using the ipmagic() function, which IPython
 automatically adds to the builtin namespace.  Type 'ipmagic?' for details.
 
 For a list of the available magic functions, use %lsmagic. For a description
 of any of them, type %magic_name?, e.g. '%cd?'.
 
 Currently the magic system has the following functions:\n"""
 
         mesc = self.shell.ESC_MAGIC
         outmsg = ("%s\n%s\n\nSummary of magic functions (from %slsmagic):"
                   "\n\n%s%s\n\n%s" % (outmsg,
                                      magic_docs,mesc,mesc,
                                      ('  '+mesc).join(self.lsmagic()),
                                      Magic.auto_status[self.shell.rc.automagic] ) )
 
         page(outmsg,screen_lines=self.shell.rc.screen_length)
   
 
     def magic_autoindent(self, parameter_s = ''):
         """Toggle autoindent on/off (if available)."""
 
         self.shell.set_autoindent()
         print "Automatic indentation is:",['OFF','ON'][self.shell.autoindent]
 
 
     def magic_automagic(self, parameter_s = ''):
         """Make magic functions callable without having to type the initial %.
 
         Without argumentsl toggles on/off (when off, you must call it as
         %automagic, of course).  With arguments it sets the value, and you can
         use any of (case insensitive):
 
          - on,1,True: to activate
 
          - off,0,False: to deactivate.
 
         Note that magic functions have lowest priority, so if there's a
         variable whose name collides with that of a magic fn, automagic won't
         work for that function (you get the variable instead). However, if you
         delete the variable (del var), the previously shadowed magic function
         becomes visible to automagic again."""
 
         rc = self.shell.rc
         arg = parameter_s.lower()
         if parameter_s in ('on','1','true'):
             rc.automagic = True
         elif parameter_s in ('off','0','false'):
             rc.automagic = False
         else:
             rc.automagic = not rc.automagic
         print '\n' + Magic.auto_status[rc.automagic]
 
 
     def magic_autocall(self, parameter_s = ''):
         """Make functions callable without having to type parentheses.
 
         Usage:
 
            %autocall [mode]
 
         The mode can be one of: 0->Off, 1->Smart, 2->Full.  If not given, the
         value is toggled on and off (remembering the previous state).
 
         In more detail, these values mean:
 
         0 -> fully disabled
 
         1 -> active, but do not apply if there are no arguments on the line.
 
         In this mode, you get:
 
         In [1]: callable
         Out[1]: <built-in function callable>
 
         In [2]: callable 'hello'
         ------> callable('hello')
         Out[2]: False
 
         2 -> Active always.  Even if no arguments are present, the callable
         object is called:
 
         In [4]: callable
         ------> callable()
 
         Note that even with autocall off, you can still use '/' at the start of
         a line to treat the first argument on the command line as a function
         and add parentheses to it:
 
         In [8]: /str 43
         ------> str(43)
         Out[8]: '43'
         """
 
         rc = self.shell.rc
 
         if parameter_s:
             arg = int(parameter_s)
         else:
             arg = 'toggle'
 
         if not arg in (0,1,2,'toggle'):
             error('Valid modes: (0->Off, 1->Smart, 2->Full')
             return
 
         if arg in (0,1,2):
             rc.autocall = arg
         else: # toggle
             if rc.autocall:
                 self._magic_state.autocall_save = rc.autocall
                 rc.autocall = 0
             else:
                 try:
                     rc.autocall = self._magic_state.autocall_save
                 except AttributeError:
                     rc.autocall = self._magic_state.autocall_save = 1
 
         print "Automatic calling is:",['OFF','Smart','Full'][rc.autocall]
 
     def magic_system_verbose(self, parameter_s = ''):
         """Set verbose printing of system calls.
 
         If called without an argument, act as a toggle"""
 
         if parameter_s:
             val = bool(eval(parameter_s))
         else:
             val = None
             
         self.shell.rc_set_toggle('system_verbose',val)
         print "System verbose printing is:",\
               ['OFF','ON'][self.shell.rc.system_verbose]
 
 
     def magic_page(self, parameter_s=''):
         """Pretty print the object and display it through a pager.
         
         %page [options] OBJECT
 
         If no object is given, use _ (last output).
         
         Options:
 
           -r: page str(object), don't pretty-print it."""
 
         # After a function contributed by Olivier Aubert, slightly modified.
 
         # Process options/args
         opts,args = self.parse_options(parameter_s,'r')
         raw = 'r' in opts
 
         oname = args and args or '_'
         info = self._ofind(oname)
         if info['found']:
             txt = (raw and str or pformat)( info['obj'] )
             page(txt)
         else:
             print 'Object `%s` not found' % oname
 
     def magic_profile(self, parameter_s=''):
         """Print your currently active IPyhton profile."""
         if self.shell.rc.profile:
             printpl('Current IPython profile: $self.shell.rc.profile.')
         else:
             print 'No profile active.'
 
     def magic_pinfo(self, parameter_s='', namespaces=None):
         """Provide detailed information about an object.
     
         '%pinfo object' is just a synonym for object? or ?object."""
     
         #print 'pinfo par: <%s>' % parameter_s  # dbg
     
         
         # detail_level: 0 -> obj? , 1 -> obj??
         detail_level = 0
         # We need to detect if we got called as 'pinfo pinfo foo', which can
         # happen if the user types 'pinfo foo?' at the cmd line.
         pinfo,qmark1,oname,qmark2 = \
                re.match('(pinfo )?(\?*)(.*?)(\??$)',parameter_s).groups()
         if pinfo or qmark1 or qmark2:
             detail_level = 1
         if "*" in oname:
             self.magic_psearch(oname)
         else:
             self._inspect('pinfo', oname, detail_level=detail_level,
                           namespaces=namespaces)
 
     def magic_pdef(self, parameter_s='', namespaces=None):
         """Print the definition header for any callable object.
 
         If the object is a class, print the constructor information."""
         self._inspect('pdef',parameter_s, namespaces)
 
     def magic_pdoc(self, parameter_s='', namespaces=None):
         """Print the docstring for an object.
 
         If the given object is a class, it will print both the class and the
         constructor docstrings."""
         self._inspect('pdoc',parameter_s, namespaces)
 
     def magic_psource(self, parameter_s='', namespaces=None):
         """Print (or run through pager) the source code for an object."""
         self._inspect('psource',parameter_s, namespaces)
 
     def magic_pfile(self, parameter_s=''):
         """Print (or run through pager) the file where an object is defined.
 
         The file opens at the line where the object definition begins. IPython
         will honor the environment variable PAGER if set, and otherwise will
         do its best to print the file in a convenient form.
 
         If the given argument is not an object currently defined, IPython will
         try to interpret it as a filename (automatically adding a .py extension
         if needed). You can thus use %pfile as a syntax highlighting code
         viewer."""
 
         # first interpret argument as an object name
         out = self._inspect('pfile',parameter_s)
         # if not, try the input as a filename
         if out == 'not found':
             try:
                 filename = get_py_filename(parameter_s)
             except IOError,msg:
                 print msg
                 return
             page(self.shell.inspector.format(file(filename).read()))
 
     def _inspect(self,meth,oname,namespaces=None,**kw):
         """Generic interface to the inspector system.
 
         This function is meant to be called by pdef, pdoc & friends."""
 
         #oname = oname.strip()
         #print '1- oname: <%r>' % oname  # dbg
         try:
             oname = oname.strip().encode('ascii')
             #print '2- oname: <%r>' % oname  # dbg
         except UnicodeEncodeError:
             print 'Python identifiers can only contain ascii characters.'
             return 'not found'
             
         info = Struct(self._ofind(oname, namespaces))
         
         if info.found:
             try:
                 IPython.generics.inspect_object(info.obj)
                 return
             except IPython.ipapi.TryNext:
                 pass
             # Get the docstring of the class property if it exists.
             path = oname.split('.')
             root = '.'.join(path[:-1])
             if info.parent is not None:
                 try:
                     target = getattr(info.parent, '__class__') 
                     # The object belongs to a class instance. 
                     try: 
                         target = getattr(target, path[-1])
                         # The class defines the object. 
                         if isinstance(target, property):
                             oname = root + '.__class__.' + path[-1]
                             info = Struct(self._ofind(oname))
                     except AttributeError: pass
                 except AttributeError: pass
                         
             pmethod = getattr(self.shell.inspector,meth)
             formatter = info.ismagic and self.format_screen or None
             if meth == 'pdoc':
                 pmethod(info.obj,oname,formatter)
             elif meth == 'pinfo':
                 pmethod(info.obj,oname,formatter,info,**kw)
             else:
                 pmethod(info.obj,oname)
         else:
             print 'Object `%s` not found.' % oname
             return 'not found'  # so callers can take other action
         
     def magic_psearch(self, parameter_s=''):
         """Search for object in namespaces by wildcard.
 
         %psearch [options] PATTERN [OBJECT TYPE]
 
         Note: ? can be used as a synonym for %psearch, at the beginning or at
         the end: both a*? and ?a* are equivalent to '%psearch a*'.  Still, the
         rest of the command line must be unchanged (options come first), so
         for example the following forms are equivalent
 
         %psearch -i a* function
         -i a* function?
         ?-i a* function
 
         Arguments:
         
           PATTERN
 
           where PATTERN is a string containing * as a wildcard similar to its
           use in a shell.  The pattern is matched in all namespaces on the
           search path. By default objects starting with a single _ are not
           matched, many IPython generated objects have a single
           underscore. The default is case insensitive matching. Matching is
           also done on the attributes of objects and not only on the objects
           in a module.
 
           [OBJECT TYPE]
 
           Is the name of a python type from the types module. The name is
           given in lowercase without the ending type, ex. StringType is
           written string. By adding a type here only objects matching the
           given type are matched. Using all here makes the pattern match all
           types (this is the default).
 
         Options:
 
           -a: makes the pattern match even objects whose names start with a
           single underscore.  These names are normally ommitted from the
           search.
 
           -i/-c: make the pattern case insensitive/sensitive.  If neither of
           these options is given, the default is read from your ipythonrc
           file.  The option name which sets this value is
           'wildcards_case_sensitive'.  If this option is not specified in your
           ipythonrc file, IPython's internal default is to do a case sensitive
           search.
 
           -e/-s NAMESPACE: exclude/search a given namespace.  The pattern you
           specifiy can be searched in any of the following namespaces:
           'builtin', 'user', 'user_global','internal', 'alias', where
           'builtin' and 'user' are the search defaults.  Note that you should
           not use quotes when specifying namespaces.
 
           'Builtin' contains the python module builtin, 'user' contains all
           user data, 'alias' only contain the shell aliases and no python
           objects, 'internal' contains objects used by IPython.  The
           'user_global' namespace is only used by embedded IPython instances,
           and it contains module-level globals.  You can add namespaces to the
           search with -s or exclude them with -e (these options can be given
           more than once).
     
         Examples:
        
         %psearch a*            -> objects beginning with an a
         %psearch -e builtin a* -> objects NOT in the builtin space starting in a
         %psearch a* function   -> all functions beginning with an a
         %psearch re.e*         -> objects beginning with an e in module re
         %psearch r*.e*         -> objects that start with e in modules starting in r
         %psearch r*.* string   -> all strings in modules beginning with r
 
         Case sensitve search:
        
         %psearch -c a*         list all object beginning with lower case a
 
         Show objects beginning with a single _:
        
         %psearch -a _*         list objects beginning with a single underscore"""
         try:
             parameter_s = parameter_s.encode('ascii')
         except UnicodeEncodeError:
             print 'Python identifiers can only contain ascii characters.'
             return
 
         # default namespaces to be searched
         def_search = ['user','builtin']
 
         # Process options/args
         opts,args = self.parse_options(parameter_s,'cias:e:',list_all=True)
         opt = opts.get
         shell = self.shell
         psearch = shell.inspector.psearch
 
         # select case options
         if opts.has_key('i'):
             ignore_case = True
         elif opts.has_key('c'):
             ignore_case = False
         else:
             ignore_case = not shell.rc.wildcards_case_sensitive
 
         # Build list of namespaces to search from user options
         def_search.extend(opt('s',[]))
         ns_exclude = ns_exclude=opt('e',[])
         ns_search = [nm for nm in def_search if nm not in ns_exclude]
         
         # Call the actual search
         try:
             psearch(args,shell.ns_table,ns_search,
                     show_all=opt('a'),ignore_case=ignore_case)
         except:
             shell.showtraceback()
 
     def magic_who_ls(self, parameter_s=''):
         """Return a sorted list of all interactive variables.
 
         If arguments are given, only variables of types matching these
         arguments are returned."""
 
         user_ns = self.shell.user_ns
         internal_ns = self.shell.internal_ns
         user_config_ns = self.shell.user_config_ns
         out = []
         typelist = parameter_s.split()
 
         for i in user_ns:
             if not (i.startswith('_') or i.startswith('_i')) \
                    and not (i in internal_ns or i in user_config_ns):
                 if typelist:
                     if type(user_ns[i]).__name__ in typelist:
                         out.append(i)
                 else:
                     out.append(i)
         out.sort()
         return out
         
     def magic_who(self, parameter_s=''):
         """Print all interactive variables, with some minimal formatting.
 
         If any arguments are given, only variables whose type matches one of
         these are printed.  For example:
 
           %who function str
 
         will only list functions and strings, excluding all other types of
         variables.  To find the proper type names, simply use type(var) at a
         command line to see how python prints type names.  For example:
 
           In [1]: type('hello')\\
           Out[1]: <type 'str'>
 
         indicates that the type name for strings is 'str'.
 
         %who always excludes executed names loaded through your configuration
         file and things which are internal to IPython.
 
         This is deliberate, as typically you may load many modules and the
         purpose of %who is to show you only what you've manually defined."""
 
         varlist = self.magic_who_ls(parameter_s)
         if not varlist:
             if parameter_s:
                 print 'No variables match your requested type.'
             else:
                 print 'Interactive namespace is empty.'
             return
 
         # if we have variables, move on...
         count = 0
         for i in varlist:
             print i+'\t',
             count += 1
             if count > 8:
                 count = 0
                 print
         print
 
     def magic_whos(self, parameter_s=''):
         """Like %who, but gives some extra information about each variable.
 
         The same type filtering of %who can be applied here.
 
         For all variables, the type is printed. Additionally it prints:
         
           - For {},[],(): their length.
 
           - For numpy and Numeric arrays, a summary with shape, number of
           elements, typecode and size in memory.
 
           - Everything else: a string representation, snipping their middle if
           too long."""
         
         varnames = self.magic_who_ls(parameter_s)
         if not varnames:
             if parameter_s:
                 print 'No variables match your requested type.'
             else:
                 print 'Interactive namespace is empty.'
             return
 
         # if we have variables, move on...
 
         # for these types, show len() instead of data:
         seq_types = [types.DictType,types.ListType,types.TupleType]
 
         # for numpy/Numeric arrays, display summary info
         try:
             import numpy
         except ImportError:
             ndarray_type = None
         else:
             ndarray_type = numpy.ndarray.__name__
         try:
             import Numeric
         except ImportError:
             array_type = None
         else:
             array_type = Numeric.ArrayType.__name__
 
         # Find all variable names and types so we can figure out column sizes
         def get_vars(i):
             return self.shell.user_ns[i]
         
         # some types are well known and can be shorter
         abbrevs = {'IPython.macro.Macro' : 'Macro'}
         def type_name(v):
             tn = type(v).__name__
             return abbrevs.get(tn,tn)
             
         varlist = map(get_vars,varnames)
 
         typelist = []
         for vv in varlist:
             tt = type_name(vv)
 
             if tt=='instance':
                 typelist.append( abbrevs.get(str(vv.__class__),
                                              str(vv.__class__)))
             else:
                 typelist.append(tt)
 
         # column labels and # of spaces as separator
         varlabel = 'Variable'
         typelabel = 'Type'
         datalabel = 'Data/Info'
         colsep = 3
         # variable format strings
         vformat    = "$vname.ljust(varwidth)$vtype.ljust(typewidth)"
         vfmt_short = '$vstr[:25]<...>$vstr[-25:]'
         aformat    = "%s: %s elems, type `%s`, %s bytes"
         # find the size of the columns to format the output nicely
         varwidth = max(max(map(len,varnames)), len(varlabel)) + colsep
         typewidth = max(max(map(len,typelist)), len(typelabel)) + colsep
         # table header
         print varlabel.ljust(varwidth) + typelabel.ljust(typewidth) + \
               ' '+datalabel+'\n' + '-'*(varwidth+typewidth+len(datalabel)+1)
         # and the table itself
         kb = 1024
         Mb = 1048576  # kb**2
         for vname,var,vtype in zip(varnames,varlist,typelist):
             print itpl(vformat),
             if vtype in seq_types:
                 print len(var)
             elif vtype in [array_type,ndarray_type]:
                 vshape = str(var.shape).replace(',','').replace(' ','x')[1:-1]
                 if vtype==ndarray_type:
                     # numpy
                     vsize  = var.size
                     vbytes = vsize*var.itemsize
                     vdtype = var.dtype
                 else:
                     # Numeric
                     vsize  = Numeric.size(var)
                     vbytes = vsize*var.itemsize()
                     vdtype = var.typecode()
                     
                 if vbytes < 100000:
                     print aformat % (vshape,vsize,vdtype,vbytes)
                 else:
                     print aformat % (vshape,vsize,vdtype,vbytes),
                     if vbytes < Mb:
                         print '(%s kb)' % (vbytes/kb,)
                     else:
                         print '(%s Mb)' % (vbytes/Mb,)
             else:
                 try:
                     vstr = str(var)
                 except UnicodeEncodeError:
                     vstr = unicode(var).encode(sys.getdefaultencoding(),
                                                'backslashreplace')
                 vstr = vstr.replace('\n','\\n')
                 if len(vstr) < 50:
                     print vstr
                 else:
                     printpl(vfmt_short)
                 
     def magic_reset(self, parameter_s=''):
         """Resets the namespace by removing all names defined by the user.
 
         Input/Output history are left around in case you need them."""
 
         ans = self.shell.ask_yes_no(
           "Once deleted, variables cannot be recovered. Proceed (y/[n])? ")
         if not ans:
             print 'Nothing done.'
             return
         user_ns = self.shell.user_ns
         for i in self.magic_who_ls():
             del(user_ns[i])
             
         # Also flush the private list of module references kept for script
         # execution protection
         self.shell._user_main_modules[:] = []
 
     def magic_logstart(self,parameter_s=''):
         """Start logging anywhere in a session.
 
         %logstart [-o|-r|-t] [log_name [log_mode]]
 
         If no name is given, it defaults to a file named 'ipython_log.py' in your
         current directory, in 'rotate' mode (see below).
 
         '%logstart name' saves to file 'name' in 'backup' mode.  It saves your
         history up to that point and then continues logging.
 
         %logstart takes a second optional parameter: logging mode. This can be one
         of (note that the modes are given unquoted):\\
           append: well, that says it.\\
           backup: rename (if exists) to name~ and start name.\\
           global: single logfile in your home dir, appended to.\\
           over  : overwrite existing log.\\
           rotate: create rotating logs name.1~, name.2~, etc.
 
         Options:
 
           -o: log also IPython's output.  In this mode, all commands which
           generate an Out[NN] prompt are recorded to the logfile, right after
           their corresponding input line.  The output lines are always
           prepended with a '#[Out]# ' marker, so that the log remains valid
           Python code.
 
           Since this marker is always the same, filtering only the output from
           a log is very easy, using for example a simple awk call:
 
             awk -F'#\\[Out\\]# ' '{if($2) {print $2}}' ipython_log.py
 
           -r: log 'raw' input.  Normally, IPython's logs contain the processed
           input, so that user lines are logged in their final form, converted
           into valid Python.  For example, %Exit is logged as
           '_ip.magic("Exit").  If the -r flag is given, all input is logged
           exactly as typed, with no transformations applied.
 
           -t: put timestamps before each input line logged (these are put in
           comments)."""
         
         opts,par = self.parse_options(parameter_s,'ort')
         log_output = 'o' in opts
         log_raw_input = 'r' in opts
         timestamp = 't' in opts
 
         rc = self.shell.rc
         logger = self.shell.logger
 
         # if no args are given, the defaults set in the logger constructor by
         # ipytohn remain valid
         if par:
             try:
                 logfname,logmode = par.split()
             except:
                 logfname = par
                 logmode = 'backup'
         else:
             logfname = logger.logfname
             logmode = logger.logmode
         # put logfname into rc struct as if it had been called on the command
         # line, so it ends up saved in the log header Save it in case we need
         # to restore it...
         old_logfile = rc.opts.get('logfile','')  
         if logfname:
             logfname = os.path.expanduser(logfname)
         rc.opts.logfile = logfname
         loghead = self.shell.loghead_tpl % (rc.opts,rc.args)
         try:
             started  = logger.logstart(logfname,loghead,logmode,
                                        log_output,timestamp,log_raw_input)
         except:
             rc.opts.logfile = old_logfile
             warn("Couldn't start log: %s" % sys.exc_info()[1])
         else:
             # log input history up to this point, optionally interleaving
             # output if requested
 
             if timestamp:
                 # disable timestamping for the previous history, since we've
                 # lost those already (no time machine here).
                 logger.timestamp = False
 
             if log_raw_input:
                 input_hist = self.shell.input_hist_raw
             else:
                 input_hist = self.shell.input_hist
                 
             if log_output:
                 log_write = logger.log_write
                 output_hist = self.shell.output_hist
                 for n in range(1,len(input_hist)-1):
                     log_write(input_hist[n].rstrip())
                     if n in output_hist:
                         log_write(repr(output_hist[n]),'output')
             else:
                 logger.log_write(input_hist[1:])
             if timestamp:
                 # re-enable timestamping
                 logger.timestamp = True
                 
             print ('Activating auto-logging. '
                    'Current session state plus future input saved.')
             logger.logstate()
 
     def magic_logstop(self,parameter_s=''):
         """Fully stop logging and close log file.
 
         In order to start logging again, a new %logstart call needs to be made,
         possibly (though not necessarily) with a new filename, mode and other
         options."""
         self.logger.logstop()
 
     def magic_logoff(self,parameter_s=''):
         """Temporarily stop logging.
 
         You must have previously started logging."""
         self.shell.logger.switch_log(0)
         
     def magic_logon(self,parameter_s=''):
         """Restart logging.
 
         This function is for restarting logging which you've temporarily
         stopped with %logoff. For starting logging for the first time, you
         must use the %logstart function, which allows you to specify an
         optional log filename."""
         
         self.shell.logger.switch_log(1)
     
     def magic_logstate(self,parameter_s=''):
         """Print the status of the logging system."""
 
         self.shell.logger.logstate()
         
     def magic_pdb(self, parameter_s=''):
         """Control the automatic calling of the pdb interactive debugger.
 
         Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
         argument it works as a toggle.
 
         When an exception is triggered, IPython can optionally call the
         interactive pdb debugger after the traceback printout. %pdb toggles
         this feature on and off.
 
         The initial state of this feature is set in your ipythonrc
         configuration file (the variable is called 'pdb').
 
         If you want to just activate the debugger AFTER an exception has fired,
         without having to type '%pdb on' and rerunning your code, you can use
         the %debug magic."""
 
         par = parameter_s.strip().lower()
 
         if par:
             try:
                 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
             except KeyError:
                 print ('Incorrect argument. Use on/1, off/0, '
                        'or nothing for a toggle.')
                 return
         else:
             # toggle
             new_pdb = not self.shell.call_pdb
 
         # set on the shell
         self.shell.call_pdb = new_pdb
         print 'Automatic pdb calling has been turned',on_off(new_pdb)
 
     def magic_debug(self, parameter_s=''):
         """Activate the interactive debugger in post-mortem mode.
 
         If an exception has just occurred, this lets you inspect its stack
         frames interactively.  Note that this will always work only on the last
         traceback that occurred, so you must call this quickly after an
         exception that you wish to inspect has fired, because if another one
         occurs, it clobbers the previous one.
 
         If you want IPython to automatically do this on every exception, see
         the %pdb magic for more details.
         """
         
         self.shell.debugger(force=True)
 
     def magic_prun(self, parameter_s ='',user_mode=1,
                    opts=None,arg_lst=None,prog_ns=None):
 
         """Run a statement through the python code profiler.
 
         Usage:\\
           %prun [options] statement
 
         The given statement (which doesn't require quote marks) is run via the
         python profiler in a manner similar to the profile.run() function.
         Namespaces are internally managed to work correctly; profile.run
         cannot be used in IPython because it makes certain assumptions about
         namespaces which do not hold under IPython.
 
         Options:
 
         -l <limit>: you can place restrictions on what or how much of the
         profile gets printed. The limit value can be:
 
           * A string: only information for function names containing this string
           is printed.
 
           * An integer: only these many lines are printed.
 
           * A float (between 0 and 1): this fraction of the report is printed
           (for example, use a limit of 0.4 to see the topmost 40% only).
 
         You can combine several limits with repeated use of the option. For
         example, '-l __init__ -l 5' will print only the topmost 5 lines of
         information about class constructors.
 
         -r: return the pstats.Stats object generated by the profiling. This
         object has all the information about the profile in it, and you can
         later use it for further analysis or in other functions.
 
        -s <key>: sort profile by given key. You can provide more than one key
         by using the option several times: '-s key1 -s key2 -s key3...'. The
         default sorting key is 'time'.
 
         The following is copied verbatim from the profile documentation
         referenced below:
 
         When more than one key is provided, additional keys are used as
         secondary criteria when the there is equality in all keys selected
         before them.
         
         Abbreviations can be used for any key names, as long as the
         abbreviation is unambiguous.  The following are the keys currently
         defined:
 
                 Valid Arg       Meaning\\
                   "calls"      call count\\
                   "cumulative" cumulative time\\
                   "file"       file name\\
                   "module"     file name\\
                   "pcalls"     primitive call count\\
                   "line"       line number\\
                   "name"       function name\\
                   "nfl"        name/file/line\\
                   "stdname"    standard name\\
                   "time"       internal time
 
         Note that all sorts on statistics are in descending order (placing
         most time consuming items first), where as name, file, and line number
         searches are in ascending order (i.e., alphabetical). The subtle
         distinction between "nfl" and "stdname" is that the standard name is a
         sort of the name as printed, which means that the embedded line
         numbers get compared in an odd way.  For example, lines 3, 20, and 40
         would (if the file names were the same) appear in the string order
         "20" "3" and "40".  In contrast, "nfl" does a numeric compare of the
         line numbers.  In fact, sort_stats("nfl") is the same as
         sort_stats("name", "file", "line").
 
         -T <filename>: save profile results as shown on screen to a text
         file. The profile is still shown on screen.
 
         -D <filename>: save (via dump_stats) profile statistics to given
         filename. This data is in a format understod by the pstats module, and
         is generated by a call to the dump_stats() method of profile
         objects. The profile is still shown on screen.
 
         If you want to run complete programs under the profiler's control, use
         '%run -p [prof_opts] filename.py [args to program]' where prof_opts
         contains profiler specific options as described here.
         
         You can read the complete documentation for the profile module with:\\
           In [1]: import profile; profile.help() """
 
         opts_def = Struct(D=[''],l=[],s=['time'],T=[''])
         # protect user quote marks
         parameter_s = parameter_s.replace('"',r'\"').replace("'",r"\'")
         
         if user_mode:  # regular user call
             opts,arg_str = self.parse_options(parameter_s,'D:l:rs:T:',
                                               list_all=1)
             namespace = self.shell.user_ns
         else:  # called to run a program by %run -p
             try:
                 filename = get_py_filename(arg_lst[0])
             except IOError,msg:
                 error(msg)
                 return
 
             arg_str = 'execfile(filename,prog_ns)'
             namespace = locals()
 
         opts.merge(opts_def)
         
         prof = profile.Profile()
         try:
             prof = prof.runctx(arg_str,namespace,namespace)
             sys_exit = ''
         except SystemExit:
             sys_exit = """*** SystemExit exception caught in code being profiled."""
 
         stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
 
         lims = opts.l
         if lims:
             lims = []  # rebuild lims with ints/floats/strings
             for lim in opts.l:
                 try:
                     lims.append(int(lim))
                 except ValueError:
                     try:
                         lims.append(float(lim))
                     except ValueError:
                         lims.append(lim)
                     
         # Trap output.
         stdout_trap = StringIO()
 
         if hasattr(stats,'stream'):
             # In newer versions of python, the stats object has a 'stream'
             # attribute to write into.
             stats.stream = stdout_trap
             stats.print_stats(*lims)
         else:
             # For older versions, we manually redirect stdout during printing
             sys_stdout = sys.stdout
             try:
                 sys.stdout = stdout_trap
                 stats.print_stats(*lims)
             finally:
                 sys.stdout = sys_stdout
                 
         output = stdout_trap.getvalue()
         output = output.rstrip()
 
         page(output,screen_lines=self.shell.rc.screen_length)
         print sys_exit,
 
         dump_file = opts.D[0]
         text_file = opts.T[0]
         if dump_file:
             prof.dump_stats(dump_file)
             print '\n*** Profile stats marshalled to file',\
                   `dump_file`+'.',sys_exit
         if text_file:
             pfile = file(text_file,'w')
             pfile.write(output)
             pfile.close()
             print '\n*** Profile printout saved to text file',\
                   `text_file`+'.',sys_exit
 
         if opts.has_key('r'):
             return stats
         else:
             return None
 
     def magic_run(self, parameter_s ='',runner=None):
         """Run the named file inside IPython as a program.
 
         Usage:\\
           %run [-n -i -t [-N<N>] -d [-b<N>] -p [profile options]] file [args]
         
         Parameters after the filename are passed as command-line arguments to
         the program (put in sys.argv). Then, control returns to IPython's
         prompt.
 
         This is similar to running at a system prompt:\\
           $ python file args\\
         but with the advantage of giving you IPython's tracebacks, and of
         loading all variables into your interactive namespace for further use
         (unless -p is used, see below).
 
         The file is executed in a namespace initially consisting only of
         __name__=='__main__' and sys.argv constructed as indicated. It thus
         sees its environment as if it were being run as a stand-alone program
         (except for sharing global objects such as previously imported
         modules). But after execution, the IPython interactive namespace gets
         updated with all variables defined in the program (except for __name__
         and sys.argv). This allows for very convenient loading of code for
         interactive work, while giving each program a 'clean sheet' to run in.
 
         Options:
         
         -n: __name__ is NOT set to '__main__', but to the running file's name
         without extension (as python does under import).  This allows running
         scripts and reloading the definitions in them without calling code
         protected by an ' if __name__ == "__main__" ' clause.
 
         -i: run the file in IPython's namespace instead of an empty one. This
         is useful if you are experimenting with code written in a text editor
         which depends on variables defined interactively.
 
         -e: ignore sys.exit() calls or SystemExit exceptions in the script
         being run.  This is particularly useful if IPython is being used to
         run unittests, which always exit with a sys.exit() call.  In such
         cases you are interested in the output of the test results, not in
         seeing a traceback of the unittest module.
 
         -t: print timing information at the end of the run.  IPython will give
         you an estimated CPU time consumption for your script, which under
         Unix uses the resource module to avoid the wraparound problems of
         time.clock().  Under Unix, an estimate of time spent on system tasks
         is also given (for Windows platforms this is reported as 0.0).
 
         If -t is given, an additional -N<N> option can be given, where <N>
         must be an integer indicating how many times you want the script to
         run.  The final timing report will include total and per run results.
 
         For example (testing the script uniq_stable.py):
 
             In [1]: run -t uniq_stable
 
             IPython CPU timings (estimated):\\
               User  :    0.19597 s.\\
               System:        0.0 s.\\
 
             In [2]: run -t -N5 uniq_stable
 
             IPython CPU timings (estimated):\\
             Total runs performed: 5\\
               Times :      Total       Per run\\
               User  :   0.910862 s,  0.1821724 s.\\
               System:        0.0 s,        0.0 s.
 
         -d: run your program under the control of pdb, the Python debugger.
         This allows you to execute your program step by step, watch variables,
         etc.  Internally, what IPython does is similar to calling:
         
           pdb.run('execfile("YOURFILENAME")')
 
         with a breakpoint set on line 1 of your file.  You can change the line
         number for this automatic breakpoint to be <N> by using the -bN option
         (where N must be an integer).  For example:
 
           %run -d -b40 myscript
 
         will set the first breakpoint at line 40 in myscript.py.  Note that
         the first breakpoint must be set on a line which actually does
         something (not a comment or docstring) for it to stop execution.
 
         When the pdb debugger starts, you will see a (Pdb) prompt.  You must
         first enter 'c' (without qoutes) to start execution up to the first
         breakpoint.
 
         Entering 'help' gives information about the use of the debugger.  You
         can easily see pdb's full documentation with "import pdb;pdb.help()"
         at a prompt.
 
         -p: run program under the control of the Python profiler module (which
         prints a detailed report of execution times, function calls, etc).
 
         You can pass other options after -p which affect the behavior of the
         profiler itself. See the docs for %prun for details.
 
         In this mode, the program's variables do NOT propagate back to the
         IPython interactive namespace (because they remain in the namespace
         where the profiler executes them).
 
         Internally this triggers a call to %prun, see its documentation for
         details on the options available specifically for profiling.
 
         There is one special usage for which the text above doesn't apply:
         if the filename ends with .ipy, the file is run as ipython script,
         just as if the commands were written on IPython prompt.
         """
 
         # get arguments and set sys.argv for program to be run.
         opts,arg_lst = self.parse_options(parameter_s,'nidtN:b:pD:l:rs:T:e',
                                           mode='list',list_all=1)
 
         try:
             filename = get_py_filename(arg_lst[0])
         except IndexError:
             warn('you must provide at least a filename.')
             print '\n%run:\n',OInspect.getdoc(self.magic_run)
             return
         except IOError,msg:
             error(msg)
             return
 
         if filename.lower().endswith('.ipy'):
             self.api.runlines(open(filename).read())
             return
         
         # Control the response to exit() calls made by the script being run
         exit_ignore = opts.has_key('e')
         
         # Make sure that the running script gets a proper sys.argv as if it
         # were run from a system shell.
         save_argv = sys.argv # save it for later restoring
         sys.argv = [filename]+ arg_lst[1:]  # put in the proper filename
 
         if opts.has_key('i'):
             # Run in user's interactive namespace
             prog_ns = self.shell.user_ns
             __name__save = self.shell.user_ns['__name__']
             prog_ns['__name__'] = '__main__'
             main_mod = FakeModule(prog_ns)
         else:
             # Run in a fresh, empty namespace
             if opts.has_key('n'):
                 name = os.path.splitext(os.path.basename(filename))[0]
             else:
                 name = '__main__'
             main_mod = FakeModule()
             prog_ns = main_mod.__dict__
             prog_ns['__name__'] = name
             # The shell MUST hold a reference to main_mod so after %run exits,
             # the python deletion mechanism doesn't zero it out (leaving
             # dangling references)
             self.shell._user_main_modules.append(main_mod)
 
         # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
         # set the __file__ global in the script's namespace
         prog_ns['__file__'] = filename
 
         # pickle fix.  See iplib for an explanation.  But we need to make sure
         # that, if we overwrite __main__, we replace it at the end
         if prog_ns['__name__'] == '__main__':
             restore_main = sys.modules['__main__']
         else:
             restore_main = False
 
         sys.modules[prog_ns['__name__']] = main_mod
         
         stats = None
         try:
             self.shell.savehist()
 
             if opts.has_key('p'):
                 stats = self.magic_prun('',0,opts,arg_lst,prog_ns)
             else:
                 if opts.has_key('d'):
                     deb = Debugger.Pdb(self.shell.rc.colors)
                     # reset Breakpoint state, which is moronically kept
                     # in a class
                     bdb.Breakpoint.next = 1
                     bdb.Breakpoint.bplist = {}
                     bdb.Breakpoint.bpbynumber = [None]
                     # Set an initial breakpoint to stop execution
                     maxtries = 10
                     bp = int(opts.get('b',[1])[0])
                     checkline = deb.checkline(filename,bp)
                     if not checkline:
                         for bp in range(bp+1,bp+maxtries+1):
                             if deb.checkline(filename,bp):
                                 break
                         else:
                             msg = ("\nI failed to find a valid line to set "
                                    "a breakpoint\n"
                                    "after trying up to line: %s.\n"
                                    "Please set a valid breakpoint manually "
                                    "with the -b option." % bp)
                             error(msg)
                             return
                     # if we find a good linenumber, set the breakpoint
                     deb.do_break('%s:%s' % (filename,bp))
                     # Start file run
                     print "NOTE: Enter 'c' at the",
                     print "%s prompt to start your script." % deb.prompt
                     try:
                         deb.run('execfile("%s")' % filename,prog_ns)
                         
                     except:
                         etype, value, tb = sys.exc_info()
                         # Skip three frames in the traceback: the %run one,
                         # one inside bdb.py, and the command-line typed by the
                         # user (run by exec in pdb itself).
                         self.shell.InteractiveTB(etype,value,tb,tb_offset=3)
                 else:
                     if runner is None:
                         runner = self.shell.safe_execfile
                     if opts.has_key('t'):
                         # timed execution
                         try:
                             nruns = int(opts['N'][0])
                             if nruns < 1:
                                 error('Number of runs must be >=1')
                                 return
                         except (KeyError):
                             nruns = 1
                         if nruns == 1:
                             t0 = clock2()
                             runner(filename,prog_ns,prog_ns,
                                    exit_ignore=exit_ignore)
                             t1 = clock2()
                             t_usr = t1[0]-t0[0]
                             t_sys = t1[1]-t1[1]
                             print "\nIPython CPU timings (estimated):"
                             print "  User  : %10s s." % t_usr
                             print "  System: %10s s." % t_sys
                         else:
                             runs = range(nruns)
                             t0 = clock2()
                             for nr in runs:
                                 runner(filename,prog_ns,prog_ns,
                                        exit_ignore=exit_ignore)
                             t1 = clock2()
                             t_usr = t1[0]-t0[0]
                             t_sys = t1[1]-t1[1]
                             print "\nIPython CPU timings (estimated):"
                             print "Total runs performed:",nruns
                             print "  Times : %10s    %10s" % ('Total','Per run')
                             print "  User  : %10s s, %10s s." % (t_usr,t_usr/nruns)
                             print "  System: %10s s, %10s s." % (t_sys,t_sys/nruns)
                             
                     else:
                         # regular execution
                         runner(filename,prog_ns,prog_ns,exit_ignore=exit_ignore)
                 if opts.has_key('i'):
                     self.shell.user_ns['__name__'] = __name__save
                 else:
                     # update IPython interactive namespace
                     del prog_ns['__name__']
                     self.shell.user_ns.update(prog_ns)
         finally:
             sys.argv = save_argv
             if restore_main:
                 sys.modules['__main__'] = restore_main
             self.shell.reloadhist()
                 
         return stats
 
     def magic_runlog(self, parameter_s =''):
         """Run files as logs.
 
         Usage:\\
           %runlog file1 file2 ...
 
         Run the named files (treating them as log files) in sequence inside
         the interpreter, and return to the prompt.  This is much slower than
         %run because each line is executed in a try/except block, but it
         allows running files with syntax errors in them.
 
         Normally IPython will guess when a file is one of its own logfiles, so
         you can typically use %run even for logs. This shorthand allows you to
         force any file to be treated as a log file."""
 
         for f in parameter_s.split():
             self.shell.safe_execfile(f,self.shell.user_ns,
                                      self.shell.user_ns,islog=1)
 
     def magic_timeit(self, parameter_s =''):
         """Time execution of a Python statement or expression
 
         Usage:\\
           %timeit [-n<N> -r<R> [-t|-c]] statement
 
         Time execution of a Python statement or expression using the timeit
         module.
 
         Options:
         -n<N>: execute the given statement <N> times in a loop. If this value
         is not given, a fitting value is chosen. 
         
         -r<R>: repeat the loop iteration <R> times and take the best result.
         Default: 3
         
         -t: use time.time to measure the time, which is the default on Unix.
         This function measures wall time.
         
         -c: use time.clock to measure the time, which is the default on
         Windows and measures wall time. On Unix, resource.getrusage is used
         instead and returns the CPU user time.
 
         -p<P>: use a precision of <P> digits to display the timing result.
         Default: 3
 
         
         Examples:\\
           In [1]: %timeit pass
           10000000 loops, best of 3: 53.3 ns per loop
 
           In [2]: u = None
 
           In [3]: %timeit u is None
           10000000 loops, best of 3: 184 ns per loop
 
           In [4]: %timeit -r 4 u == None
           1000000 loops, best of 4: 242 ns per loop
 
           In [5]: import time
 
           In [6]: %timeit -n1 time.sleep(2)
           1 loops, best of 3: 2 s per loop
           
 
         The times reported by %timeit will be slightly higher than those
         reported by the timeit.py script when variables are accessed. This is
         due to the fact that %timeit executes the statement in the namespace
         of the shell, compared with timeit.py, which uses a single setup
         statement to import function or create variables. Generally, the bias
         does not matter as long as results from timeit.py are not mixed with
         those from %timeit."""
 
         import timeit
         import math
 
         units = ["s", "ms", "\xc2\xb5s", "ns"]
         scaling = [1, 1e3, 1e6, 1e9]
 
         opts, stmt = self.parse_options(parameter_s,'n:r:tcp:',
                                         posix=False)
         if stmt == "":
             return
         timefunc = timeit.default_timer
         number = int(getattr(opts, "n", 0))
         repeat = int(getattr(opts, "r", timeit.default_repeat))
         precision = int(getattr(opts, "p", 3))
         if hasattr(opts, "t"):
             timefunc = time.time
         if hasattr(opts, "c"):
             timefunc = clock
 
         timer = timeit.Timer(timer=timefunc)
         # this code has tight coupling to the inner workings of timeit.Timer,
         # but is there a better way to achieve that the code stmt has access
         # to the shell namespace?
 
         src = timeit.template % {'stmt': timeit.reindent(stmt, 8),
                                  'setup': "pass"}
         # Track compilation time so it can be reported if too long
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
         
         t0 = clock()
         code = compile(src, "<magic-timeit>", "exec")
         tc = clock()-t0
         
         ns = {}
         exec code in self.shell.user_ns, ns
         timer.inner = ns["inner"]
         
         if number == 0:
             # determine number so that 0.2 <= total time < 2.0
             number = 1
             for i in range(1, 10):
                 number *= 10
                 if timer.timeit(number) >= 0.2:
                     break
         
         best = min(timer.repeat(repeat, number)) / number
 
         if best > 0.0:
             order = min(-int(math.floor(math.log10(best)) // 3), 3)
         else:
             order = 3
         print "%d loops, best of %d: %.*g %s per loop" % (number, repeat,
                                                           precision,
                                                           best * scaling[order],
                                                           units[order])
         if tc > tc_min:
             print "Compiler time: %.2f s" % tc
         
     def magic_time(self,parameter_s = ''):
         """Time execution of a Python statement or expression.
 
         The CPU and wall clock times are printed, and the value of the
         expression (if any) is returned.  Note that under Win32, system time
         is always reported as 0, since it can not be measured.
 
         This function provides very basic timing functionality.  In Python
         2.3, the timeit module offers more control and sophistication, so this
         could be rewritten to use it (patches welcome).
         
         Some examples:
 
           In [1]: time 2**128
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
           Out[1]: 340282366920938463463374607431768211456L
 
           In [2]: n = 1000000
 
           In [3]: time sum(range(n))
           CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
           Wall time: 1.37
           Out[3]: 499999500000L
 
           In [4]: time print 'hello world'
           hello world
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
 
           Note that the time needed by Python to compile the given expression
           will be reported if it is more than 0.1s.  In this example, the
           actual exponentiation is done by Python at compilation time, so while
           the expression can take a noticeable amount of time to compute, that
           time is purely due to the compilation:
 
           In [5]: time 3**9999;
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00 s
 
           In [6]: time 3**999999;
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00 s
           Compiler : 0.78 s
           """
         
         # fail immediately if the given expression can't be compiled
         
         expr = self.shell.prefilter(parameter_s,False)
 
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
         
         try:
             mode = 'eval'
             t0 = clock()
             code = compile(expr,'<timed eval>',mode)
             tc = clock()-t0
         except SyntaxError:
             mode = 'exec'
             t0 = clock()
             code = compile(expr,'<timed exec>',mode)
             tc = clock()-t0
         # skew measurement as little as possible
         glob = self.shell.user_ns
         clk = clock2
         wtime = time.time
         # time execution
         wall_st = wtime()
         if mode=='eval':
             st = clk()
             out = eval(code,glob)
             end = clk()
         else:
             st = clk()
             exec code in glob
             end = clk()
             out = None
         wall_end = wtime()
         # Compute actual times and report
         wall_time = wall_end-wall_st
         cpu_user = end[0]-st[0]
         cpu_sys = end[1]-st[1]
         cpu_tot = cpu_user+cpu_sys
         print "CPU times: user %.2f s, sys: %.2f s, total: %.2f s" % \
               (cpu_user,cpu_sys,cpu_tot)
         print "Wall time: %.2f s" % wall_time
         if tc > tc_min:
             print "Compiler : %.2f s" % tc
         return out
 
     def magic_macro(self,parameter_s = ''):
         """Define a set of input lines as a macro for future re-execution.
 
         Usage:\\
           %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
         
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed as the
           command line is used instead.
 
         This will define a global variable called `name` which is a string
         made of joining the slices and lines you specify (n1,n2,... numbers
         above) from your input history into a single string. This variable
         acts like an automatic function which re-executes those lines as if
         you had typed them. You just type 'name' at the prompt and the code
         executes.
 
         The notation for indicating number ranges is: n1-n2 means 'use line
         numbers n1,...n2' (the endpoint is included).  That is, '5-7' means
         using the lines numbered 5,6 and 7.
 
         Note: as a 'hidden' feature, you can also use traditional python slice
         notation, where N:M means numbers N through M-1.
 
         For example, if your history contains (%hist prints it):
         
           44: x=1\\
           45: y=3\\
           46: z=x+y\\
           47: print x\\
           48: a=5\\
           49: print 'x',x,'y',y\\
 
         you can create a macro with lines 44 through 47 (included) and line 49
         called my_macro with:
 
           In [51]: %macro my_macro 44-47 49
 
         Now, typing `my_macro` (without quotes) will re-execute all this code
         in one pass.
 
         You don't need to give the line-numbers in order, and any given line
         number can appear multiple times. You can assemble macros with any
         lines from your input history in any order.
 
         The macro is a simple object which holds its value in an attribute,
         but IPython's display system checks for macros and executes them as
         code instead of printing them when you type their name.
 
         You can view a macro's contents by explicitly printing it with:
         
           'print macro_name'.
 
         For one-off cases which DON'T contain magic function calls in them you
         can obtain similar results by explicitly executing slices from your
         input history with:
 
           In [60]: exec In[44:48]+In[49]"""
 
         opts,args = self.parse_options(parameter_s,'r',mode='list')
         if not args:
             macs = [k for k,v in self.shell.user_ns.items() if isinstance(v, Macro)]
             macs.sort()
             return macs
         if len(args) == 1:
             raise UsageError(
                 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
         name,ranges = args[0], args[1:]
         
         #print 'rng',ranges  # dbg
         lines = self.extract_input_slices(ranges,opts.has_key('r'))
         macro = Macro(lines)
         self.shell.user_ns.update({name:macro})
         print 'Macro `%s` created. To execute, type its name (without quotes).' % name
         print 'Macro contents:'
         print macro,
 
     def magic_save(self,parameter_s = ''):
         """Save a set of lines to a given filename.
 
         Usage:\\
           %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
         
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed as the
           command line is used instead.
 
         This function uses the same syntax as %macro for line extraction, but
         instead of creating a macro it saves the resulting string to the
         filename you specify.
 
         It adds a '.py' extension to the file if you don't do so yourself, and
         it asks for confirmation before overwriting existing files."""
 
         opts,args = self.parse_options(parameter_s,'r',mode='list')
         fname,ranges = args[0], args[1:]
         if not fname.endswith('.py'):
             fname += '.py'
         if os.path.isfile(fname):
             ans = raw_input('File `%s` exists. Overwrite (y/[N])? ' % fname)
             if ans.lower() not in ['y','yes']:
                 print 'Operation cancelled.'
                 return
         cmds = ''.join(self.extract_input_slices(ranges,opts.has_key('r')))
         f = file(fname,'w')
         f.write(cmds)
         f.close()
         print 'The following commands were written to file `%s`:' % fname
         print cmds
 
     def _edit_macro(self,mname,macro):
         """open an editor with the macro data in a file"""
         filename = self.shell.mktempfile(macro.value)
         self.shell.hooks.editor(filename)
 
         # and make a new macro object, to replace the old one
         mfile = open(filename)
         mvalue = mfile.read()
         mfile.close()
         self.shell.user_ns[mname] = Macro(mvalue)
 
     def magic_ed(self,parameter_s=''):
         """Alias to %edit."""
         return self.magic_edit(parameter_s)
 
     def magic_edit(self,parameter_s='',last_call=['','']):
         """Bring up an editor and execute the resulting code.
 
         Usage:
           %edit [options] [args]
 
         %edit runs IPython's editor hook.  The default version of this hook is
         set to call the __IPYTHON__.rc.editor command.  This is read from your
         environment variable $EDITOR.  If this isn't found, it will default to
         vi under Linux/Unix and to notepad under Windows.  See the end of this
         docstring for how to change the editor hook.
 
         You can also set the value of this editor via the command line option
         '-editor' or in your ipythonrc file. This is useful if you wish to use
         specifically for IPython an editor different from your typical default
         (and for Windows users who typically don't set environment variables).
 
         This command allows you to conveniently edit multi-line code right in
         your IPython session.
         
         If called without arguments, %edit opens up an empty editor with a
         temporary file and will execute the contents of this file when you
         close it (don't forget to save it!).
 
 
         Options:
 
         -n <number>: open the editor at a specified line number.  By default,
         the IPython editor hook uses the unix syntax 'editor +N filename', but
         you can configure this by providing your own modified hook if your
         favorite editor supports line-number specifications with a different
         syntax.
         
         -p: this will call the editor with the same data as the previous time
         it was used, regardless of how long ago (in your current session) it
         was.
 
         -r: use 'raw' input.  This option only applies to input taken from the
         user's history.  By default, the 'processed' history is used, so that
         magics are loaded in their transformed version to valid Python.  If
         this option is given, the raw input as typed as the command line is
         used instead.  When you exit the editor, it will be executed by
         IPython's own processor.
         
         -x: do not execute the edited code immediately upon exit. This is
         mainly useful if you are editing programs which need to be called with
         command line arguments, which you can then do using %run.
 
 
         Arguments:
 
         If arguments are given, the following possibilites exist:
 
         - The arguments are numbers or pairs of colon-separated numbers (like
         1 4:8 9). These are interpreted as lines of previous input to be
         loaded into the editor. The syntax is the same of the %macro command.
 
         - If the argument doesn't start with a number, it is evaluated as a
         variable and its contents loaded into the editor. You can thus edit
         any string which contains python code (including the result of
         previous edits).
 
         - If the argument is the name of an object (other than a string),
         IPython will try to locate the file where it was defined and open the
         editor at the point where it is defined. You can use `%edit function`
         to load an editor exactly at the point where 'function' is defined,
         edit it and have the file be executed automatically.
 
         If the object is a macro (see %macro for details), this opens up your
         specified editor with a temporary file containing the macro's data.
         Upon exit, the macro is reloaded with the contents of the file.
 
         Note: opening at an exact line is only supported under Unix, and some
         editors (like kedit and gedit up to Gnome 2.8) do not understand the
         '+NUMBER' parameter necessary for this feature. Good editors like
         (X)Emacs, vi, jed, pico and joe all do.
 
         - If the argument is not found as a variable, IPython will look for a
         file with that name (adding .py if necessary) and load it into the
         editor. It will execute its contents with execfile() when you exit,
         loading any code in the file into your interactive namespace.
 
         After executing your code, %edit will return as output the code you
         typed in the editor (except when it was an existing file). This way
         you can reload the code in further invocations of %edit as a variable,
         via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
         the output.
 
         Note that %edit is also available through the alias %ed.
 
         This is an example of creating a simple function inside the editor and
         then modifying it. First, start up the editor:
 
         In [1]: ed\\
         Editing... done. Executing edited code...\\
         Out[1]: 'def foo():\\n    print "foo() was defined in an editing session"\\n'
 
         We can then call the function foo():
         
         In [2]: foo()\\
         foo() was defined in an editing session
 
         Now we edit foo.  IPython automatically loads the editor with the
         (temporary) file where foo() was previously defined:
         
         In [3]: ed foo\\
         Editing... done. Executing edited code...
 
         And if we call foo() again we get the modified version:
         
         In [4]: foo()\\
         foo() has now been changed!
 
         Here is an example of how to edit a code snippet successive
         times. First we call the editor:
 
         In [8]: ed\\
         Editing... done. Executing edited code...\\
         hello\\
         Out[8]: "print 'hello'\\n"
 
         Now we call it again with the previous output (stored in _):
 
         In [9]: ed _\\
         Editing... done. Executing edited code...\\
         hello world\\
         Out[9]: "print 'hello world'\\n"
 
         Now we call it with the output #8 (stored in _8, also as Out[8]):
 
         In [10]: ed _8\\
         Editing... done. Executing edited code...\\
         hello again\\
         Out[10]: "print 'hello again'\\n"
 
 
         Changing the default editor hook:
 
         If you wish to write your own editor hook, you can put it in a
         configuration file which you load at startup time.  The default hook
         is defined in the IPython.hooks module, and you can use that as a
         starting example for further modifications.  That file also has
         general instructions on how to set a new hook for use once you've
         defined it."""
         
         # FIXME: This function has become a convoluted mess.  It needs a
         # ground-up rewrite with clean, simple logic.
 
         def make_filename(arg):
             "Make a filename from the given args"
             try:
                 filename = get_py_filename(arg)
             except IOError:
                 if args.endswith('.py'):
                     filename = arg
                 else:
                     filename = None
             return filename
 
         # custom exceptions
         class DataIsObject(Exception): pass
 
         opts,args = self.parse_options(parameter_s,'prxn:')
         # Set a few locals from the options for convenience:
         opts_p = opts.has_key('p')
         opts_r = opts.has_key('r')
         
         # Default line number value
         lineno = opts.get('n',None)
 
         if opts_p:
             args = '_%s' % last_call[0]
             if not self.shell.user_ns.has_key(args):
                 args = last_call[1]
             
         # use last_call to remember the state of the previous call, but don't
         # let it be clobbered by successive '-p' calls.
         try:
             last_call[0] = self.shell.outputcache.prompt_count
             if not opts_p:
                 last_call[1] = parameter_s
         except:
             pass
 
         # by default this is done with temp files, except when the given
         # arg is a filename
         use_temp = 1
 
         if re.match(r'\d',args):
             # Mode where user specifies ranges of lines, like in %macro.
             # This means that you can't edit files whose names begin with
             # numbers this way. Tough.
             ranges = args.split()
             data = ''.join(self.extract_input_slices(ranges,opts_r))
         elif args.endswith('.py'):
             filename = make_filename(args)
             data = ''
             use_temp = 0
         elif args:
             try:
                 # Load the parameter given as a variable. If not a string,
                 # process it as an object instead (below)
 
                 #print '*** args',args,'type',type(args)  # dbg
                 data = eval(args,self.shell.user_ns)
                 if not type(data) in StringTypes:
                     raise DataIsObject
 
             except (NameError,SyntaxError):
                 # given argument is not a variable, try as a filename
                 filename = make_filename(args)
                 if filename is None:
                     warn("Argument given (%s) can't be found as a variable "
                          "or as a filename." % args)
                     return
 
                 data = ''
                 use_temp = 0
             except DataIsObject:
 
                 # macros have a special edit function
                 if isinstance(data,Macro):
                     self._edit_macro(args,data)
                     return
                                 
                 # For objects, try to edit the file where they are defined
                 try:
                     filename = inspect.getabsfile(data)
                     if 'fakemodule' in filename.lower() and inspect.isclass(data):                     
                         # class created by %edit? Try to find source
                         # by looking for method definitions instead, the
                         # __module__ in those classes is FakeModule.
                         attrs = [getattr(data, aname) for aname in dir(data)]
                         for attr in attrs:
                             if not inspect.ismethod(attr):
                                 continue
                             filename = inspect.getabsfile(attr)
                             if filename and 'fakemodule' not in filename.lower():
                                 # change the attribute to be the edit target instead
                                 data = attr 
                                 break
                     
                     datafile = 1
                 except TypeError:
                     filename = make_filename(args)
                     datafile = 1
                     warn('Could not find file where `%s` is defined.\n'
                          'Opening a file named `%s`' % (args,filename))
                 # Now, make sure we can actually read the source (if it was in
                 # a temp file it's gone by now).
                 if datafile:
                     try:
                         if lineno is None:
                             lineno = inspect.getsourcelines(data)[1]
                     except IOError:
                         filename = make_filename(args)
                         if filename is None:
                             warn('The file `%s` where `%s` was defined cannot '
                                  'be read.' % (filename,data))
                             return
                 use_temp = 0
         else:
             data = ''
 
         if use_temp:
             filename = self.shell.mktempfile(data)
             print 'IPython will make a temporary file named:',filename
 
         # do actual editing here
         print 'Editing...',
         sys.stdout.flush()
         self.shell.hooks.editor(filename,lineno)
         if opts.has_key('x'):  # -x prevents actual execution
             print
         else:
             print 'done. Executing edited code...'
             if opts_r:
                 self.shell.runlines(file_read(filename))
             else:
                 self.shell.safe_execfile(filename,self.shell.user_ns,
                                          self.shell.user_ns)
         if use_temp:
             try:
                 return open(filename).read()
             except IOError,msg:
                 if msg.filename == filename:
                     warn('File not found. Did you forget to save?')
                     return
                 else:
                     self.shell.showtraceback()
 
     def magic_xmode(self,parameter_s = ''):
         """Switch modes for the exception handlers.
 
         Valid modes: Plain, Context and Verbose.
 
         If called without arguments, acts as a toggle."""
 
         def xmode_switch_err(name):
             warn('Error changing %s exception modes.\n%s' %
                  (name,sys.exc_info()[1]))
 
         shell = self.shell
         new_mode = parameter_s.strip().capitalize()
         try:
             shell.InteractiveTB.set_mode(mode=new_mode)
             print 'Exception reporting mode:',shell.InteractiveTB.mode
         except:
             xmode_switch_err('user')
 
         # threaded shells use a special handler in sys.excepthook
         if shell.isthreaded:
             try:
                 shell.sys_excepthook.set_mode(mode=new_mode)
             except:
                 xmode_switch_err('threaded')
             
     def magic_colors(self,parameter_s = ''):
         """Switch color scheme for prompts, info system and exception handlers.
 
         Currently implemented schemes: NoColor, Linux, LightBG.
 
         Color scheme names are not case-sensitive."""
 
         def color_switch_err(name):
             warn('Error changing %s color schemes.\n%s' %
                  (name,sys.exc_info()[1]))
             
         
         new_scheme = parameter_s.strip()
         if not new_scheme:
             raise UsageError(
                 "%colors: you must specify a color scheme. See '%colors?'")
             return
         # local shortcut
         shell = self.shell
 
         import IPython.rlineimpl as readline
 
         if not readline.have_readline and sys.platform == "win32":
             msg = """\
 Proper color support under MS Windows requires the pyreadline library.
 You can find it at:
 http://ipython.scipy.org/moin/PyReadline/Intro
 Gary's readline needs the ctypes module, from:
 http://starship.python.net/crew/theller/ctypes
 (Note that ctypes is already part of Python versions 2.5 and newer).
 
 Defaulting color scheme to 'NoColor'"""
             new_scheme = 'NoColor'
             warn(msg)
         
         # readline option is 0
         if not shell.has_readline:
             new_scheme = 'NoColor'
             
         # Set prompt colors
         try:
             shell.outputcache.set_colors(new_scheme)
         except:
             color_switch_err('prompt')
         else:
             shell.rc.colors = \
                        shell.outputcache.color_table.active_scheme_name
         # Set exception colors
         try:
             shell.InteractiveTB.set_colors(scheme = new_scheme)
             shell.SyntaxTB.set_colors(scheme = new_scheme)
         except:
             color_switch_err('exception')
 
         # threaded shells use a verbose traceback in sys.excepthook
         if shell.isthreaded:
             try:
                 shell.sys_excepthook.set_colors(scheme=new_scheme)
             except:
                 color_switch_err('system exception handler')
         
         # Set info (for 'object?') colors
         if shell.rc.color_info:
             try:
                 shell.inspector.set_active_scheme(new_scheme)
             except:
                 color_switch_err('object inspector')
         else:
             shell.inspector.set_active_scheme('NoColor')
                 
     def magic_color_info(self,parameter_s = ''):
         """Toggle color_info.
 
         The color_info configuration parameter controls whether colors are
         used for displaying object details (by things like %psource, %pfile or
         the '?' system). This function toggles this value with each call.
 
         Note that unless you have a fairly recent pager (less works better
         than more) in your system, using colored object information displays
         will not work properly. Test it and see."""
         
         self.shell.rc.color_info = 1 - self.shell.rc.color_info
         self.magic_colors(self.shell.rc.colors)
         print 'Object introspection functions have now coloring:',
         print ['OFF','ON'][self.shell.rc.color_info]
 
     def magic_Pprint(self, parameter_s=''):
         """Toggle pretty printing on/off."""
         
         self.shell.rc.pprint = 1 - self.shell.rc.pprint
         print 'Pretty printing has been turned', \
               ['OFF','ON'][self.shell.rc.pprint]
         
     def magic_exit(self, parameter_s=''):
         """Exit IPython, confirming if configured to do so.
 
         You can configure whether IPython asks for confirmation upon exit by
         setting the confirm_exit flag in the ipythonrc file."""
 
         self.shell.exit()
 
     def magic_quit(self, parameter_s=''):
         """Exit IPython, confirming if configured to do so (like %exit)"""
 
         self.shell.exit()
         
     def magic_Exit(self, parameter_s=''):
         """Exit IPython without confirmation."""
 
         self.shell.exit_now = True
 
     #......................................................................
     # Functions to implement unix shell-type things
     
     def magic_alias(self, parameter_s = ''):
         """Define an alias for a system command.
 
         '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
 
         Then, typing 'alias_name params' will execute the system command 'cmd
         params' (from your underlying operating system).
 
         Aliases have lower precedence than magic functions and Python normal
         variables, so if 'foo' is both a Python variable and an alias, the
         alias can not be executed until 'del foo' removes the Python variable.
 
         You can use the %l specifier in an alias definition to represent the
         whole line when the alias is called.  For example:
 
           In [2]: alias all echo "Input in brackets: <%l>"\\
           In [3]: all hello world\\
           Input in brackets: <hello world>
 
         You can also define aliases with parameters using %s specifiers (one
         per parameter):
         
           In [1]: alias parts echo first %s second %s\\
           In [2]: %parts A B\\
           first A second B\\
           In [3]: %parts A\\
           Incorrect number of arguments: 2 expected.\\
           parts is an alias to: 'echo first %s second %s'
 
         Note that %l and %s are mutually exclusive.  You can only use one or
         the other in your aliases.
 
         Aliases expand Python variables just like system calls using ! or !! 
         do: all expressions prefixed with '$' get expanded.  For details of
         the semantic rules, see PEP-215:
         http://www.python.org/peps/pep-0215.html.  This is the library used by
         IPython for variable expansion.  If you want to access a true shell
         variable, an extra $ is necessary to prevent its expansion by IPython:
 
         In [6]: alias show echo\\
         In [7]: PATH='A Python string'\\
         In [8]: show $PATH\\
         A Python string\\
         In [9]: show $$PATH\\
         /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
 
         You can use the alias facility to acess all of $PATH.  See the %rehash
         and %rehashx functions, which automatically create aliases for the
         contents of your $PATH.
 
         If called with no parameters, %alias prints the current alias table."""
 
         par = parameter_s.strip()
         if not par:
             stored = self.db.get('stored_aliases', {} )
             atab = self.shell.alias_table
             aliases = atab.keys()
             aliases.sort()
             res = []
             showlast = []
             for alias in aliases:
                 special = False
                 try:
                     tgt = atab[alias][1]
                 except (TypeError, AttributeError):
                     # unsubscriptable? probably a callable
                     tgt = atab[alias]
                     special = True
                 # 'interesting' aliases
                 if (alias in stored or
                     special or 
                     alias.lower() != os.path.splitext(tgt)[0].lower() or
                     ' ' in tgt):
                     showlast.append((alias, tgt))
                 else:
                     res.append((alias, tgt ))                
             
             # show most interesting aliases last
             res.extend(showlast)
             print "Total number of aliases:",len(aliases)
             return res
         try:
             alias,cmd = par.split(None,1)
         except:
             print OInspect.getdoc(self.magic_alias)
         else:
             nargs = cmd.count('%s')
             if nargs>0 and cmd.find('%l')>=0:
                 error('The %s and %l specifiers are mutually exclusive '
                       'in alias definitions.')
             else:  # all looks OK
                 self.shell.alias_table[alias] = (nargs,cmd)
                 self.shell.alias_table_validate(verbose=0)
     # end magic_alias
 
     def magic_unalias(self, parameter_s = ''):
         """Remove an alias"""
 
         aname = parameter_s.strip()
         if aname in self.shell.alias_table:
             del self.shell.alias_table[aname]
         stored = self.db.get('stored_aliases', {} )
         if aname in stored:
             print "Removing %stored alias",aname
             del stored[aname]
             self.db['stored_aliases'] = stored
             
 
     def magic_rehashx(self, parameter_s = ''):
         """Update the alias table with all executable files in $PATH.
 
         This version explicitly checks that every entry in $PATH is a file
         with execute access (os.X_OK), so it is much slower than %rehash.
 
         Under Windows, it checks executability as a match agains a
         '|'-separated string of extensions, stored in the IPython config
         variable win_exec_ext.  This defaults to 'exe|com|bat'.
         
         This function also resets the root module cache of module completer,
         used on slow filesystems.
         """
         
         
         ip = self.api
 
         # for the benefit of module completer in ipy_completers.py
         del ip.db['rootmodules']
         
         path = [os.path.abspath(os.path.expanduser(p)) for p in 
             os.environ.get('PATH','').split(os.pathsep)]
         path = filter(os.path.isdir,path)
         
         alias_table = self.shell.alias_table
         syscmdlist = []
         if os.name == 'posix':
             isexec = lambda fname:os.path.isfile(fname) and \
                      os.access(fname,os.X_OK)
         else:
 
             try:
                 winext = os.environ['pathext'].replace(';','|').replace('.','')
             except KeyError:
                 winext = 'exe|com|bat|py'
             if 'py' not in winext:
                 winext += '|py'
             execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
             isexec = lambda fname:os.path.isfile(fname) and execre.match(fname)
         savedir = os.getcwd()
         try:
             # write the whole loop for posix/Windows so we don't have an if in
             # the innermost part
             if os.name == 'posix':
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):
                         if isexec(ff) and ff not in self.shell.no_alias:
                             # each entry in the alias table must be (N,name),
                             # where N is the number of positional arguments of the
                             # alias.
                             alias_table[ff] = (0,ff)
                             syscmdlist.append(ff)
             else:
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):
                         base, ext = os.path.splitext(ff)
-                        if isexec(ff) and base not in self.shell.no_alias:
+                        if isexec(ff) and base.lower() not in self.shell.no_alias:
                             if ext.lower() == '.exe':
                                 ff = base
                             alias_table[base.lower()] = (0,ff)
                             syscmdlist.append(ff)
             # Make sure the alias table doesn't contain keywords or builtins
             self.shell.alias_table_validate()
             # Call again init_auto_alias() so we get 'rm -i' and other
             # modified aliases since %rehashx will probably clobber them
             
             # no, we don't want them. if %rehashx clobbers them, good,
             # we'll probably get better versions
             # self.shell.init_auto_alias()
             db = ip.db
             db['syscmdlist'] = syscmdlist
         finally:
             os.chdir(savedir)
         
     def magic_pwd(self, parameter_s = ''):
         """Return the current working directory path."""
         return os.getcwd()
 
     def magic_cd(self, parameter_s=''):
         """Change the current working directory.
 
         This command automatically maintains an internal list of directories
         you visit during your IPython session, in the variable _dh. The
         command %dhist shows this history nicely formatted. You can also
         do 'cd -<tab>' to see directory history conveniently.
 
         Usage:
 
           cd 'dir': changes to directory 'dir'.
 
           cd -: changes to the last visited directory.
 
           cd -<n>: changes to the n-th directory in the directory history.
 
           cd -b <bookmark_name>: jump to a bookmark set by %bookmark
              (note: cd <bookmark_name> is enough if there is no
               directory <bookmark_name>, but a bookmark with the name exists.)
               'cd -b <tab>' allows you to tab-complete bookmark names. 
 
         Options:
 
         -q: quiet.  Do not print the working directory after the cd command is
         executed.  By default IPython's cd command does print this directory,
         since the default prompts do not display path information.
         
         Note that !cd doesn't work for this purpose because the shell where
         !command runs is immediately discarded after executing 'command'."""
 
         parameter_s = parameter_s.strip()
         #bkms = self.shell.persist.get("bookmarks",{})
 
+        oldcwd = os.getcwd()
         numcd = re.match(r'(-)(\d+)$',parameter_s)
         # jump in directory history by number
         if numcd:
             nn = int(numcd.group(2))
             try:
                 ps = self.shell.user_ns['_dh'][nn]
             except IndexError:
                 print 'The requested directory does not exist in history.'
                 return
             else:
                 opts = {}
         else:
             #turn all non-space-escaping backslashes to slashes, 
             # for c:\windows\directory\names\
             parameter_s = re.sub(r'\\(?! )','/', parameter_s)            
             opts,ps = self.parse_options(parameter_s,'qb',mode='string')
         # jump to previous
         if ps == '-':
             try:
                 ps = self.shell.user_ns['_dh'][-2]
             except IndexError:
                 raise UsageError('%cd -: No previous directory to change to.')
         # jump to bookmark if needed
         else:
             if not os.path.isdir(ps) or opts.has_key('b'):
                 bkms = self.db.get('bookmarks', {})
             
                 if bkms.has_key(ps):
                     target = bkms[ps]
                     print '(bookmark:%s) -> %s' % (ps,target)
                     ps = target
                 else:
                     if opts.has_key('b'):
                         raise UsageError("Bookmark '%s' not found.  "
                               "Use '%%bookmark -l' to see your bookmarks." % ps)
             
         # at this point ps should point to the target dir
         if ps:
-            try:
+            try:                
                 os.chdir(os.path.expanduser(ps))
                 if self.shell.rc.term_title:
                     #print 'set term title:',self.shell.rc.term_title  # dbg
                     ttitle = 'IPy ' + abbrev_cwd()
                     platutils.set_term_title(ttitle)
             except OSError:
                 print sys.exc_info()[1]
             else:
                 cwd = os.getcwd()
                 dhist = self.shell.user_ns['_dh']
-                dhist.append(cwd)
-                self.db['dhist'] = compress_dhist(dhist)[-100:]
+                if oldcwd != cwd:
+                    dhist.append(cwd)
+                    self.db['dhist'] = compress_dhist(dhist)[-100:]
                 
         else:
             os.chdir(self.shell.home_dir)
             if self.shell.rc.term_title:
                 platutils.set_term_title("IPy ~")
             cwd = os.getcwd()
             dhist = self.shell.user_ns['_dh']
-            dhist.append(cwd)
-            self.db['dhist'] = compress_dhist(dhist)[-100:]
+            
+            if oldcwd != cwd:
+                dhist.append(cwd)
+                self.db['dhist'] = compress_dhist(dhist)[-100:]
         if not 'q' in opts and self.shell.user_ns['_dh']:
             print self.shell.user_ns['_dh'][-1]
 
 
     def magic_env(self, parameter_s=''):
         """List environment variables."""
         
         return os.environ.data
 
     def magic_pushd(self, parameter_s=''):
         """Place the current dir on stack and change directory.
         
         Usage:\\
           %pushd ['dirname']
         """
         
         dir_s = self.shell.dir_stack
         tgt = os.path.expanduser(parameter_s)
         cwd = os.getcwd().replace(self.home_dir,'~')
         if tgt:
             self.magic_cd(parameter_s)
         dir_s.insert(0,cwd)
         return self.magic_dirs()
 
     def magic_popd(self, parameter_s=''):
         """Change to directory popped off the top of the stack.
         """
         if not self.shell.dir_stack:
             raise UsageError("%popd on empty stack")
         top = self.shell.dir_stack.pop(0)
         self.magic_cd(top)
         print "popd ->",top
 
     def magic_dirs(self, parameter_s=''):
         """Return the current directory stack."""
 
         return self.shell.dir_stack
 
     def magic_dhist(self, parameter_s=''):
         """Print your history of visited directories.
 
         %dhist       -> print full history\\
         %dhist n     -> print last n entries only\\
         %dhist n1 n2 -> print entries between n1 and n2 (n1 not included)\\
 
         This history is automatically maintained by the %cd command, and
         always available as the global list variable _dh. You can use %cd -<n>
         to go to directory number <n>.
         
         Note that most of time, you should view directory history by entering
         cd -<TAB>.
         
         """
 
         dh = self.shell.user_ns['_dh']
         if parameter_s:
             try:
                 args = map(int,parameter_s.split())
             except:
                 self.arg_err(Magic.magic_dhist)
                 return
             if len(args) == 1:
                 ini,fin = max(len(dh)-(args[0]),0),len(dh)
             elif len(args) == 2:
                 ini,fin = args
             else:
                 self.arg_err(Magic.magic_dhist)
                 return
         else:
             ini,fin = 0,len(dh)
         nlprint(dh,
                 header = 'Directory history (kept in _dh)',
                 start=ini,stop=fin)
 
 
     def magic_sc(self, parameter_s=''):
         """Shell capture - execute a shell command and capture its output.
 
         DEPRECATED. Suboptimal, retained for backwards compatibility.
         
         You should use the form 'var = !command' instead. Example:
          
          "%sc -l myfiles = ls ~" should now be written as
             
          "myfiles = !ls ~"
          
         myfiles.s, myfiles.l and myfiles.n still apply as documented
         below.
 
         --
         %sc [options] varname=command
 
         IPython will run the given command using commands.getoutput(), and
         will then update the user's interactive namespace with a variable
         called varname, containing the value of the call.  Your command can
         contain shell wildcards, pipes, etc.
 
         The '=' sign in the syntax is mandatory, and the variable name you
         supply must follow Python's standard conventions for valid names.
         
         (A special format without variable name exists for internal use)
 
         Options:
 
           -l: list output.  Split the output on newlines into a list before
           assigning it to the given variable.  By default the output is stored
           as a single string.
 
           -v: verbose.  Print the contents of the variable.
 
         In most cases you should not need to split as a list, because the
         returned value is a special type of string which can automatically
         provide its contents either as a list (split on newlines) or as a
         space-separated string.  These are convenient, respectively, either
         for sequential processing or to be passed to a shell command.
 
         For example:
 
             # Capture into variable a
             In [9]: sc a=ls *py
 
             # a is a string with embedded newlines
             In [10]: a
             Out[10]: 'setup.py\nwin32_manual_post_install.py'
 
             # which can be seen as a list:
             In [11]: a.l
             Out[11]: ['setup.py', 'win32_manual_post_install.py']
 
             # or as a whitespace-separated string:
             In [12]: a.s
             Out[12]: 'setup.py win32_manual_post_install.py'
 
             # a.s is useful to pass as a single command line:
             In [13]: !wc -l $a.s
               146 setup.py
               130 win32_manual_post_install.py
               276 total
 
             # while the list form is useful to loop over:
             In [14]: for f in a.l:
                ....:      !wc -l $f
                ....:
             146 setup.py
             130 win32_manual_post_install.py
 
         Similiarly, the lists returned by the -l option are also special, in
         the sense that you can equally invoke the .s attribute on them to
         automatically get a whitespace-separated string from their contents:
 
             In [1]: sc -l b=ls *py
 
             In [2]: b
             Out[2]: ['setup.py', 'win32_manual_post_install.py']
 
             In [3]: b.s
             Out[3]: 'setup.py win32_manual_post_install.py'
 
         In summary, both the lists and strings used for ouptut capture have
         the following special attributes:
 
             .l (or .list) : value as list.
             .n (or .nlstr): value as newline-separated string.
             .s (or .spstr): value as space-separated string.
         """
 
         opts,args = self.parse_options(parameter_s,'lv')
         # Try to get a variable name and command to run
         try:
             # the variable name must be obtained from the parse_options
             # output, which uses shlex.split to strip options out.
             var,_ = args.split('=',1)
             var = var.strip()
             # But the the command has to be extracted from the original input
             # parameter_s, not on what parse_options returns, to avoid the
             # quote stripping which shlex.split performs on it.
             _,cmd = parameter_s.split('=',1)
         except ValueError:
             var,cmd = '',''
         # If all looks ok, proceed
         out,err = self.shell.getoutputerror(cmd)
         if err:
             print >> Term.cerr,err
         if opts.has_key('l'):
             out = SList(out.split('\n'))
         else:
             out = LSString(out)
         if opts.has_key('v'):
             print '%s ==\n%s' % (var,pformat(out))
         if var:
             self.shell.user_ns.update({var:out})
         else:
             return out
 
     def magic_sx(self, parameter_s=''):
         """Shell execute - run a shell command and capture its output.
 
         %sx command
 
         IPython will run the given command using commands.getoutput(), and
         return the result formatted as a list (split on '\\n').  Since the
         output is _returned_, it will be stored in ipython's regular output
         cache Out[N] and in the '_N' automatic variables.
 
         Notes:
 
         1) If an input line begins with '!!', then %sx is automatically
         invoked.  That is, while:
           !ls
         causes ipython to simply issue system('ls'), typing
           !!ls
         is a shorthand equivalent to:
           %sx ls
         
         2) %sx differs from %sc in that %sx automatically splits into a list,
         like '%sc -l'.  The reason for this is to make it as easy as possible
         to process line-oriented shell output via further python commands.
         %sc is meant to provide much finer control, but requires more
         typing.
 
         3) Just like %sc -l, this is a list with special attributes:
 
           .l (or .list) : value as list.
           .n (or .nlstr): value as newline-separated string.
           .s (or .spstr): value as whitespace-separated string.
 
         This is very useful when trying to use such lists as arguments to
         system commands."""
 
         if parameter_s:
             out,err = self.shell.getoutputerror(parameter_s)
             if err:
                 print >> Term.cerr,err
             return SList(out.split('\n'))
 
     def magic_bg(self, parameter_s=''):
         """Run a job in the background, in a separate thread.
 
         For example,
 
           %bg myfunc(x,y,z=1)
 
         will execute 'myfunc(x,y,z=1)' in a background thread.  As soon as the
         execution starts, a message will be printed indicating the job
         number.  If your job number is 5, you can use
 
           myvar = jobs.result(5)  or  myvar = jobs[5].result
 
         to assign this result to variable 'myvar'.
 
         IPython has a job manager, accessible via the 'jobs' object.  You can
         type jobs? to get more information about it, and use jobs.<TAB> to see
         its attributes.  All attributes not starting with an underscore are
         meant for public use.
 
         In particular, look at the jobs.new() method, which is used to create
         new jobs.  This magic %bg function is just a convenience wrapper
         around jobs.new(), for expression-based jobs.  If you want to create a
         new job with an explicit function object and arguments, you must call
         jobs.new() directly.
 
         The jobs.new docstring also describes in detail several important
         caveats associated with a thread-based model for background job
         execution.  Type jobs.new? for details.
 
         You can check the status of all jobs with jobs.status().
 
         The jobs variable is set by IPython into the Python builtin namespace.
         If you ever declare a variable named 'jobs', you will shadow this
         name.  You can either delete your global jobs variable to regain
         access to the job manager, or make a new name and assign it manually
         to the manager (stored in IPython's namespace).  For example, to
         assign the job manager to the Jobs name, use:
 
           Jobs = __builtins__.jobs"""
         
         self.shell.jobs.new(parameter_s,self.shell.user_ns)
         
     def magic_r(self, parameter_s=''):
         """Repeat previous input.
 
         Note: Consider using the more powerfull %rep instead!
         
         If given an argument, repeats the previous command which starts with
         the same string, otherwise it just repeats the previous input.
 
         Shell escaped commands (with ! as first character) are not recognized
         by this system, only pure python code and magic commands.
         """
 
         start = parameter_s.strip()
         esc_magic = self.shell.ESC_MAGIC
         # Identify magic commands even if automagic is on (which means
         # the in-memory version is different from that typed by the user).
         if self.shell.rc.automagic:
             start_magic = esc_magic+start
         else:
             start_magic = start
         # Look through the input history in reverse
         for n in range(len(self.shell.input_hist)-2,0,-1):
             input = self.shell.input_hist[n]
             # skip plain 'r' lines so we don't recurse to infinity
             if input != '_ip.magic("r")\n' and \
                    (input.startswith(start) or input.startswith(start_magic)):
                 #print 'match',`input`  # dbg
                 print 'Executing:',input,
                 self.shell.runlines(input)
                 return
         print 'No previous input matching `%s` found.' % start
 
     
     def magic_bookmark(self, parameter_s=''):
         """Manage IPython's bookmark system.
 
         %bookmark <name>       - set bookmark to current dir
         %bookmark <name> <dir> - set bookmark to <dir>
         %bookmark -l           - list all bookmarks
         %bookmark -d <name>    - remove bookmark
         %bookmark -r           - remove all bookmarks
 
         You can later on access a bookmarked folder with:
           %cd -b <name>
         or simply '%cd <name>' if there is no directory called <name> AND
         there is such a bookmark defined.
 
         Your bookmarks persist through IPython sessions, but they are
         associated with each profile."""
 
         opts,args = self.parse_options(parameter_s,'drl',mode='list')
         if len(args) > 2:
             raise UsageError("%bookmark: too many arguments")
 
         bkms = self.db.get('bookmarks',{})
             
         if opts.has_key('d'):
             try:
                 todel = args[0]
             except IndexError:
                 raise UsageError(
                     "%bookmark -d: must provide a bookmark to delete")
             else:
                 try:
                     del bkms[todel]
                 except KeyError:
                     raise UsageError(
                         "%%bookmark -d: Can't delete bookmark '%s'" % todel)
 
         elif opts.has_key('r'):
             bkms = {}
         elif opts.has_key('l'):
             bks = bkms.keys()
             bks.sort()
             if bks:
                 size = max(map(len,bks))
             else:
                 size = 0
             fmt = '%-'+str(size)+'s -> %s'
             print 'Current bookmarks:'
             for bk in bks:
                 print fmt % (bk,bkms[bk])
         else:
             if not args:
                 raise UsageError("%bookmark: You must specify the bookmark name")
             elif len(args)==1:
                 bkms[args[0]] = os.getcwd()
             elif len(args)==2:
                 bkms[args[0]] = args[1]
         self.db['bookmarks'] = bkms
 
     def magic_pycat(self, parameter_s=''):
         """Show a syntax-highlighted file through a pager.
 
         This magic is similar to the cat utility, but it will assume the file
         to be Python source and will show it with syntax highlighting. """
         
         try:
             filename = get_py_filename(parameter_s)
             cont = file_read(filename)
         except IOError:
             try:
                 cont = eval(parameter_s,self.user_ns)
             except NameError:
                 cont = None
         if cont is None:
             print "Error: no such file or variable"
             return
             
         page(self.shell.pycolorize(cont),
              screen_lines=self.shell.rc.screen_length)
 
     def magic_cpaste(self, parameter_s=''):
-        """Allows you to paste & execute a pre-formatted code block from clipboard
+        """Allows you to paste & execute a pre-formatted code block from clipboard.
         
         You must terminate the block with '--' (two minus-signs) alone on the
         line. You can also provide your own sentinel with '%paste -s %%' ('%%' 
         is the new sentinel for this operation)
         
         The block is dedented prior to execution to enable execution of method
         definitions. '>' and '+' characters at the beginning of a line are
-        ignored, to allow pasting directly from e-mails or diff files. The
+        ignored, to allow pasting directly from e-mails, diff files and
+        doctests (the '...' continuation prompt is also stripped).  The
         executed block is also assigned to variable named 'pasted_block' for
         later editing with '%edit pasted_block'.
         
         You can also pass a variable name as an argument, e.g. '%cpaste foo'.
         This assigns the pasted block to variable 'foo' as string, without 
-        dedenting or executing it.
+        dedenting or executing it (preceding >>> and + is still stripped)
         
         Do not be alarmed by garbled output on Windows (it's a readline bug). 
         Just press enter and type -- (and press enter again) and the block 
         will be what was just pasted.
         
         IPython statements (magics, shell escapes) are not supported (yet).
         """
         opts,args = self.parse_options(parameter_s,'s:',mode='string')
         par = args.strip()
         sentinel = opts.get('s','--')
+
+        # Regular expressions that declare text we strip from the input:
+        strip_re =  [r'^\s*In \[\d+\]:', # IPython input prompt
+                     r'^\s*(\s?>)+', # Python input prompt
+                     r'^\s*\.{3,}', # Continuation prompts
+                     r'^\++',
+                     ]
+
+        strip_from_start = map(re.compile,strip_re)
         
         from IPython import iplib
         lines = []
         print "Pasting code; enter '%s' alone on the line to stop." % sentinel
         while 1:
             l = iplib.raw_input_original(':')
             if l ==sentinel:
                 break
-            lines.append(l.lstrip('>').lstrip('+'))
+            
+            for pat in strip_from_start: 
+                l = pat.sub('',l)
+            lines.append(l)
+                         
         block = "\n".join(lines) + '\n'
         #print "block:\n",block
         if not par:
             b = textwrap.dedent(block)
             exec b in self.user_ns            
             self.user_ns['pasted_block'] = b
         else:
             self.user_ns[par] = block
             print "Block assigned to '%s'" % par
             
     def magic_quickref(self,arg):
         """ Show a quick reference sheet """
         import IPython.usage
         qr = IPython.usage.quick_reference + self.magic_magic('-brief')
         
         page(qr)
         
     def magic_upgrade(self,arg):
         """ Upgrade your IPython installation
         
         This will copy the config files that don't yet exist in your 
         ipython dir from the system config dir. Use this after upgrading 
         IPython if you don't wish to delete your .ipython dir.
 
         Call with -nolegacy to get rid of ipythonrc* files (recommended for
         new users)
 
         """
         ip = self.getapi()
         ipinstallation = path(IPython.__file__).dirname()
         upgrade_script = '%s "%s"' % (sys.executable,ipinstallation / 'upgrade_dir.py')
         src_config = ipinstallation / 'UserConfig'
         userdir = path(ip.options.ipythondir)
         cmd = '%s "%s" "%s"' % (upgrade_script, src_config, userdir)
         print ">",cmd
         shell(cmd)
         if arg == '-nolegacy':
             legacy = userdir.files('ipythonrc*')
             print "Nuking legacy files:",legacy
             
             [p.remove() for p in legacy]
             suffix = (sys.platform == 'win32' and '.ini' or '')
             (userdir / ('ipythonrc' + suffix)).write_text('# Empty, see ipy_user_conf.py\n')
 
 
     def magic_doctest_mode(self,parameter_s=''):
         """Toggle doctest mode on and off.
 
         This mode allows you to toggle the prompt behavior between normal
         IPython prompts and ones that are as similar to the default IPython
         interpreter as possible.
 
         It also supports the pasting of code snippets that have leading '>>>'
         and '...' prompts in them.  This means that you can paste doctests from
         files or docstrings (even if they have leading whitespace), and the
         code will execute correctly.  You can then use '%history -tn' to see
         the translated history without line numbers; this will give you the
         input after removal of all the leading prompts and whitespace, which
         can be pasted back into an editor.
 
         With these features, you can switch into this mode easily whenever you
         need to do testing and changes to doctests, without having to leave
         your existing IPython session.
         """
 
         # XXX - Fix this to have cleaner activate/deactivate calls.
         from IPython.Extensions import InterpreterPasteInput as ipaste
         from IPython.ipstruct import Struct
 
         # Shorthands
         shell = self.shell
         oc = shell.outputcache
         rc = shell.rc
         meta = shell.meta
         # dstore is a data store kept in the instance metadata bag to track any
         # changes we make, so we can undo them later.
         dstore = meta.setdefault('doctest_mode',Struct())
         save_dstore = dstore.setdefault
 
         # save a few values we'll need to recover later
         mode = save_dstore('mode',False)
         save_dstore('rc_pprint',rc.pprint)
         save_dstore('xmode',shell.InteractiveTB.mode)
         save_dstore('rc_separate_out',rc.separate_out)
         save_dstore('rc_separate_out2',rc.separate_out2)
         save_dstore('rc_prompts_pad_left',rc.prompts_pad_left)
 
         if mode == False:
             # turn on
             ipaste.activate_prefilter()
 
             oc.prompt1.p_template = '>>> '
             oc.prompt2.p_template = '... '
             oc.prompt_out.p_template = ''
 
             oc.output_sep = ''
             oc.output_sep2 = ''
 
             oc.prompt1.pad_left = oc.prompt2.pad_left = \
                                   oc.prompt_out.pad_left = False
 
             rc.pprint = False
 
             shell.magic_xmode('Plain')
 
         else:
             # turn off
             ipaste.deactivate_prefilter()
 
             oc.prompt1.p_template = rc.prompt_in1
             oc.prompt2.p_template = rc.prompt_in2
             oc.prompt_out.p_template = rc.prompt_out
 
             oc.output_sep = dstore.rc_separate_out
             oc.output_sep2 = dstore.rc_separate_out2
 
             oc.prompt1.pad_left = oc.prompt2.pad_left = \
                          oc.prompt_out.pad_left = dstore.rc_prompts_pad_left
 
             rc.pprint = dstore.rc_pprint
 
             shell.magic_xmode(dstore.xmode)
 
         # Store new mode and inform
         dstore.mode = bool(1-int(mode))
         print 'Doctest mode is:',
         print ['OFF','ON'][dstore.mode]
 
 # end Magic

IPython/OInspect.py

@@ -1,582 +1,593 b''
 # -*- coding: utf-8 -*-
 """Tools for inspecting Python objects.
 
 Uses syntax highlighting for presenting the various information elements.
 
 Similar in spirit to the inspect module, but all calls take a name argument to
 reference the name under which an object is being read.
 
 $Id: OInspect.py 2843 2007-10-15 21:22:32Z fperez $
 """
 
 #*****************************************************************************
 #       Copyright (C) 2001-2004 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Fernando']
 __license__ = Release.license
 
 __all__ = ['Inspector','InspectColors']
 
 # stdlib modules
 import __builtin__
 import inspect
 import linecache
 import string
 import StringIO
 import types
 import os
 import sys
 # IPython's own
 from IPython import PyColorize
 from IPython.genutils import page,indent,Term,mkdict
 from IPython.Itpl import itpl
 from IPython.wildcard import list_namespace
 from IPython.ColorANSI import *
 
 #****************************************************************************
 # HACK!!! This is a crude fix for bugs in python 2.3's inspect module.  We
 # simply monkeypatch inspect with code copied from python 2.4.
 if sys.version_info[:2] == (2,3):
     from inspect import ismodule, getabsfile, modulesbyfile
     def getmodule(object):
         """Return the module an object was defined in, or None if not found."""
         if ismodule(object):
             return object
         if hasattr(object, '__module__'):
             return sys.modules.get(object.__module__)
         try:
             file = getabsfile(object)
         except TypeError:
             return None
         if file in modulesbyfile:
             return sys.modules.get(modulesbyfile[file])
         for module in sys.modules.values():
             if hasattr(module, '__file__'):
                 modulesbyfile[
                     os.path.realpath(
                             getabsfile(module))] = module.__name__
         if file in modulesbyfile:
             return sys.modules.get(modulesbyfile[file])
         main = sys.modules['__main__']
         if not hasattr(object, '__name__'):
             return None
         if hasattr(main, object.__name__):
             mainobject = getattr(main, object.__name__)
             if mainobject is object:
                 return main
         builtin = sys.modules['__builtin__']
         if hasattr(builtin, object.__name__):
             builtinobject = getattr(builtin, object.__name__)
             if builtinobject is object:
                 return builtin
 
     inspect.getmodule = getmodule
 
 #****************************************************************************
 # Builtin color schemes
 
 Colors = TermColors  # just a shorthand
 
 # Build a few color schemes
 NoColor = ColorScheme(
     'NoColor',{
     'header' : Colors.NoColor,
     'normal' : Colors.NoColor  # color off (usu. Colors.Normal)
     }  )
 
 LinuxColors = ColorScheme(
     'Linux',{
     'header' : Colors.LightRed,
     'normal' : Colors.Normal  # color off (usu. Colors.Normal)
     } )
 
 LightBGColors = ColorScheme(
     'LightBG',{
     'header' : Colors.Red,
     'normal' : Colors.Normal  # color off (usu. Colors.Normal)
     }  )
 
 # Build table of color schemes (needed by the parser)
 InspectColors = ColorSchemeTable([NoColor,LinuxColors,LightBGColors],
                                  'Linux')
 
 #****************************************************************************
 # Auxiliary functions
 def getdoc(obj):
     """Stable wrapper around inspect.getdoc.
 
     This can't crash because of attribute problems.
 
     It also attempts to call a getdoc() method on the given object.  This
     allows objects which provide their docstrings via non-standard mechanisms
     (like Pyro proxies) to still be inspected by ipython's ? system."""
 
     ds = None  # default return value
     try:
         ds = inspect.getdoc(obj)
     except:
         # Harden against an inspect failure, which can occur with
         # SWIG-wrapped extensions.
         pass
     # Allow objects to offer customized documentation via a getdoc method:
     try:
         ds2 = obj.getdoc()
     except:
         pass
     else:
         # if we get extra info, we add it to the normal docstring.
         if ds is None:
             ds = ds2
         else:
             ds = '%s\n%s' % (ds,ds2)
     return ds
 
 def getsource(obj,is_binary=False):
     """Wrapper around inspect.getsource.
 
     This can be modified by other projects to provide customized source
     extraction.
 
     Inputs:
 
     - obj: an object whose source code we will attempt to extract.
 
     Optional inputs:
 
     - is_binary: whether the object is known to come from a binary source.
     This implementation will skip returning any output for binary objects, but
     custom extractors may know how to meaningfully process them."""
     
     if is_binary:
         return None
     else:
         return inspect.getsource(obj)
 
 #****************************************************************************
 # Class definitions
 
 class myStringIO(StringIO.StringIO):
     """Adds a writeln method to normal StringIO."""
     def writeln(self,*arg,**kw):
         """Does a write() and then a write('\n')"""
         self.write(*arg,**kw)
         self.write('\n')
 
 class Inspector:
     def __init__(self,color_table,code_color_table,scheme,
                  str_detail_level=0):
         self.color_table = color_table
         self.parser = PyColorize.Parser(code_color_table,out='str')
         self.format = self.parser.format
         self.str_detail_level = str_detail_level
         self.set_active_scheme(scheme)
 
     def __getargspec(self,obj):
         """Get the names and default values of a function's arguments.
 
         A tuple of four things is returned: (args, varargs, varkw, defaults).
         'args' is a list of the argument names (it may contain nested lists).
         'varargs' and 'varkw' are the names of the * and ** arguments or None.
         'defaults' is an n-tuple of the default values of the last n arguments.
 
         Modified version of inspect.getargspec from the Python Standard
         Library."""
 
         if inspect.isfunction(obj):
             func_obj = obj
         elif inspect.ismethod(obj):
             func_obj = obj.im_func
         else:
             raise TypeError, 'arg is not a Python function'
         args, varargs, varkw = inspect.getargs(func_obj.func_code)
         return args, varargs, varkw, func_obj.func_defaults
 
     def __getdef(self,obj,oname=''):
         """Return the definition header for any callable object.
 
         If any exception is generated, None is returned instead and the
         exception is suppressed."""
         
         try:
             return oname + inspect.formatargspec(*self.__getargspec(obj))
         except:
             return None
  
     def __head(self,h):
         """Return a header string with proper colors."""
         return '%s%s%s' % (self.color_table.active_colors.header,h,
                            self.color_table.active_colors.normal)
 
     def set_active_scheme(self,scheme):
         self.color_table.set_active_scheme(scheme)
         self.parser.color_table.set_active_scheme(scheme)
     
     def noinfo(self,msg,oname):
         """Generic message when no information is found."""
         print 'No %s found' % msg,
         if oname:
             print 'for %s' % oname
         else:
             print
             
     def pdef(self,obj,oname=''):
         """Print the definition header for any callable object.
 
         If the object is a class, print the constructor information."""
 
         if not callable(obj):
             print 'Object is not callable.'
             return
 
         header = ''
 
         if inspect.isclass(obj):
             header = self.__head('Class constructor information:\n')
             obj = obj.__init__
         elif type(obj) is types.InstanceType:
             obj = obj.__call__
 
         output = self.__getdef(obj,oname)
         if output is None:
             self.noinfo('definition header',oname)
         else:
             print >>Term.cout, header,self.format(output),
 
     def pdoc(self,obj,oname='',formatter = None):
         """Print the docstring for any object.
 
         Optional:
         -formatter: a function to run the docstring through for specially
         formatted docstrings."""
 
         head = self.__head  # so that itpl can find it even if private
         ds = getdoc(obj)
         if formatter:
             ds = formatter(ds)
         if inspect.isclass(obj):
             init_ds = getdoc(obj.__init__)
             output = itpl('$head("Class Docstring:")\n'
                           '$indent(ds)\n'
                           '$head("Constructor Docstring"):\n'
                           '$indent(init_ds)')
         elif (type(obj) is types.InstanceType or isinstance(obj,object)) \
                  and hasattr(obj,'__call__'):
             call_ds = getdoc(obj.__call__)
             if call_ds:
                 output = itpl('$head("Class Docstring:")\n$indent(ds)\n'
                               '$head("Calling Docstring:")\n$indent(call_ds)')
             else:
                 output = ds
         else:
             output = ds
         if output is None:
             self.noinfo('documentation',oname)
             return
         page(output)
 
     def psource(self,obj,oname=''):
         """Print the source code for an object."""
 
         # Flush the source cache because inspect can return out-of-date source
         linecache.checkcache()
         try:
             src = getsource(obj) 
         except:
             self.noinfo('source',oname)
         else:
             page(self.format(src))
 
     def pfile(self,obj,oname=''):
         """Show the whole file where an object was defined."""
         try:
             sourcelines,lineno = inspect.getsourcelines(obj)
         except:
             self.noinfo('file',oname)
         else:
             # run contents of file through pager starting at line
             # where the object is defined
             ofile = inspect.getabsfile(obj)
             
             if (ofile.endswith('.so') or ofile.endswith('.dll')):
                 print 'File %r is binary, not printing.' % ofile
             elif not os.path.isfile(ofile):
                 print 'File %r does not exist, not printing.' % ofile
             else:
                 # Print only text files, not extension binaries.
                 page(self.format(open(ofile).read()),lineno)
             #page(self.format(open(inspect.getabsfile(obj)).read()),lineno)
 
     def pinfo(self,obj,oname='',formatter=None,info=None,detail_level=0):
         """Show detailed information about an object.
 
         Optional arguments:
         
         - oname: name of the variable pointing to the object.
 
         - formatter: special formatter for docstrings (see pdoc)
 
         - info: a structure with some information fields which may have been
         precomputed already.
 
         - detail_level: if set to 1, more information is given.
         """
 
         obj_type = type(obj)
 
         header = self.__head
         if info is None:
             ismagic = 0
             isalias = 0
             ospace = ''
         else:
             ismagic = info.ismagic
             isalias = info.isalias
             ospace = info.namespace
         # Get docstring, special-casing aliases:
         if isalias:
             if not callable(obj):
                 try:
                     ds = "Alias to the system command:\n  %s" % obj[1]
                 except:
                     ds = "Alias: " + str(obj)
             else:
                 ds = "Alias to " + str(obj)
                 if obj.__doc__:
                     ds += "\nDocstring:\n" + obj.__doc__
         else:
             ds = getdoc(obj)
             if ds is None:
                 ds = '<no docstring>'
         if formatter is not None:
             ds = formatter(ds)
 
         # store output in a list which gets joined with \n at the end.
         out = myStringIO()
         
         string_max = 200 # max size of strings to show (snipped if longer)
         shalf = int((string_max -5)/2)
 
         if ismagic:
             obj_type_name = 'Magic function'
         elif isalias:
             obj_type_name = 'System alias'
         else:
             obj_type_name = obj_type.__name__
         out.writeln(header('Type:\t\t')+obj_type_name)
 
         try:
             bclass = obj.__class__
             out.writeln(header('Base Class:\t')+str(bclass))
         except: pass
 
         # String form, but snip if too long in ? form (full in ??)
         if detail_level >= self.str_detail_level:
             try:
                 ostr = str(obj)
                 str_head = 'String Form:'
                 if not detail_level and len(ostr)>string_max:
                     ostr = ostr[:shalf] + ' <...> ' + ostr[-shalf:]
                     ostr = ("\n" + " " * len(str_head.expandtabs())).\
                            join(map(string.strip,ostr.split("\n")))
                 if ostr.find('\n') > -1:
                     # Print multi-line strings starting at the next line.
                     str_sep = '\n'
                 else:
                     str_sep = '\t'
                 out.writeln("%s%s%s" % (header(str_head),str_sep,ostr))
             except:
                 pass
 
         if ospace:
             out.writeln(header('Namespace:\t')+ospace)
 
         # Length (for strings and lists)
         try:
             length = str(len(obj))
             out.writeln(header('Length:\t\t')+length)
         except: pass
 
         # Filename where object was defined
         binary_file = False
         try:
-            fname = inspect.getabsfile(obj)
+            try:
+                fname = inspect.getabsfile(obj)
+            except TypeError:
+                # For an instance, the file that matters is where its class was
+                # declared.
+                if hasattr(obj,'__class__'):
+                    fname = inspect.getabsfile(obj.__class__)
             if fname.endswith('<string>'):
                 fname = 'Dynamically generated function. No source code available.'
             if (fname.endswith('.so') or fname.endswith('.dll')):
                 binary_file = True
             out.writeln(header('File:\t\t')+fname)
         except:
             # if anything goes wrong, we don't want to show source, so it's as
             # if the file was binary
             binary_file = True
 
         # reconstruct the function definition and print it:
         defln = self.__getdef(obj,oname)
         if defln:
             out.write(header('Definition:\t')+self.format(defln))
  
         # Docstrings only in detail 0 mode, since source contains them (we
         # avoid repetitions).  If source fails, we add them back, see below.
         if ds and detail_level == 0:
                 out.writeln(header('Docstring:\n') + indent(ds))
                 
         # Original source code for any callable
         if detail_level:
             # Flush the source cache because inspect can return out-of-date
             # source
             linecache.checkcache()
             source_success = False
             try:
-                source = self.format(getsource(obj,binary_file))
-                if source:
+                try:
+                    src = getsource(obj,binary_file)
+                except TypeError:
+                    if hasattr(obj,'__class__'):
+                        src = getsource(obj.__class__,binary_file)
+                if src is not None:
+                    source = self.format(src)
                     out.write(header('Source:\n')+source.rstrip())
                     source_success = True
             except Exception, msg:
                 pass
             
             if ds and not source_success:
                 out.writeln(header('Docstring [source file open failed]:\n')
                             + indent(ds))
 
         # Constructor docstring for classes
         if inspect.isclass(obj):
             # reconstruct the function definition and print it:
             try:
                 obj_init =  obj.__init__
             except AttributeError:
                 init_def = init_ds = None
             else:
                 init_def = self.__getdef(obj_init,oname)
                 init_ds  = getdoc(obj_init)
                 # Skip Python's auto-generated docstrings
                 if init_ds and \
                        init_ds.startswith('x.__init__(...) initializes'):
                     init_ds = None
 
             if init_def or init_ds:
                 out.writeln(header('\nConstructor information:'))
                 if init_def:
                     out.write(header('Definition:\t')+ self.format(init_def))
                 if init_ds:
                     out.writeln(header('Docstring:\n') + indent(init_ds))
         # and class docstring for instances:
         elif obj_type is types.InstanceType or \
                  isinstance(obj,object):
 
             # First, check whether the instance docstring is identical to the
             # class one, and print it separately if they don't coincide.  In
             # most cases they will, but it's nice to print all the info for
             # objects which use instance-customized docstrings.
             if ds:
                 try:
                     cls = getattr(obj,'__class__')
                 except:
                     class_ds = None
                 else:
                     class_ds = getdoc(cls)
                 # Skip Python's auto-generated docstrings
                 if class_ds and \
                        (class_ds.startswith('function(code, globals[,') or \
                    class_ds.startswith('instancemethod(function, instance,') or \
                    class_ds.startswith('module(name[,') ):
                     class_ds = None
                 if class_ds and ds != class_ds:
                     out.writeln(header('Class Docstring:\n') +
                                 indent(class_ds))
 
             # Next, try to show constructor docstrings
             try:
                 init_ds = getdoc(obj.__init__)
                 # Skip Python's auto-generated docstrings
                 if init_ds and \
                        init_ds.startswith('x.__init__(...) initializes'):
                     init_ds = None
             except AttributeError:
                 init_ds = None
             if init_ds:
                 out.writeln(header('Constructor Docstring:\n') +
                             indent(init_ds))
 
             # Call form docstring for callable instances
             if hasattr(obj,'__call__'):
                 #out.writeln(header('Callable:\t')+'Yes')
                 call_def = self.__getdef(obj.__call__,oname)
                 #if call_def is None:
                 #    out.writeln(header('Call def:\t')+
                 #              'Calling definition not available.')
                 if call_def is not None:
                     out.writeln(header('Call def:\t')+self.format(call_def))
                 call_ds = getdoc(obj.__call__)
                 # Skip Python's auto-generated docstrings
                 if call_ds and call_ds.startswith('x.__call__(...) <==> x(...)'):
                     call_ds = None
                 if call_ds:
                     out.writeln(header('Call docstring:\n') + indent(call_ds))
 
         # Finally send to printer/pager
         output = out.getvalue()
         if output:
             page(output)
         # end pinfo
 
     def psearch(self,pattern,ns_table,ns_search=[],
                 ignore_case=False,show_all=False):
         """Search namespaces with wildcards for objects.
 
         Arguments:
 
         - pattern: string containing shell-like wildcards to use in namespace
         searches and optionally a type specification to narrow the search to
         objects of that type.
 
         - ns_table: dict of name->namespaces for search.
 
         Optional arguments:
         
           - ns_search: list of namespace names to include in search.
 
           - ignore_case(False): make the search case-insensitive.
 
           - show_all(False): show all names, including those starting with
           underscores.
         """
         #print 'ps pattern:<%r>' % pattern # dbg
         
         # defaults
         type_pattern = 'all'
         filter = ''
 
         cmds = pattern.split()
         len_cmds  =  len(cmds)
         if len_cmds == 1:
             # Only filter pattern given
             filter = cmds[0]
         elif len_cmds == 2:
             # Both filter and type specified
             filter,type_pattern = cmds
         else:
             raise ValueError('invalid argument string for psearch: <%s>' %
                              pattern)
 
         # filter search namespaces
         for name in ns_search:
             if name not in ns_table:
                 raise ValueError('invalid namespace <%s>. Valid names: %s' %
                                  (name,ns_table.keys()))
 
         #print 'type_pattern:',type_pattern # dbg
         search_result = []
         for ns_name in ns_search:
             ns = ns_table[ns_name]
             tmp_res = list(list_namespace(ns,type_pattern,filter,
                                           ignore_case=ignore_case,
                                           show_all=show_all))
             search_result.extend(tmp_res)
         search_result.sort()
 
         page('\n'.join(search_result))

IPython/Release.py

@@ -1,83 +1,89 b''
 # -*- coding: utf-8 -*-
 """Release data for the IPython project.
 
 $Id: Release.py 3002 2008-02-01 07:17:00Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
 #
 #       Copyright (c) 2001 Janko Hauser <[e-mail]> and Nathaniel Gray
 #       <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 # Name of the package for release purposes.  This is the name which labels
 # the tarballs and RPMs made by distutils, so it's best to lowercase it.
 name = 'ipython'
 
 # For versions with substrings (like 0.6.16.svn), use an extra . to separate
 # the new substring.  We have to avoid using either dashes or underscores,
 # because bdist_rpm does not accept dashes (an RPM) convention, and
 # bdist_deb does not accept underscores (a Debian convention).
 
-revision = '3001'
+revision = '128'
+branch = 'ipython'
 
-version = '0.8.3.svn.r' + revision.rstrip('M')
+if branch == 'ipython':
+    version = '0.8.4.bzr.r' + revision
+else:
+    version = '0.8.4.bzr.r%s.%s'  % (revision,branch)
+
+version = '0.8.4'
 
 description = "An enhanced interactive Python shell."
 
 long_description = \
 """
 IPython provides a replacement for the interactive Python interpreter with
 extra functionality.
 
 Main features:
 
  * Comprehensive object introspection.
 
  * Input history, persistent across sessions.
 
  * Caching of output results during a session with automatically generated
    references.
 
  * Readline based name completion.
 
  * Extensible system of 'magic' commands for controlling the environment and
    performing many tasks related either to IPython or the operating system.
 
  * Configuration system with easy switching between different setups (simpler
    than changing $PYTHONSTARTUP environment variables every time).
 
  * Session logging and reloading.
 
  * Extensible syntax processing for special purpose situations.
 
  * Access to the system shell with user-extensible alias system.
 
  * Easily embeddable in other Python programs.
 
  * Integrated access to the pdb debugger and the Python profiler. 
 
  The latest development version is always available at the IPython subversion
  repository_.
 
 .. _repository: http://ipython.scipy.org/svn/ipython/ipython/trunk#egg=ipython-dev
  """
 
 license = 'BSD'
 
 authors = {'Fernando' : ('Fernando Perez','[e-mail]'),
            'Janko'    : ('Janko Hauser','[e-mail]'),
            'Nathan'   : ('Nathaniel Gray','[e-mail]'),
            'Ville'    : ('Ville Vainio','[e-mail]')           
            }
 
 url = 'http://ipython.scipy.org'
 
 download_url = 'http://ipython.scipy.org/dist'
 
 platforms = ['Linux','Mac OSX','Windows XP/2000/NT','Windows 95/98/ME']
 
 keywords = ['Interactive','Interpreter','Shell']

IPython/Shell.py

@@ -1,1205 +1,1225 b''
 # -*- coding: utf-8 -*-
 """IPython Shell classes.
 
 All the matplotlib support code was co-developed with John Hunter,
 matplotlib's author.
 
 $Id: Shell.py 3024 2008-02-07 15:34:42Z darren.dale $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Fernando']
 __license__ = Release.license
 
 # Code begins
 # Stdlib imports
 import __builtin__
 import __main__
 import Queue
 import inspect
 import os
 import sys
 import thread
 import threading
 import time
 
 from signal import signal, SIGINT
 
 try:
     import ctypes
     HAS_CTYPES = True
 except ImportError:
     HAS_CTYPES = False
 
 # IPython imports
 import IPython
 from IPython import ultraTB, ipapi
 from IPython.genutils import Term,warn,error,flag_calls, ask_yes_no
 from IPython.iplib import InteractiveShell
 from IPython.ipmaker import make_IPython
 from IPython.Magic import Magic
 from IPython.ipstruct import Struct
 
+try: # Python 2.3 compatibility
+    set
+except NameError:
+    import sets
+    set = sets.Set
+
+
 # Globals
 # global flag to pass around information about Ctrl-C without exceptions
 KBINT = False
 
 # global flag to turn on/off Tk support.
 USE_TK = False
 
 # ID for the main thread, used for cross-thread exceptions
 MAIN_THREAD_ID = thread.get_ident()
 
 # Tag when runcode() is active, for exception handling
 CODE_RUN = None
 
 #-----------------------------------------------------------------------------
 # This class is trivial now, but I want to have it in to publish a clean
 # interface. Later when the internals are reorganized, code that uses this
 # shouldn't have to change.
 
 class IPShell:
     """Create an IPython instance."""
     
     def __init__(self,argv=None,user_ns=None,user_global_ns=None,
                  debug=1,shell_class=InteractiveShell):
         self.IP = make_IPython(argv,user_ns=user_ns,
                                user_global_ns=user_global_ns,
                                debug=debug,shell_class=shell_class)
 
     def mainloop(self,sys_exit=0,banner=None):
         self.IP.mainloop(banner)
         if sys_exit:
             sys.exit()
 
 #-----------------------------------------------------------------------------
 def kill_embedded(self,parameter_s=''):
     """%kill_embedded : deactivate for good the current embedded IPython.
 
     This function (after asking for confirmation) sets an internal flag so that
     an embedded IPython will never activate again.  This is useful to
     permanently disable a shell that is being called inside a loop: once you've
     figured out what you needed from it, you may then kill it and the program
     will then continue to run without the interactive shell interfering again.
     """
     
     kill = ask_yes_no("Are you sure you want to kill this embedded instance "
                      "(y/n)? [y/N] ",'n')
     if kill:
         self.shell.embedded_active = False
         print "This embedded IPython will not reactivate anymore once you exit."
     
 class IPShellEmbed:
     """Allow embedding an IPython shell into a running program.
 
     Instances of this class are callable, with the __call__ method being an
     alias to the embed() method of an InteractiveShell instance.
 
     Usage (see also the example-embed.py file for a running example):
 
     ipshell = IPShellEmbed([argv,banner,exit_msg,rc_override])
 
     - argv: list containing valid command-line options for IPython, as they
     would appear in sys.argv[1:].
 
     For example, the following command-line options:
 
       $ ipython -prompt_in1 'Input <\\#>' -colors LightBG
 
     would be passed in the argv list as:
 
       ['-prompt_in1','Input <\\#>','-colors','LightBG']
 
     - banner: string which gets printed every time the interpreter starts.
 
     - exit_msg: string which gets printed every time the interpreter exits.
 
     - rc_override: a dict or Struct of configuration options such as those
     used by IPython. These options are read from your ~/.ipython/ipythonrc
     file when the Shell object is created. Passing an explicit rc_override
     dict with any options you want allows you to override those values at
     creation time without having to modify the file. This way you can create
     embeddable instances configured in any way you want without editing any
     global files (thus keeping your interactive IPython configuration
     unchanged).
 
     Then the ipshell instance can be called anywhere inside your code:
     
     ipshell(header='') -> Opens up an IPython shell.
 
     - header: string printed by the IPython shell upon startup. This can let
     you know where in your code you are when dropping into the shell. Note
     that 'banner' gets prepended to all calls, so header is used for
     location-specific information.
 
     For more details, see the __call__ method below.
 
     When the IPython shell is exited with Ctrl-D, normal program execution
     resumes.
 
     This functionality was inspired by a posting on comp.lang.python by cmkl
     <[e-mail]> on Dec. 06/01 concerning similar uses of pyrepl, and
     by the IDL stop/continue commands."""
 
     def __init__(self,argv=None,banner='',exit_msg=None,rc_override=None,
                  user_ns=None):
         """Note that argv here is a string, NOT a list."""
         self.set_banner(banner)
         self.set_exit_msg(exit_msg)
         self.set_dummy_mode(0)
 
         # sys.displayhook is a global, we need to save the user's original
         # Don't rely on __displayhook__, as the user may have changed that.
         self.sys_displayhook_ori = sys.displayhook
 
         # save readline completer status
         try:
             #print 'Save completer',sys.ipcompleter  # dbg
             self.sys_ipcompleter_ori = sys.ipcompleter
         except:
             pass # not nested with IPython
         
         self.IP = make_IPython(argv,rc_override=rc_override,
                                embedded=True,
                                user_ns=user_ns)
 
         ip = ipapi.IPApi(self.IP)
         ip.expose_magic("kill_embedded",kill_embedded)
 
         # copy our own displayhook also
         self.sys_displayhook_embed = sys.displayhook
         # and leave the system's display hook clean
         sys.displayhook = self.sys_displayhook_ori
         # don't use the ipython crash handler so that user exceptions aren't
         # trapped
         sys.excepthook = ultraTB.FormattedTB(color_scheme = self.IP.rc.colors,
                                              mode = self.IP.rc.xmode,
                                              call_pdb = self.IP.rc.pdb)
         self.restore_system_completer()
 
     def restore_system_completer(self):
         """Restores the readline completer which was in place.
 
         This allows embedded IPython within IPython not to disrupt the
         parent's completion.
         """
         
         try:
             self.IP.readline.set_completer(self.sys_ipcompleter_ori)
             sys.ipcompleter = self.sys_ipcompleter_ori
         except:
             pass
 
     def __call__(self,header='',local_ns=None,global_ns=None,dummy=None):
         """Activate the interactive interpreter.
 
         __call__(self,header='',local_ns=None,global_ns,dummy=None) -> Start
         the interpreter shell with the given local and global namespaces, and
         optionally print a header string at startup.
 
         The shell can be globally activated/deactivated using the
         set/get_dummy_mode methods. This allows you to turn off a shell used
         for debugging globally.
 
         However, *each* time you call the shell you can override the current
         state of dummy_mode with the optional keyword parameter 'dummy'. For
         example, if you set dummy mode on with IPShell.set_dummy_mode(1), you
         can still have a specific call work by making it as IPShell(dummy=0).
 
         The optional keyword parameter dummy controls whether the call
         actually does anything.  """
 
         # If the user has turned it off, go away
         if not self.IP.embedded_active:
             return
 
         # Normal exits from interactive mode set this flag, so the shell can't
         # re-enter (it checks this variable at the start of interactive mode).
         self.IP.exit_now = False
 
         # Allow the dummy parameter to override the global __dummy_mode
         if dummy or (dummy != 0 and self.__dummy_mode):
             return
 
         # Set global subsystems (display,completions) to our values
         sys.displayhook = self.sys_displayhook_embed
         if self.IP.has_readline:
             self.IP.set_completer()
 
         if self.banner and header:
             format = '%s\n%s\n'
         else:
             format = '%s%s\n'
         banner =  format % (self.banner,header)
 
         # Call the embedding code with a stack depth of 1 so it can skip over
         # our call and get the original caller's namespaces.
         self.IP.embed_mainloop(banner,local_ns,global_ns,stack_depth=1)
 
         if self.exit_msg:
             print self.exit_msg
             
         # Restore global systems (display, completion)
         sys.displayhook = self.sys_displayhook_ori
         self.restore_system_completer()
     
     def set_dummy_mode(self,dummy):
         """Sets the embeddable shell's dummy mode parameter.
 
         set_dummy_mode(dummy): dummy = 0 or 1.
 
         This parameter is persistent and makes calls to the embeddable shell
         silently return without performing any action. This allows you to
         globally activate or deactivate a shell you're using with a single call.
 
         If you need to manually"""
 
         if dummy not in [0,1,False,True]:
             raise ValueError,'dummy parameter must be boolean'
         self.__dummy_mode = dummy
 
     def get_dummy_mode(self):
         """Return the current value of the dummy mode parameter.
         """
         return self.__dummy_mode
     
     def set_banner(self,banner):
         """Sets the global banner.
 
         This banner gets prepended to every header printed when the shell
         instance is called."""
 
         self.banner = banner
 
     def set_exit_msg(self,exit_msg):
         """Sets the global exit_msg.
 
         This exit message gets printed upon exiting every time the embedded
         shell is called. It is None by default. """
 
         self.exit_msg = exit_msg
 
 #-----------------------------------------------------------------------------
 if HAS_CTYPES:
     #  Add async exception support.  Trick taken from:
     # http://sebulba.wikispaces.com/recipe+thread2
     def _async_raise(tid, exctype):
         """raises the exception, performs cleanup if needed"""
         if not inspect.isclass(exctype):
             raise TypeError("Only types can be raised (not instances)")
         res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid,
                                                          ctypes.py_object(exctype))
         if res == 0:
             raise ValueError("invalid thread id")
         elif res != 1:
             # """if it returns a number greater than one, you're in trouble, 
             # and you should call it again with exc=NULL to revert the effect"""
             ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, 0)
             raise SystemError("PyThreadState_SetAsyncExc failed")
 
     def sigint_handler (signum,stack_frame):
         """Sigint handler for threaded apps.
 
         This is a horrible hack to pass information about SIGINT _without_
         using exceptions, since I haven't been able to properly manage
         cross-thread exceptions in GTK/WX.  In fact, I don't think it can be
         done (or at least that's my understanding from a c.l.py thread where
         this was discussed)."""
 
         global KBINT
 
         if CODE_RUN:
             _async_raise(MAIN_THREAD_ID,KeyboardInterrupt)
         else:
             KBINT = True
             print '\nKeyboardInterrupt - Press <Enter> to continue.',
             Term.cout.flush()
 
 else:
     def sigint_handler (signum,stack_frame):
         """Sigint handler for threaded apps.
 
         This is a horrible hack to pass information about SIGINT _without_
         using exceptions, since I haven't been able to properly manage
         cross-thread exceptions in GTK/WX.  In fact, I don't think it can be
         done (or at least that's my understanding from a c.l.py thread where
         this was discussed)."""
 
         global KBINT
 
         print '\nKeyboardInterrupt - Press <Enter> to continue.',
         Term.cout.flush()
         # Set global flag so that runsource can know that Ctrl-C was hit
         KBINT = True
 
 
 class MTInteractiveShell(InteractiveShell):
     """Simple multi-threaded shell."""
 
     # Threading strategy taken from:
     # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65109, by Brian
     # McErlean and John Finlay.  Modified with corrections by Antoon Pardon,
     # from the pygtk mailing list, to avoid lockups with system calls.
 
     # class attribute to indicate whether the class supports threads or not.
     # Subclasses with thread support should override this as needed.
     isthreaded = True
 
     def __init__(self,name,usage=None,rc=Struct(opts=None,args=None),
                  user_ns=None,user_global_ns=None,banner2='',**kw):
         """Similar to the normal InteractiveShell, but with threading control"""
         
         InteractiveShell.__init__(self,name,usage,rc,user_ns,
                                   user_global_ns,banner2)
 
-        # Locking control variable.
-        self.thread_ready = threading.Condition(threading.RLock())
 
-        # A queue to hold the code to be executed.  A scalar variable is NOT
-        # enough, because uses like macros cause reentrancy.
+        # A queue to hold the code to be executed. 
         self.code_queue = Queue.Queue()
 
         # Stuff to do at closing time
-        self._kill = False
-        on_kill = kw.get('on_kill')
-        if on_kill is None:
-            on_kill = []
+        self._kill = None
+        on_kill = kw.get('on_kill', [])
         # Check that all things to kill are callable:
         for t in on_kill:
             if not callable(t):
                 raise TypeError,'on_kill must be a list of callables'
         self.on_kill = on_kill
-
+        # thread identity of the "worker thread" (that may execute code directly)
+        self.worker_ident = None
+        
     def runsource(self, source, filename="<input>", symbol="single"):
         """Compile and run some source in the interpreter.
 
         Modified version of code.py's runsource(), to handle threading issues.
         See the original for full docstring details."""
-
+        
         global KBINT
         
         # If Ctrl-C was typed, we reset the flag and return right away
         if KBINT:
             KBINT = False
             return False
+
+        if self._kill:
+            # can't queue new code if we are being killed
+            return True
         
         try:
             code = self.compile(source, filename, symbol)
         except (OverflowError, SyntaxError, ValueError):
             # Case 1
             self.showsyntaxerror(filename)
             return False
 
         if code is None:
             # Case 2
             return True
 
+        # shortcut - if we are in worker thread, or the worker thread is not running, 
+        # execute directly (to allow recursion and prevent deadlock if code is run early 
+        # in IPython construction)
+        
+        if (self.worker_ident is None or self.worker_ident == thread.get_ident()):
+            InteractiveShell.runcode(self,code)
+            return
+
         # Case 3
         # Store code in queue, so the execution thread can handle it.
 
-        # Note that with macros and other applications, we MAY re-enter this
-        # section, so we have to acquire the lock with non-blocking semantics,
-        # else we deadlock.
-        got_lock = self.thread_ready.acquire()
-        self.code_queue.put(code)
-        if got_lock:
-           self.thread_ready.wait()  # Wait until processed in timeout interval
-        self.thread_ready.release()
-
+        completed_ev, received_ev = threading.Event(), threading.Event() 
+        
+        self.code_queue.put((code,completed_ev, received_ev))
+        # first make sure the message was received, with timeout
+        received_ev.wait(5)
+        if not received_ev.isSet():
+            # the mainloop is dead, start executing code directly
+            print "Warning: Timeout for mainloop thread exceeded"
+            print "switching to nonthreaded mode (until mainloop wakes up again)"
+            self.worker_ident = None
+        else:
+            completed_ev.wait()
         return False
 
     def runcode(self):
         """Execute a code object.
 
         Multithreaded wrapper around IPython's runcode()."""
-
+        
         global CODE_RUN
-
-        # lock thread-protected stuff
-        got_lock = self.thread_ready.acquire()
+        
+        # we are in worker thread, stash out the id for runsource() 
+        self.worker_ident = thread.get_ident()
 
         if self._kill:
             print >>Term.cout, 'Closing threads...',
             Term.cout.flush()
             for tokill in self.on_kill:
                 tokill()
             print >>Term.cout, 'Done.'
+            # allow kill() to return
+            self._kill.set()
+            return True
 
         # Install sigint handler.  We do it every time to ensure that if user
         # code modifies it, we restore our own handling.
         try:
             signal(SIGINT,sigint_handler)
         except SystemError:
             # This happens under Windows, which seems to have all sorts
             # of problems with signal handling.  Oh well...
             pass
 
         # Flush queue of pending code by calling the run methood of the parent
         # class with all items which may be in the queue.
         code_to_run = None
         while 1:
             try:
-                code_to_run = self.code_queue.get_nowait()
+                code_to_run, completed_ev, received_ev = self.code_queue.get_nowait()                
             except Queue.Empty:
                 break
-
+            received_ev.set()
+            
             # Exceptions need to be raised differently depending on which
             # thread is active.  This convoluted try/except is only there to
             # protect against asynchronous exceptions, to ensure that a KBINT
             # at the wrong time doesn't deadlock everything.  The global
             # CODE_TO_RUN is set to true/false as close as possible to the
             # runcode() call, so that the KBINT handler is correctly informed.
             try:
                 try:
                    CODE_RUN = True
                    InteractiveShell.runcode(self,code_to_run)
                 except KeyboardInterrupt:
                    print "Keyboard interrupted in mainloop"
                    while not self.code_queue.empty():
-                      self.code_queue.get_nowait()
+                      code, ev1,ev2 = self.code_queue.get_nowait()
+                      ev1.set()
+                      ev2.set()                      
                    break
             finally:
-               if got_lock:
-                  CODE_RUN = False
-
-        # We're done with thread-protected variables
-        if code_to_run is not None:
-           self.thread_ready.notify()
-        self.thread_ready.release()
-
-        # We're done...
-        CODE_RUN = False
+                CODE_RUN = False
+                # allow runsource() return from wait
+                completed_ev.set()
+                
+        
         # This MUST return true for gtk threading to work
         return True
 
     def kill(self):
         """Kill the thread, returning when it has been shut down."""
-        got_lock = self.thread_ready.acquire(False)
-        self._kill = True
-        if got_lock:
-            self.thread_ready.release()
+        self._kill = threading.Event()
+        self._kill.wait()
 
 class MatplotlibShellBase:
     """Mixin class to provide the necessary modifications to regular IPython
     shell classes for matplotlib support.
 
     Given Python's MRO, this should be used as the FIRST class in the
     inheritance hierarchy, so that it overrides the relevant methods."""
     
     def _matplotlib_config(self,name,user_ns):
         """Return items needed to setup the user's shell with matplotlib"""
 
         # Initialize matplotlib to interactive mode always
         import matplotlib
         from matplotlib import backends
         matplotlib.interactive(True)
 
         def use(arg):
             """IPython wrapper for matplotlib's backend switcher.
 
             In interactive use, we can not allow switching to a different
             interactive backend, since thread conflicts will most likely crash
             the python interpreter.  This routine does a safety check first,
             and refuses to perform a dangerous switch.  It still allows
             switching to non-interactive backends."""
 
             if arg in backends.interactive_bk and arg != self.mpl_backend:
                 m=('invalid matplotlib backend switch.\n'
                    'This script attempted to switch to the interactive '
                    'backend: `%s`\n'
                    'Your current choice of interactive backend is: `%s`\n\n'
                    'Switching interactive matplotlib backends at runtime\n'
                    'would crash the python interpreter, '
                    'and IPython has blocked it.\n\n'
                    'You need to either change your choice of matplotlib backend\n'
                    'by editing your .matplotlibrc file, or run this script as a \n'
                    'standalone file from the command line, not using IPython.\n' %
                    (arg,self.mpl_backend) )
                 raise RuntimeError, m
             else:
                 self.mpl_use(arg)
                 self.mpl_use._called = True
         
         self.matplotlib = matplotlib
         self.mpl_backend = matplotlib.rcParams['backend']
 
         # we also need to block switching of interactive backends by use()
         self.mpl_use = matplotlib.use
         self.mpl_use._called = False
         # overwrite the original matplotlib.use with our wrapper
         matplotlib.use = use
 
         # This must be imported last in the matplotlib series, after
         # backend/interactivity choices have been made
         import matplotlib.pylab as pylab
         self.pylab = pylab
 
         self.pylab.show._needmain = False
         # We need to detect at runtime whether show() is called by the user.
         # For this, we wrap it into a decorator which adds a 'called' flag.
         self.pylab.draw_if_interactive = flag_calls(self.pylab.draw_if_interactive)
 
         # Build a user namespace initialized with matplotlib/matlab features.
         user_ns = IPython.ipapi.make_user_ns(user_ns)
 
         exec ("import matplotlib\n"
               "import matplotlib.pylab as pylab\n") in user_ns
         
         # Build matplotlib info banner
         b="""
   Welcome to pylab, a matplotlib-based Python environment.
   For more information, type 'help(pylab)'.
 """
         return user_ns,b
 
     def mplot_exec(self,fname,*where,**kw):
         """Execute a matplotlib script.
 
         This is a call to execfile(), but wrapped in safeties to properly
         handle interactive rendering and backend switching."""
 
         #print '*** Matplotlib runner ***' # dbg
         # turn off rendering until end of script
         isInteractive = self.matplotlib.rcParams['interactive']
         self.matplotlib.interactive(False)
         self.safe_execfile(fname,*where,**kw)
         self.matplotlib.interactive(isInteractive)
         # make rendering call now, if the user tried to do it
         if self.pylab.draw_if_interactive.called:
             self.pylab.draw()
             self.pylab.draw_if_interactive.called = False
                 
         # if a backend switch was performed, reverse it now
         if self.mpl_use._called:
             self.matplotlib.rcParams['backend'] = self.mpl_backend
         
     def magic_run(self,parameter_s=''):
         Magic.magic_run(self,parameter_s,runner=self.mplot_exec)
 
     # Fix the docstring so users see the original as well
     magic_run.__doc__ = "%s\n%s" % (Magic.magic_run.__doc__,
                                     "\n        *** Modified %run for Matplotlib,"
                                     " with proper interactive handling ***")
 
 # Now we provide 2 versions of a matplotlib-aware IPython base shells, single
 # and multithreaded.  Note that these are meant for internal use, the IPShell*
 # classes below are the ones meant for public consumption.
 
 class MatplotlibShell(MatplotlibShellBase,InteractiveShell):
     """Single-threaded shell with matplotlib support."""
 
     def __init__(self,name,usage=None,rc=Struct(opts=None,args=None),
                  user_ns=None,user_global_ns=None,**kw):
         user_ns,b2 = self._matplotlib_config(name,user_ns)
         InteractiveShell.__init__(self,name,usage,rc,user_ns,user_global_ns,
                                   banner2=b2,**kw)
 
 class MatplotlibMTShell(MatplotlibShellBase,MTInteractiveShell):
     """Multi-threaded shell with matplotlib support."""
 
     def __init__(self,name,usage=None,rc=Struct(opts=None,args=None),
                  user_ns=None,user_global_ns=None, **kw):
         user_ns,b2 = self._matplotlib_config(name,user_ns)
         MTInteractiveShell.__init__(self,name,usage,rc,user_ns,user_global_ns,
                                     banner2=b2,**kw)
 
 #-----------------------------------------------------------------------------
 # Utility functions for the different GUI enabled IPShell* classes.
 
 def get_tk():
     """Tries to import Tkinter and returns a withdrawn Tkinter root
     window.  If Tkinter is already imported or not available, this
     returns None.  This function calls `hijack_tk` underneath.
     """
     if not USE_TK or sys.modules.has_key('Tkinter'):
         return None
     else:
         try:
             import Tkinter
         except ImportError:
             return None
         else:
             hijack_tk()
             r = Tkinter.Tk()
             r.withdraw()
             return r
 
 def hijack_tk():
     """Modifies Tkinter's mainloop with a dummy so when a module calls
     mainloop, it does not block.
 
     """
     def misc_mainloop(self, n=0):
         pass
     def tkinter_mainloop(n=0):
         pass
     
     import Tkinter
     Tkinter.Misc.mainloop = misc_mainloop
     Tkinter.mainloop = tkinter_mainloop
 
 def update_tk(tk):
     """Updates the Tkinter event loop.  This is typically called from
     the respective WX or GTK mainloops.
     """    
     if tk:
         tk.update()
 
 def hijack_wx():
     """Modifies wxPython's MainLoop with a dummy so user code does not
     block IPython.  The hijacked mainloop function is returned.
     """    
     def dummy_mainloop(*args, **kw):
         pass
 
     try:
         import wx
     except ImportError:
         # For very old versions of WX
         import wxPython as wx
         
     ver = wx.__version__
     orig_mainloop = None
     if ver[:3] >= '2.5':
         import wx
         if hasattr(wx, '_core_'): core = getattr(wx, '_core_')
         elif hasattr(wx, '_core'): core = getattr(wx, '_core')
         else: raise AttributeError('Could not find wx core module')
         orig_mainloop = core.PyApp_MainLoop
         core.PyApp_MainLoop = dummy_mainloop
     elif ver[:3] == '2.4':
         orig_mainloop = wx.wxc.wxPyApp_MainLoop
         wx.wxc.wxPyApp_MainLoop = dummy_mainloop
     else:
         warn("Unable to find either wxPython version 2.4 or >= 2.5.")
     return orig_mainloop
 
 def hijack_gtk():
     """Modifies pyGTK's mainloop with a dummy so user code does not
     block IPython.  This function returns the original `gtk.mainloop`
     function that has been hijacked.
     """    
     def dummy_mainloop(*args, **kw):
         pass
     import gtk
     if gtk.pygtk_version >= (2,4,0): orig_mainloop = gtk.main
     else:                            orig_mainloop = gtk.mainloop
     gtk.mainloop = dummy_mainloop
     gtk.main = dummy_mainloop
     return orig_mainloop
 
 def hijack_qt():
     """Modifies PyQt's mainloop with a dummy so user code does not 
     block IPython.  This function returns the original 
     `qt.qApp.exec_loop` function that has been hijacked.
     """    
     def dummy_mainloop(*args, **kw):
         pass
     import qt
     orig_mainloop = qt.qApp.exec_loop
     qt.qApp.exec_loop = dummy_mainloop
     qt.QApplication.exec_loop = dummy_mainloop
     return orig_mainloop
 
 def hijack_qt4():
     """Modifies PyQt4's mainloop with a dummy so user code does not
     block IPython.  This function returns the original 
     `QtGui.qApp.exec_` function that has been hijacked.
     """    
     def dummy_mainloop(*args, **kw):
         pass
     from PyQt4 import QtGui, QtCore
     orig_mainloop = QtGui.qApp.exec_
     QtGui.qApp.exec_ = dummy_mainloop
     QtGui.QApplication.exec_ = dummy_mainloop
     QtCore.QCoreApplication.exec_ = dummy_mainloop
     return orig_mainloop
 
 #-----------------------------------------------------------------------------
 # The IPShell* classes below are the ones meant to be run by external code as
 # IPython instances.  Note that unless a specific threading strategy is
 # desired, the factory function start() below should be used instead (it
 # selects the proper threaded class).
 
 class IPThread(threading.Thread):
     def run(self):
         self.IP.mainloop(self._banner)
         self.IP.kill()
 
 class IPShellGTK(IPThread):
     """Run a gtk mainloop() in a separate thread.
     
     Python commands can be passed to the thread where they will be executed.
     This is implemented by periodically checking for passed code using a
     GTK timeout callback."""
     
     TIMEOUT = 100 # Millisecond interval between timeouts.
 
     def __init__(self,argv=None,user_ns=None,user_global_ns=None,
                  debug=1,shell_class=MTInteractiveShell):
 
         import gtk
         
         self.gtk = gtk
         self.gtk_mainloop = hijack_gtk()
 
         # Allows us to use both Tk and GTK.
         self.tk = get_tk()
         
         if gtk.pygtk_version >= (2,4,0): mainquit = self.gtk.main_quit
         else:                            mainquit = self.gtk.mainquit
 
         self.IP = make_IPython(argv,user_ns=user_ns,
                                user_global_ns=user_global_ns,
                                debug=debug,
                                shell_class=shell_class,
                                on_kill=[mainquit])
 
         # HACK: slot for banner in self; it will be passed to the mainloop
         # method only and .run() needs it.  The actual value will be set by
         # .mainloop().
         self._banner = None 
 
         threading.Thread.__init__(self)
 
     def mainloop(self,sys_exit=0,banner=None):
 
         self._banner = banner
         
         if self.gtk.pygtk_version >= (2,4,0):
             import gobject
             gobject.idle_add(self.on_timer)
         else:
             self.gtk.idle_add(self.on_timer)
 
         if sys.platform != 'win32':
             try:
                 if self.gtk.gtk_version[0] >= 2:
                     self.gtk.gdk.threads_init()
             except AttributeError:
                 pass
             except RuntimeError:
                 error('Your pyGTK likely has not been compiled with '
                       'threading support.\n'
                       'The exception printout is below.\n'
                       'You can either rebuild pyGTK with threads, or '
                       'try using \n'
                       'matplotlib with a different backend (like Tk or WX).\n'
                       'Note that matplotlib will most likely not work in its '
                       'current state!')
                 self.IP.InteractiveTB()
 
         self.start()
         self.gtk.gdk.threads_enter()
         self.gtk_mainloop()
         self.gtk.gdk.threads_leave()
         self.join()
 
     def on_timer(self):
         """Called when GTK is idle.
 
         Must return True always, otherwise GTK stops calling it"""
         
         update_tk(self.tk)
         self.IP.runcode()
         time.sleep(0.01)
         return True
 
 
 class IPShellWX(IPThread):
     """Run a wx mainloop() in a separate thread.
     
     Python commands can be passed to the thread where they will be executed.
     This is implemented by periodically checking for passed code using a
     GTK timeout callback."""
     
     TIMEOUT = 100 # Millisecond interval between timeouts.
 
     def __init__(self,argv=None,user_ns=None,user_global_ns=None,
                  debug=1,shell_class=MTInteractiveShell):
 
         self.IP = make_IPython(argv,user_ns=user_ns,
                                user_global_ns=user_global_ns,
                                debug=debug,
                                shell_class=shell_class,
                                on_kill=[self.wxexit])
 
         wantedwxversion=self.IP.rc.wxversion
         if wantedwxversion!="0":
             try:
                 import wxversion
             except ImportError:
                 error('The wxversion module is needed for WX version selection')
             else:
                 try:
                     wxversion.select(wantedwxversion)
                 except:
                     self.IP.InteractiveTB()
                     error('Requested wxPython version %s could not be loaded' %
                                                                wantedwxversion)
 
         import wx
 
         threading.Thread.__init__(self)
         self.wx = wx
         self.wx_mainloop = hijack_wx()
 
         # Allows us to use both Tk and GTK.
         self.tk = get_tk()
         
         # HACK: slot for banner in self; it will be passed to the mainloop
         # method only and .run() needs it.  The actual value will be set by
         # .mainloop().
         self._banner = None 
 
         self.app = None
 
     def wxexit(self, *args):
         if self.app is not None:
             self.app.agent.timer.Stop()
             self.app.ExitMainLoop()
 
     def mainloop(self,sys_exit=0,banner=None):
 
         self._banner = banner
         
         self.start()
 
         class TimerAgent(self.wx.MiniFrame):
             wx = self.wx
             IP = self.IP
             tk = self.tk
             def __init__(self, parent, interval):
                 style = self.wx.DEFAULT_FRAME_STYLE | self.wx.TINY_CAPTION_HORIZ
                 self.wx.MiniFrame.__init__(self, parent, -1, ' ', pos=(200, 200),
                                              size=(100, 100),style=style)
                 self.Show(False)
                 self.interval = interval
                 self.timerId = self.wx.NewId()                                
 
             def StartWork(self):
                 self.timer = self.wx.Timer(self, self.timerId)
                 self.wx.EVT_TIMER(self,  self.timerId, self.OnTimer)
                 self.timer.Start(self.interval)
 
             def OnTimer(self, event):
                 update_tk(self.tk)
                 self.IP.runcode()
 
         class App(self.wx.App):
             wx = self.wx
             TIMEOUT = self.TIMEOUT
             def OnInit(self):
                 'Create the main window and insert the custom frame'
                 self.agent = TimerAgent(None, self.TIMEOUT)
                 self.agent.Show(False)
                 self.agent.StartWork()
                 return True
 
         self.app = App(redirect=False)
         self.wx_mainloop(self.app)
         self.join()
 
 
 class IPShellQt(IPThread):
     """Run a Qt event loop in a separate thread.
     
     Python commands can be passed to the thread where they will be executed.
     This is implemented by periodically checking for passed code using a
     Qt timer / slot."""
     
     TIMEOUT = 100 # Millisecond interval between timeouts.
 
     def __init__(self, argv=None, user_ns=None, user_global_ns=None,
                  debug=0, shell_class=MTInteractiveShell):
 
         import qt
 
         self.exec_loop = hijack_qt()
 
         # Allows us to use both Tk and QT.
         self.tk = get_tk()
 
         self.IP = make_IPython(argv,
                                user_ns=user_ns,
                                user_global_ns=user_global_ns,
                                debug=debug,
                                shell_class=shell_class,
                                on_kill=[qt.qApp.exit])
 
         # HACK: slot for banner in self; it will be passed to the mainloop
         # method only and .run() needs it.  The actual value will be set by
         # .mainloop().
         self._banner = None 
         
         threading.Thread.__init__(self)
 
     def mainloop(self, sys_exit=0, banner=None):
 
         import qt
 
         self._banner = banner
 
         if qt.QApplication.startingUp():
             a = qt.QApplication(sys.argv)
 
         self.timer = qt.QTimer()
         qt.QObject.connect(self.timer,
                            qt.SIGNAL('timeout()'),
                            self.on_timer)
 
         self.start()
         self.timer.start(self.TIMEOUT, True)
         while True:
             if self.IP._kill: break
             self.exec_loop()
         self.join()
 
     def on_timer(self):
         update_tk(self.tk)
         result = self.IP.runcode()
         self.timer.start(self.TIMEOUT, True)
         return result
 
 
 class IPShellQt4(IPThread):
     """Run a Qt event loop in a separate thread.
 
     Python commands can be passed to the thread where they will be executed.
     This is implemented by periodically checking for passed code using a
     Qt timer / slot."""
 
     TIMEOUT = 100 # Millisecond interval between timeouts.
 
     def __init__(self, argv=None, user_ns=None, user_global_ns=None,
                  debug=0, shell_class=MTInteractiveShell):
 
         from PyQt4 import QtCore, QtGui
 
         try:
             # present in PyQt4-4.2.1 or later
             QtCore.pyqtRemoveInputHook()
         except AttributeError:
             pass
 
         if QtCore.PYQT_VERSION_STR == '4.3':
             warn('''PyQt4 version 4.3 detected.
 If you experience repeated threading warnings, please update PyQt4.
 ''')
 
         self.exec_ = hijack_qt4()
 
         # Allows us to use both Tk and QT.
         self.tk = get_tk()
 
         self.IP = make_IPython(argv,
                                user_ns=user_ns,
                                user_global_ns=user_global_ns,
                                debug=debug,
                                shell_class=shell_class,
                                on_kill=[QtGui.qApp.exit])
 
         # HACK: slot for banner in self; it will be passed to the mainloop
         # method only and .run() needs it.  The actual value will be set by
         # .mainloop().
         self._banner = None
 
         threading.Thread.__init__(self)
 
     def mainloop(self, sys_exit=0, banner=None):
 
         from PyQt4 import QtCore, QtGui
 
         self._banner = banner
 
         if QtGui.QApplication.startingUp():
             a = QtGui.QApplication(sys.argv)
 
         self.timer = QtCore.QTimer()
         QtCore.QObject.connect(self.timer,
                                QtCore.SIGNAL('timeout()'),
                                self.on_timer)
 
         self.start()
         self.timer.start(self.TIMEOUT)
         while True:
             if self.IP._kill: break
             self.exec_()
         self.join()
 
     def on_timer(self):
         update_tk(self.tk)
         result = self.IP.runcode()
         self.timer.start(self.TIMEOUT)
         return result
 
 
 # A set of matplotlib public IPython shell classes, for single-threaded (Tk*
 # and FLTK*) and multithreaded (GTK*, WX* and Qt*) backends to use.
 def _load_pylab(user_ns):
     """Allow users to disable pulling all of pylab into the top-level
     namespace.
 
     This little utility must be called AFTER the actual ipython instance is
     running, since only then will the options file have been fully parsed."""
     
     ip = IPython.ipapi.get()
     if ip.options.pylab_import_all:
-        exec "from matplotlib.pylab import *" in user_ns
+        ip.ex("from matplotlib.pylab import *")
+        ip.IP.user_config_ns.update(ip.user_ns)
+        
 
 class IPShellMatplotlib(IPShell):
     """Subclass IPShell with MatplotlibShell as the internal shell.
 
     Single-threaded class, meant for the Tk* and FLTK* backends.
 
     Having this on a separate class simplifies the external driver code."""
     
     def __init__(self,argv=None,user_ns=None,user_global_ns=None,debug=1):
         IPShell.__init__(self,argv,user_ns,user_global_ns,debug,
                          shell_class=MatplotlibShell)
         _load_pylab(self.IP.user_ns)
 
 class IPShellMatplotlibGTK(IPShellGTK):
     """Subclass IPShellGTK with MatplotlibMTShell as the internal shell.
 
     Multi-threaded class, meant for the GTK* backends."""
     
     def __init__(self,argv=None,user_ns=None,user_global_ns=None,debug=1):
         IPShellGTK.__init__(self,argv,user_ns,user_global_ns,debug,
                             shell_class=MatplotlibMTShell)
         _load_pylab(self.IP.user_ns)
 
 class IPShellMatplotlibWX(IPShellWX):
     """Subclass IPShellWX with MatplotlibMTShell as the internal shell.
 
     Multi-threaded class, meant for the WX* backends."""
     
     def __init__(self,argv=None,user_ns=None,user_global_ns=None,debug=1):
         IPShellWX.__init__(self,argv,user_ns,user_global_ns,debug,
                            shell_class=MatplotlibMTShell)
         _load_pylab(self.IP.user_ns)
 
 class IPShellMatplotlibQt(IPShellQt):
     """Subclass IPShellQt with MatplotlibMTShell as the internal shell.
 
     Multi-threaded class, meant for the Qt* backends."""
     
     def __init__(self,argv=None,user_ns=None,user_global_ns=None,debug=1):
         IPShellQt.__init__(self,argv,user_ns,user_global_ns,debug,
                            shell_class=MatplotlibMTShell)
         _load_pylab(self.IP.user_ns)
 
 class IPShellMatplotlibQt4(IPShellQt4):
     """Subclass IPShellQt4 with MatplotlibMTShell as the internal shell.
 
     Multi-threaded class, meant for the Qt4* backends."""
 
     def __init__(self,argv=None,user_ns=None,user_global_ns=None,debug=1):
         IPShellQt4.__init__(self,argv,user_ns,user_global_ns,debug,
                            shell_class=MatplotlibMTShell)
         _load_pylab(self.IP.user_ns)
 
 #-----------------------------------------------------------------------------
 # Factory functions to actually start the proper thread-aware shell
 
 def _select_shell(argv):
     """Select a shell from the given argv vector.
 
     This function implements the threading selection policy, allowing runtime
     control of the threading mode, both for general users and for matplotlib.
 
     Return:
       Shell class to be instantiated for runtime operation.
     """
     
     global USE_TK
 
     mpl_shell = {'gthread' : IPShellMatplotlibGTK,
                  'wthread' : IPShellMatplotlibWX,
                  'qthread' : IPShellMatplotlibQt,
                  'q4thread' : IPShellMatplotlibQt4,
                  'tkthread' : IPShellMatplotlib,  # Tk is built-in
                  }
 
     th_shell = {'gthread' : IPShellGTK,
                 'wthread' : IPShellWX,
                 'qthread' : IPShellQt,
                 'q4thread' : IPShellQt4,
                 'tkthread' : IPShell, # Tk is built-in
                 }
 
     backends = {'gthread' : 'GTKAgg',
                 'wthread' : 'WXAgg',
                 'qthread' : 'QtAgg',
                 'q4thread' :'Qt4Agg',
                 'tkthread' :'TkAgg',
                 }
 
     all_opts = set(['tk','pylab','gthread','qthread','q4thread','wthread',
                     'tkthread'])
     user_opts = set([s.replace('-','') for s in argv[:3]])
-    special_opts = user_opts & all_opts
+    special_opts = user_opts & all_opts            
 
     if 'tk' in special_opts:
         USE_TK = True
         special_opts.remove('tk')
 
     if 'pylab' in special_opts:
 
         try:
             import matplotlib
         except ImportError:
             error('matplotlib could NOT be imported!  Starting normal IPython.')
             return IPShell
         
         special_opts.remove('pylab')
         # If there's any option left, it means the user wants to force the
         # threading backend, else it's auto-selected from the rc file
         if special_opts:
             th_mode = special_opts.pop()
             matplotlib.rcParams['backend'] = backends[th_mode]
         else:
             backend = matplotlib.rcParams['backend']
             if backend.startswith('GTK'):
                 th_mode = 'gthread'
             elif backend.startswith('WX'):
                 th_mode = 'wthread'
             elif backend.startswith('Qt4'):
                 th_mode = 'q4thread'
             elif backend.startswith('Qt'):
                 th_mode = 'qthread'
             else:
                 # Any other backend, use plain Tk
                 th_mode = 'tkthread'
                 
         return mpl_shell[th_mode]
     else:
         # No pylab requested, just plain threads
         try:
             th_mode = special_opts.pop()
         except KeyError:
             th_mode = 'tkthread'
         return th_shell[th_mode]
 
 
 # This is the one which should be called by external code.
 def start(user_ns = None):
     """Return a running shell instance, dealing with threading options.
 
     This is a factory function which will instantiate the proper IPython shell
     based on the user's threading choice.  Such a selector is needed because
     different GUI toolkits require different thread handling details."""
 
     shell = _select_shell(sys.argv)
     return shell(user_ns = user_ns)
 
 # Some aliases for backwards compatibility
 IPythonShell = IPShell
 IPythonShellEmbed = IPShellEmbed
 #************************ End of file <Shell.py> ***************************

IPython/UserConfig/ipy_user_conf.py

@@ -1,86 +1,107 b''
 """ User configuration file for IPython
 
 This is a more flexible and safe way to configure ipython than *rc files
 (ipythonrc, ipythonrc-pysh etc.)
 
 This file is always imported on ipython startup. You can import the
 ipython extensions you need here (see IPython/Extensions directory).
 
 Feel free to edit this file to customize your ipython experience.
 
 Note that as such this file does nothing, for backwards compatibility.
 Consult e.g. file 'ipy_profile_sh.py' for an example of the things 
 you can do here.
 
 See http://ipython.scipy.org/moin/IpythonExtensionApi for detailed
 description on what you could do here.
 """
 
 # Most of your config files and extensions will probably start with this import
 
 import IPython.ipapi
 ip = IPython.ipapi.get()
 
 # You probably want to uncomment this if you did %upgrade -nolegacy
 # import ipy_defaults    
 
 import os   
 
 def main():   
 
     # uncomment if you want to get ipython -p sh behaviour
     # without having to use command line switches  
     # import ipy_profile_sh
 
     # Configure your favourite editor?
     # Good idea e.g. for %edit os.path.isfile
 
     #import ipy_editors
     
     # Choose one of these:
     
     #ipy_editors.scite()
     #ipy_editors.scite('c:/opt/scite/scite.exe')
     #ipy_editors.komodo()
     #ipy_editors.idle()
     # ... or many others, try 'ipy_editors??' after import to see them
     
     # Or roll your own:
     #ipy_editors.install_editor("c:/opt/jed +$line $file")
     
     
     o = ip.options
     # An example on how to set options
     #o.autocall = 1
     o.system_verbose = 0
     
     #import_all("os sys")
     #execf('~/_ipython/ns.py')
 
 
     # -- prompt
     # A different, more compact set of prompts from the default ones, that
     # always show your current location in the filesystem:
 
     #o.prompt_in1 = r'\C_LightBlue[\C_LightCyan\Y2\C_LightBlue]\C_Normal\n\C_Green|\#>'
     #o.prompt_in2 = r'.\D: '
     #o.prompt_out = r'[\#] '
     
     # Try one of these color settings if you can't read the text easily
     # autoexec is a list of IPython commands to execute on startup
     #o.autoexec.append('%colors LightBG')
     #o.autoexec.append('%colors NoColor')
     #o.autoexec.append('%colors Linux')
     
+    # for sane integer division that converts to float (1/2 == 0.5)
+    #o.autoexec.append('from __future__ import division')
+    
+    # For %tasks and %kill
+    #import jobctrl 
+    
+    # For autoreloading of modules (%autoreload, %aimport)    
+    #import ipy_autoreload
+    
+    # For winpdb support (%wdb)
+    #import ipy_winpdb
+    
+    # For bzr completer, requires bzrlib (the python installation of bzr)
+    #ip.load('ipy_bzr')
+    
+    # Tab completer that is not quite so picky (i.e. 
+    # "foo".<TAB> and str(2).<TAB> will work). Complete 
+    # at your own risk!
+    #import ipy_greedycompleter
+    
+    
     
 # some config helper functions you can use 
 def import_all(modules):
     """ Usage: import_all("os sys") """ 
     for m in modules.split():
         ip.ex("from %s import *" % m)
         
 def execf(fname):
     """ Execute a file in user namespace """
     ip.ex('execfile("%s")' % os.path.expanduser(fname))
 
 main()

IPython/completer.py

@@ -1,644 +1,644 b''
 """Word completion for IPython.
 
 This module is a fork of the rlcompleter module in the Python standard
 library.  The original enhancements made to rlcompleter have been sent
 upstream and were accepted as of Python 2.3, but we need a lot more
 functionality specific to IPython, so this module will continue to live as an
 IPython-specific utility.
 
 ---------------------------------------------------------------------------
 Original rlcompleter documentation:
 
 This requires the latest extension to the readline module (the
 completes keywords, built-ins and globals in __main__; when completing
 NAME.NAME..., it evaluates (!) the expression up to the last dot and
 completes its attributes.
 
 It's very cool to do "import string" type "string.", hit the
 completion key (twice), and see the list of names defined by the
 string module!
 
 Tip: to use the tab key as the completion key, call
 
     readline.parse_and_bind("tab: complete")
 
 Notes:
 
 - Exceptions raised by the completer function are *ignored* (and
 generally cause the completion to fail).  This is a feature -- since
 readline sets the tty device in raw (or cbreak) mode, printing a
 traceback wouldn't work well without some complicated hoopla to save,
 reset and restore the tty state.
 
 - The evaluation of the NAME.NAME... form may cause arbitrary
 application defined code to be executed if an object with a
 __getattr__ hook is found.  Since it is the responsibility of the
 application (or the user) to enable this feature, I consider this an
 acceptable risk.  More complicated expressions (e.g. function calls or
 indexing operations) are *not* evaluated.
 
 - GNU readline is also used by the built-in functions input() and
 raw_input(), and thus these also benefit/suffer from the completer
 features.  Clearly an interactive application can benefit by
 specifying its own completer function and using raw_input() for all
 its input.
 
 - When the original stdin is not a tty device, GNU readline is never
 used, and this module (and the readline module) are silently inactive.
 
 """
 
 #*****************************************************************************
 #
 # Since this file is essentially a minimally modified copy of the rlcompleter
 # module which is part of the standard Python distribution, I assume that the
 # proper procedure is to maintain its copyright as belonging to the Python
 # Software Foundation (in addition to my own, for all new code).
 #
 #       Copyright (C) 2001 Python Software Foundation, www.python.org
 #       Copyright (C) 2001-2006 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #
 #*****************************************************************************
 
 import __builtin__
 import __main__
 import glob
 import keyword
 import os
 import re
 import shlex
 import sys
 import IPython.rlineimpl as readline    
 import itertools
 from IPython.ipstruct import Struct
 from IPython import ipapi
 from IPython import generics
 import types
 
 # Python 2.4 offers sets as a builtin
 try:
     set()
 except NameError:
     from sets import Set as set
 
 from IPython.genutils import debugx, dir2
 
 __all__ = ['Completer','IPCompleter']
 
 class Completer:
     def __init__(self,namespace=None,global_namespace=None):
         """Create a new completer for the command line.
 
         Completer([namespace,global_namespace]) -> completer instance.
 
         If unspecified, the default namespace where completions are performed
         is __main__ (technically, __main__.__dict__). Namespaces should be
         given as dictionaries.
 
         An optional second namespace can be given.  This allows the completer
         to handle cases where both the local and global scopes need to be
         distinguished.
 
         Completer instances should be used as the completion mechanism of
         readline via the set_completer() call:
 
         readline.set_completer(Completer(my_namespace).complete)
         """
 
         # some minimal strict typechecks.  For some core data structures, I
         # want actual basic python types, not just anything that looks like
         # one.  This is especially true for namespaces.
         for ns in (namespace,global_namespace):
             if ns is not None and type(ns) != types.DictType:
                 raise TypeError,'namespace must be a dictionary'
 
         # Don't bind to namespace quite yet, but flag whether the user wants a
         # specific namespace or to use __main__.__dict__. This will allow us
         # to bind to __main__.__dict__ at completion time, not now.
         if namespace is None:
             self.use_main_ns = 1
         else:
             self.use_main_ns = 0
             self.namespace = namespace
 
         # The global namespace, if given, can be bound directly
         if global_namespace is None:
             self.global_namespace = {}
         else:
             self.global_namespace = global_namespace
 
     def complete(self, text, state):
         """Return the next possible completion for 'text'.
 
         This is called successively with state == 0, 1, 2, ... until it
         returns None.  The completion should begin with 'text'.
 
         """
         if self.use_main_ns:
             self.namespace = __main__.__dict__
             
         if state == 0:
             if "." in text:
                 self.matches = self.attr_matches(text)
             else:
                 self.matches = self.global_matches(text)
         try:
             return self.matches[state]
         except IndexError:
             return None
 
     def global_matches(self, text):
         """Compute matches when text is a simple name.
 
         Return a list of all keywords, built-in functions and names currently
         defined in self.namespace or self.global_namespace that match.
 
         """
         matches = []
         match_append = matches.append
         n = len(text)
         for lst in [keyword.kwlist,
                     __builtin__.__dict__.keys(),
                     self.namespace.keys(),
                     self.global_namespace.keys()]:
             for word in lst:
                 if word[:n] == text and word != "__builtins__":
                     match_append(word)
         return matches
 
     def attr_matches(self, text):
         """Compute matches when text contains a dot.
 
         Assuming the text is of the form NAME.NAME....[NAME], and is
         evaluatable in self.namespace or self.global_namespace, it will be
         evaluated and its attributes (as revealed by dir()) are used as
         possible completions.  (For class instances, class members are are
         also considered.)
 
         WARNING: this can still invoke arbitrary C code, if an object
         with a __getattr__ hook is evaluated.
 
         """
         import re
 
         # Another option, seems to work great. Catches things like ''.<tab>
         m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text)
 
         if not m:
             return []
         
         expr, attr = m.group(1, 3)
         try:
             obj = eval(expr, self.namespace)
         except:
             try:
                 obj = eval(expr, self.global_namespace)
             except:
                 return []
 
         words = dir2(obj)
         
         try:
             words = generics.complete_object(obj, words)
         except ipapi.TryNext:
             pass
         # Build match list to return
         n = len(attr)
         res = ["%s.%s" % (expr, w) for w in words if w[:n] == attr ]
         return res
 
 class IPCompleter(Completer):
     """Extension of the completer class with IPython-specific features"""
 
     def __init__(self,shell,namespace=None,global_namespace=None,
                  omit__names=0,alias_table=None):
         """IPCompleter() -> completer
 
         Return a completer object suitable for use by the readline library
         via readline.set_completer().
 
         Inputs:
 
         - shell: a pointer to the ipython shell itself.  This is needed
         because this completer knows about magic functions, and those can
         only be accessed via the ipython instance.
 
         - namespace: an optional dict where completions are performed.
 
         - global_namespace: secondary optional dict for completions, to
         handle cases (such as IPython embedded inside functions) where
         both Python scopes are visible.
 
         - The optional omit__names parameter sets the completer to omit the
         'magic' names (__magicname__) for python objects unless the text
         to be completed explicitly starts with one or more underscores.
 
         - If alias_table is supplied, it should be a dictionary of aliases
         to complete. """
 
         Completer.__init__(self,namespace,global_namespace)
         self.magic_prefix = shell.name+'.magic_'
         self.magic_escape = shell.ESC_MAGIC
         self.readline = readline
         delims = self.readline.get_completer_delims()
         delims = delims.replace(self.magic_escape,'')
         self.readline.set_completer_delims(delims)
         self.get_line_buffer = self.readline.get_line_buffer
         self.get_endidx = self.readline.get_endidx
         self.omit__names = omit__names
         self.merge_completions = shell.rc.readline_merge_completions        
         if alias_table is None:
             alias_table = {}
         self.alias_table = alias_table
         # Regexp to split filenames with spaces in them
         self.space_name_re = re.compile(r'([^\\] )')
         # Hold a local ref. to glob.glob for speed
         self.glob = glob.glob
 
         # Determine if we are running on 'dumb' terminals, like (X)Emacs
         # buffers, to avoid completion problems.
         term = os.environ.get('TERM','xterm')
         self.dumb_terminal = term in ['dumb','emacs']
         
         # Special handling of backslashes needed in win32 platforms
         if sys.platform == "win32":
             self.clean_glob = self._clean_glob_win32
         else:
             self.clean_glob = self._clean_glob
         self.matchers = [self.python_matches,
                          self.file_matches,
                          self.alias_matches,
                          self.python_func_kw_matches]
 
     
     # Code contributed by Alex Schmolck, for ipython/emacs integration
     def all_completions(self, text):
         """Return all possible completions for the benefit of emacs."""
 
         completions = []
         comp_append = completions.append
         try:
             for i in xrange(sys.maxint):
                 res = self.complete(text, i)
 
                 if not res: break
 
                 comp_append(res)
         #XXX workaround for ``notDefined.<tab>``
         except NameError:
             pass
         return completions
     # /end Alex Schmolck code.
 
     def _clean_glob(self,text):
         return self.glob("%s*" % text)
 
     def _clean_glob_win32(self,text):
         return [f.replace("\\","/")
                 for f in self.glob("%s*" % text)]            
 
     def file_matches(self, text):
         """Match filenames, expanding ~USER type strings.
 
         Most of the seemingly convoluted logic in this completer is an
         attempt to handle filenames with spaces in them.  And yet it's not
         quite perfect, because Python's readline doesn't expose all of the
         GNU readline details needed for this to be done correctly.
 
         For a filename with a space in it, the printed completions will be
         only the parts after what's already been typed (instead of the
         full completions, as is normally done).  I don't think with the
         current (as of Python 2.3) Python readline it's possible to do
         better."""
 
         #print 'Completer->file_matches: <%s>' % text # dbg
 
         # chars that require escaping with backslash - i.e. chars
         # that readline treats incorrectly as delimiters, but we
         # don't want to treat as delimiters in filename matching
         # when escaped with backslash
 
-        protectables = ' ()[]{}'
+        protectables = ' '
 
         if text.startswith('!'):
             text = text[1:]
             text_prefix = '!'
         else:
             text_prefix = ''
             
         def protect_filename(s):
             return "".join([(ch in protectables and '\\' + ch or ch)
                             for ch in s])
 
         def single_dir_expand(matches):
             "Recursively expand match lists containing a single dir."
             
             if len(matches) == 1 and os.path.isdir(matches[0]):
                 # Takes care of links to directories also.  Use '/'
                 # explicitly, even under Windows, so that name completions
                 # don't end up escaped.
                 d = matches[0]
                 if d[-1] in ['/','\\']:
                     d = d[:-1]
 
                 subdirs = os.listdir(d)
                 if subdirs:
                     matches = [ (d + '/' + p) for p in subdirs]
                     return single_dir_expand(matches)
                 else:
                     return matches
             else:
                 return matches
         
         lbuf = self.lbuf
         open_quotes = 0  # track strings with open quotes
         try:
             lsplit = shlex.split(lbuf)[-1]
         except ValueError:
             # typically an unmatched ", or backslash without escaped char.
             if lbuf.count('"')==1:
                 open_quotes = 1
                 lsplit = lbuf.split('"')[-1]
             elif lbuf.count("'")==1:
                 open_quotes = 1
                 lsplit = lbuf.split("'")[-1]
             else:
                 return []
         except IndexError:
             # tab pressed on empty line
             lsplit = ""
 
         if lsplit != protect_filename(lsplit):
             # if protectables are found, do matching on the whole escaped
             # name
             has_protectables = 1
             text0,text = text,lsplit
         else:
             has_protectables = 0
             text = os.path.expanduser(text)
 
         if text == "":
             return [text_prefix + protect_filename(f) for f in self.glob("*")]
 
         m0 = self.clean_glob(text.replace('\\',''))
         if has_protectables:
             # If we had protectables, we need to revert our changes to the
             # beginning of filename so that we don't double-write the part
             # of the filename we have so far
             len_lsplit = len(lsplit)
             matches = [text_prefix + text0 + 
                        protect_filename(f[len_lsplit:]) for f in m0]
         else:
             if open_quotes:
                 # if we have a string with an open quote, we don't need to
                 # protect the names at all (and we _shouldn't_, as it
                 # would cause bugs when the filesystem call is made).
                 matches = m0
             else:
                 matches = [text_prefix + 
                            protect_filename(f) for f in m0]
 
         #print 'mm',matches  # dbg
         return single_dir_expand(matches)
 
     def alias_matches(self, text):
         """Match internal system aliases"""        
         #print 'Completer->alias_matches:',text,'lb',self.lbuf # dbg
         
         # if we are not in the first 'item', alias matching 
         # doesn't make sense - unless we are starting with 'sudo' command.
         if ' ' in self.lbuf.lstrip() and not self.lbuf.lstrip().startswith('sudo'):
             return []
         text = os.path.expanduser(text)
         aliases =  self.alias_table.keys()
         if text == "":
             return aliases
         else:
             return [alias for alias in aliases if alias.startswith(text)]
 
     def python_matches(self,text):
         """Match attributes or global python names"""
 
         #print 'Completer->python_matches, txt=<%s>' % text # dbg
         if "." in text:
             try:
                 matches = self.attr_matches(text)
                 if text.endswith('.') and self.omit__names:
                     if self.omit__names == 1:
                         # true if txt is _not_ a __ name, false otherwise:
                         no__name = (lambda txt:
                                     re.match(r'.*\.__.*?__',txt) is None)
                     else:
                         # true if txt is _not_ a _ name, false otherwise:
                         no__name = (lambda txt:
                                     re.match(r'.*\._.*?',txt) is None)
                     matches = filter(no__name, matches)
             except NameError:
                 # catches <undefined attributes>.<tab>
                 matches = []
         else:
             matches = self.global_matches(text)
             # this is so completion finds magics when automagic is on:
             if (matches == [] and 
                  not text.startswith(os.sep) and
                  not ' ' in self.lbuf):
                 matches = self.attr_matches(self.magic_prefix+text)
         return matches
 
     def _default_arguments(self, obj):
         """Return the list of default arguments of obj if it is callable,
         or empty list otherwise."""
 
         if not (inspect.isfunction(obj) or inspect.ismethod(obj)):
             # for classes, check for __init__,__new__
             if inspect.isclass(obj):
                 obj = (getattr(obj,'__init__',None) or
                        getattr(obj,'__new__',None))
             # for all others, check if they are __call__able
             elif hasattr(obj, '__call__'):
                 obj = obj.__call__
             # XXX: is there a way to handle the builtins ?
         try:
             args,_,_1,defaults = inspect.getargspec(obj)
             if defaults:
                 return args[-len(defaults):]
         except TypeError: pass
         return []
 
     def python_func_kw_matches(self,text):
         """Match named parameters (kwargs) of the last open function"""
 
         if "." in text: # a parameter cannot be dotted
             return []
         try: regexp = self.__funcParamsRegex
         except AttributeError:
             regexp = self.__funcParamsRegex = re.compile(r'''
                 '.*?' |    # single quoted strings or
                 ".*?" |    # double quoted strings or
                 \w+   |    # identifier
                 \S         # other characters
                 ''', re.VERBOSE | re.DOTALL)
         # 1. find the nearest identifier that comes before an unclosed
         # parenthesis e.g. for "foo (1+bar(x), pa", the candidate is "foo"
         tokens = regexp.findall(self.get_line_buffer())
         tokens.reverse()
         iterTokens = iter(tokens); openPar = 0
         for token in iterTokens:
             if token == ')':
                 openPar -= 1
             elif token == '(':
                 openPar += 1
                 if openPar > 0:
                     # found the last unclosed parenthesis
                     break
         else:
             return []
         # 2. Concatenate dotted names ("foo.bar" for "foo.bar(x, pa" )
         ids = []
         isId = re.compile(r'\w+$').match
         while True:
             try:
                 ids.append(iterTokens.next())
                 if not isId(ids[-1]):
                     ids.pop(); break
                 if not iterTokens.next() == '.':
                     break
             except StopIteration:
                 break
         # lookup the candidate callable matches either using global_matches
         # or attr_matches for dotted names
         if len(ids) == 1:
             callableMatches = self.global_matches(ids[0])
         else:
             callableMatches = self.attr_matches('.'.join(ids[::-1]))
         argMatches = []
         for callableMatch in callableMatches:
             try: namedArgs = self._default_arguments(eval(callableMatch,
                                                          self.namespace))
             except: continue
             for namedArg in namedArgs:
                 if namedArg.startswith(text):
                     argMatches.append("%s=" %namedArg)
         return argMatches
 
     def dispatch_custom_completer(self,text):
         #print "Custom! '%s' %s" % (text, self.custom_completers) # dbg
         line = self.full_lbuf        
         if not line.strip():
             return None
 
         event = Struct()
         event.line = line
         event.symbol = text
         cmd = line.split(None,1)[0]
         event.command = cmd
         #print "\ncustom:{%s]\n" % event # dbg
         
         # for foo etc, try also to find completer for %foo
         if not cmd.startswith(self.magic_escape):
             try_magic = self.custom_completers.s_matches(
               self.magic_escape + cmd)            
         else:
             try_magic = []
         
         
         for c in itertools.chain(
                                  self.custom_completers.s_matches(cmd),
                                  try_magic,
                                  self.custom_completers.flat_matches(self.lbuf)):
             #print "try",c # dbg
             try:
                 res = c(event)
                 # first, try case sensitive match
                 withcase = [r for r in res if r.startswith(text)]
                 if withcase:
                     return withcase
                 # if none, then case insensitive ones are ok too
                 return [r for r in res if r.lower().startswith(text.lower())]
             except ipapi.TryNext:
                 pass
             
         return None
                
     def complete(self, text, state,line_buffer=None):
         """Return the next possible completion for 'text'.
 
         This is called successively with state == 0, 1, 2, ... until it
         returns None.  The completion should begin with 'text'.
 
         :Keywords:
         - line_buffer: string
         If not given, the completer attempts to obtain the current line buffer
         via readline.  This keyword allows clients which are requesting for
         text completions in non-readline contexts to inform the completer of
         the entire text.
         """
 
         #print '\n*** COMPLETE: <%s> (%s)' % (text,state)  # dbg
 
         # if there is only a tab on a line with only whitespace, instead
         # of the mostly useless 'do you want to see all million
         # completions' message, just do the right thing and give the user
         # his tab!  Incidentally, this enables pasting of tabbed text from
         # an editor (as long as autoindent is off).
 
         # It should be noted that at least pyreadline still shows
         # file completions - is there a way around it?
         
         # don't apply this on 'dumb' terminals, such as emacs buffers, so we
         # don't interfere with their own tab-completion mechanism.
         if line_buffer is None:
             self.full_lbuf = self.get_line_buffer()
         else:
             self.full_lbuf = line_buffer
             
         if not (self.dumb_terminal or self.full_lbuf.strip()):
             self.readline.insert_text('\t')
             return None
         
         magic_escape = self.magic_escape
         magic_prefix = self.magic_prefix
 
         self.lbuf = self.full_lbuf[:self.get_endidx()]
 
         try:
             if text.startswith(magic_escape):
                 text = text.replace(magic_escape,magic_prefix)
             elif text.startswith('~'):
                 text = os.path.expanduser(text)
             if state == 0:
                 custom_res = self.dispatch_custom_completer(text)
                 if custom_res is not None:
                     # did custom completers produce something?
                     self.matches = custom_res
                 else:
                     # Extend the list of completions with the results of each
                     # matcher, so we return results to the user from all
                     # namespaces.
                     if self.merge_completions:
                         self.matches = []
                         for matcher in self.matchers:
                             self.matches.extend(matcher(text))
                     else:
                         for matcher in self.matchers:
                             self.matches = matcher(text)
                             if self.matches:
                                 break
                     def uniq(alist):
                         set = {}
                         return [set.setdefault(e,e) for e in alist if e not in set]
                     self.matches = uniq(self.matches)                
             try:
                 ret = self.matches[state].replace(magic_prefix,magic_escape)                
                 return ret
             except IndexError:
                 return None
         except:
             #from IPython.ultraTB import AutoFormattedTB; # dbg
             #tb=AutoFormattedTB('Verbose');tb() #dbg
             
             # If completion fails, don't annoy the user.
             return None

IPython/deep_reload.py

@@ -1,186 +1,188 b''
 # -*- coding: utf-8 -*-
 """
 A module to change reload() so that it acts recursively.
 To enable it type:
     >>> import __builtin__, deep_reload
     >>> __builtin__.reload = deep_reload.reload
 You can then disable it with:
     >>> __builtin__.reload = deep_reload.original_reload
     
 Alternatively, you can add a dreload builtin alongside normal reload with:
     >>> __builtin__.dreload = deep_reload.reload
     
 This code is almost entirely based on knee.py from the standard library.
 
 $Id: deep_reload.py 958 2005-12-27 23:17:51Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001 Nathaniel Gray <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release # do it explicitly so pydoc can see it - pydoc bug
 __author__  = '%s <%s>' %  Release.authors['Nathan']
 __license__ = Release.license
 __version__ = "0.5"
 __date__    = "21 August 2001"
 
 import __builtin__
 import imp
 import sys
 
 # Replacement for __import__()
-def deep_import_hook(name, globals=None, locals=None, fromlist=None):
+def deep_import_hook(name, globals=None, locals=None, fromlist=None, level=-1):
+    # For now level is ignored, it's just there to prevent crash  
+    # with from __future__ import absolute_import
     parent = determine_parent(globals)
     q, tail = find_head_package(parent, name)
     m = load_tail(q, tail)
     if not fromlist:
         return q
     if hasattr(m, "__path__"):
         ensure_fromlist(m, fromlist)
     return m
 
 def determine_parent(globals):
     if not globals or  not globals.has_key("__name__"):
         return None
     pname = globals['__name__']
     if globals.has_key("__path__"):
         parent = sys.modules[pname]
         assert globals is parent.__dict__
         return parent
     if '.' in pname:
         i = pname.rfind('.')
         pname = pname[:i]
         parent = sys.modules[pname]
         assert parent.__name__ == pname
         return parent
     return None
 
 def find_head_package(parent, name):
     # Import the first
     if '.' in name:
         # 'some.nested.package' -> head = 'some', tail = 'nested.package'
         i = name.find('.')
         head = name[:i]
         tail = name[i+1:]
     else:
         # 'packagename' -> head = 'packagename', tail = ''
         head = name
         tail = ""
     if parent:
         # If this is a subpackage then qname = parent's name + head
         qname = "%s.%s" % (parent.__name__, head)
     else:
         qname = head
     q = import_module(head, qname, parent)
     if q: return q, tail
     if parent:
         qname = head
         parent = None
         q = import_module(head, qname, parent)
         if q: return q, tail
     raise ImportError, "No module named " + qname
 
 def load_tail(q, tail):
     m = q
     while tail:
         i = tail.find('.')
         if i < 0: i = len(tail)
         head, tail = tail[:i], tail[i+1:]
 
         # fperez: fix dotted.name reloading failures by changing:
         #mname = "%s.%s" % (m.__name__, head)
         # to:
         mname = m.__name__
         # This needs more testing!!! (I don't understand this module too well)
         
         #print '** head,tail=|%s|->|%s|, mname=|%s|' % (head,tail,mname)  # dbg
         m = import_module(head, mname, m)
         if not m:
             raise ImportError, "No module named " + mname
     return m
 
 def ensure_fromlist(m, fromlist, recursive=0):
     for sub in fromlist:
         if sub == "*":
             if not recursive:
                 try:
                     all = m.__all__
                 except AttributeError:
                     pass
                 else:
                     ensure_fromlist(m, all, 1)
             continue
         if sub != "*" and not hasattr(m, sub):
             subname = "%s.%s" % (m.__name__, sub)
             submod = import_module(sub, subname, m)
             if not submod:
                 raise ImportError, "No module named " + subname
 
 # Need to keep track of what we've already reloaded to prevent cyclic evil
 found_now = {}
 
 def import_module(partname, fqname, parent):
     global found_now
     if found_now.has_key(fqname):
         try:
             return sys.modules[fqname]    
         except KeyError:
             pass
     
     print 'Reloading', fqname #, sys.excepthook is sys.__excepthook__, \
             #sys.displayhook is sys.__displayhook__
     
     found_now[fqname] = 1
     try:
         fp, pathname, stuff = imp.find_module(partname,
                                               parent and parent.__path__)
     except ImportError:
         return None
         
     try:
         m = imp.load_module(fqname, fp, pathname, stuff)
     finally:
         if fp: fp.close()
         
     if parent:
         setattr(parent, partname, m)
 
     return m
 
 def deep_reload_hook(module):
     name = module.__name__
     if '.' not in name:
         return import_module(name, name, None)
     i = name.rfind('.')
     pname = name[:i]
     parent = sys.modules[pname]
     return import_module(name[i+1:], name, parent)
 
 # Save the original hooks
 original_reload = __builtin__.reload
 
 # Replacement for reload()
 def reload(module, exclude=['sys', '__builtin__', '__main__']):
     """Recursively reload all modules used in the given module.  Optionally
     takes a list of modules to exclude from reloading.  The default exclude
     list contains sys, __main__, and __builtin__, to prevent, e.g., resetting 
     display, exception, and io hooks.
     """
     global found_now
     for i in exclude:
         found_now[i] = 1
     original_import = __builtin__.__import__
     __builtin__.__import__ = deep_import_hook    
     try:
         ret = deep_reload_hook(module)
     finally:
         __builtin__.__import__ = original_import
         found_now = {}
     return ret
 
 # Uncomment the following to automatically activate deep reloading whenever
 # this module is imported
 #__builtin__.reload = reload

IPython/external/mglob.py

@@ -1,203 +1,203 b''
 #!/usr/bin/env python
 
 r""" mglob - enhanced file list expansion module
 
 Use as stand-alone utility (for xargs, `backticks` etc.), 
 or a globbing library for own python programs. Globbing the sys.argv is something 
 that almost every Windows script has to perform manually, and this module is here
 to help with that task. Also Unix users will benefit from enhanced modes 
 such as recursion, exclusion, directory omission...
 
 Unlike glob.glob, directories are not included in the glob unless specified 
 with 'dir:'
 
 'expand' is the function to use in python programs. Typical use
 to expand argv (esp. in windows)::
 
     try:
         import mglob 
         files = mglob.expand(sys.argv[1:])
     except ImportError:
         print "mglob not found; try 'easy_install mglob' for extra features"
         files = sys.argv[1:] 
 
 Note that for unix, shell expands *normal* wildcards (*.cpp, etc.) in argv.
 Therefore, you might want to use quotes with normal wildcards to prevent this 
 expansion, in order for mglob to see the wildcards and get the wanted behaviour.
 Not quoting the wildcards is harmless and typically has equivalent results, though.
 
 Author: Ville Vainio <[e-mail]>
 License: MIT Open Source license
 
 """
 
 #Assigned in variable for "usage" printing convenience"
 
 globsyntax = """\
-This program allows specifying filenames with "mglob" mechanism.
-Supported syntax in globs (wilcard matching patterns)::
-
- *.cpp ?ellowo*                
-     - obvious. Differs from normal glob in that dirs are not included.
-       Unix users might want to write this as: "*.cpp" "?ellowo*"
- rec:/usr/share=*.txt,*.doc    
-     - get all *.txt and *.doc under /usr/share, 
-       recursively
- rec:/usr/share
-     - All files under /usr/share, recursively
- rec:*.py
-     - All .py files under current working dir, recursively
- foo                           
-     - File or dir foo
- !*.bak readme*                   
-     - readme*, exclude files ending with .bak
- !.svn/ !.hg/ !*_Data/ rec:.
-     - Skip .svn, .hg, foo_Data dirs (and their subdirs) in recurse.
-       Trailing / is the key, \ does not work!
- dir:foo                       
-     - the directory foo if it exists (not files in foo)
- dir:*                         
-     - all directories in current folder
- foo.py bar.* !h* rec:*.py
-     - Obvious. !h* exclusion only applies for rec:*.py.
-       foo.py is *not* included twice.
- @filelist.txt
-     - All files listed in 'filelist.txt' file, on separate lines.
+    This program allows specifying filenames with "mglob" mechanism.
+    Supported syntax in globs (wilcard matching patterns)::
+    
+     *.cpp ?ellowo*                
+         - obvious. Differs from normal glob in that dirs are not included.
+           Unix users might want to write this as: "*.cpp" "?ellowo*"
+     rec:/usr/share=*.txt,*.doc    
+         - get all *.txt and *.doc under /usr/share, 
+           recursively
+     rec:/usr/share
+         - All files under /usr/share, recursively
+     rec:*.py
+         - All .py files under current working dir, recursively
+     foo                           
+         - File or dir foo
+     !*.bak readme*                   
+         - readme*, exclude files ending with .bak
+     !.svn/ !.hg/ !*_Data/ rec:.
+         - Skip .svn, .hg, foo_Data dirs (and their subdirs) in recurse.
+           Trailing / is the key, \ does not work!
+     dir:foo                       
+         - the directory foo if it exists (not files in foo)
+     dir:*                         
+         - all directories in current folder
+     foo.py bar.* !h* rec:*.py
+         - Obvious. !h* exclusion only applies for rec:*.py.
+           foo.py is *not* included twice.
+     @filelist.txt
+         - All files listed in 'filelist.txt' file, on separate lines.
  """
 
 
 __version__ = "0.2"
 
 
 import os,glob,fnmatch,sys
 from sets import Set as set
 
                 
 def expand(flist,exp_dirs = False):
     """ Expand the glob(s) in flist.
     
     flist may be either a whitespace-separated list of globs/files
     or an array of globs/files.
     
     if exp_dirs is true, directory names in glob are expanded to the files
     contained in them - otherwise, directory names are returned as is.
     
     """
     if isinstance(flist, basestring):
         flist = flist.split()
     done_set = set()
     denied_set = set()
 
     def recfind(p, pats = ["*"]):
         denied_dirs = ["*" + d+"*" for d in denied_set if d.endswith("/")]
         #print "de", denied_dirs
         for (dp,dnames,fnames) in os.walk(p):
             # see if we should ignore the whole directory
             dp_norm = dp.replace("\\","/") + "/"
             deny = False
             #print "dp",dp
             for deny_pat in denied_dirs:
                 if fnmatch.fnmatch( dp_norm, deny_pat):
                     deny = True
                     break
             if deny:
                 continue
 
                     
             for f in fnames:
                 matched = False
                 for p in pats:
                     if fnmatch.fnmatch(f,p):
                         matched = True
                         break
                 if matched:
                     yield os.path.join(dp,f)            
 
     def once_filter(seq):
         for it in seq:
             p = os.path.abspath(it)
             if p in done_set:
                 continue
             done_set.add(p)
             deny = False
             for deny_pat in denied_set:
                 if fnmatch.fnmatch(os.path.basename(p), deny_pat):
                     deny = True
                     break
             if not deny:
                 yield it
         return
             
     res = []
 
     for ent in flist:
         ent = os.path.expanduser(os.path.expandvars(ent))
         if ent.lower().startswith('rec:'):
             fields = ent[4:].split('=')            
             if len(fields) == 2:
                 pth, patlist = fields
             elif len(fields) == 1:
                 if os.path.isdir(fields[0]):
                     # single arg is dir
                     pth, patlist = fields[0], '*'
                 else: 
                     # single arg is pattern
                     pth, patlist = '.', fields[0]
                     
             elif len(fields) == 0:
                 pth, pathlist = '.','*'
                 
             pats = patlist.split(',')
             res.extend(once_filter(recfind(pth, pats)))
         # filelist
         elif ent.startswith('@') and os.path.isfile(ent[1:]):
             res.extend(once_filter(open(ent[1:]).read().splitlines()))
         # exclusion
         elif ent.startswith('!'):
             denied_set.add(ent[1:])
         # glob only dirs
         elif ent.lower().startswith('dir:'):
             res.extend(once_filter(filter(os.path.isdir,glob.glob(ent[4:]))))
             
         # get all files in the specified dir
         elif os.path.isdir(ent) and exp_dirs:
             res.extend(once_filter(filter(os.path.isfile,glob.glob(ent + os.sep+"*"))))
             
         # glob only files
 
         elif '*' in ent or '?' in ent:
             res.extend(once_filter(filter(os.path.isfile,glob.glob(ent))))
 
         else:
             res.extend(once_filter([ent]))
     return res
             
             
 def test():
     assert (
         expand("*.py ~/.ipython/*.py rec:/usr/share/doc-base") == 
         expand( ['*.py', '~/.ipython/*.py', 'rec:/usr/share/doc-base'] ) 
         )
     
 def main():
     if len(sys.argv) < 2:
         print globsyntax
         return
     
     print "\n".join(expand(sys.argv[1:])),
 
 def mglob_f(self, arg):
     from IPython.genutils import SList
     if arg.strip():
         return SList(expand(arg))
     print "Please specify pattern!"
     print globsyntax
 
 def init_ipython(ip):
     """ register %mglob for IPython """
     mglob_f.__doc__ = globsyntax
     ip.expose_magic("mglob",mglob_f)  
     
 # test()
 if __name__ == "__main__":
     main()

IPython/external/path.py

@@ -1,970 +1,973 b''
 """ path.py - An object representing a path to a file or directory.
 
 Example:
 
 from IPython.external.path import path
 d = path('/home/guido/bin')
 for f in d.files('*.py'):
     f.chmod(0755)
 
 This module requires Python 2.2 or later.
 
 
 URL:     http://www.jorendorff.com/articles/python/path
 Author:  Jason Orendorff <jason.orendorff\x40gmail\x2ecom> (and others - see the url!)
 Date:    9 Mar 2007
 """
 
 
 # TODO
 #   - Tree-walking functions don't avoid symlink loops.  Matt Harrison
 #     sent me a patch for this.
 #   - Bug in write_text().  It doesn't support Universal newline mode.
 #   - Better error message in listdir() when self isn't a
 #     directory. (On Windows, the error message really sucks.)
 #   - Make sure everything has a good docstring.
 #   - Add methods for regex find and replace.
 #   - guess_content_type() method?
 #   - Perhaps support arguments to touch().
 
 from __future__ import generators
 
-import sys, warnings, os, fnmatch, glob, shutil, codecs, md5
+import sys, warnings, os, fnmatch, glob, shutil, codecs
+# deprecated in python 2.6
+warnings.filterwarnings('ignore', r'.*md5.*')
+import md5
 
 __version__ = '2.2'
 __all__ = ['path']
 
 # Platform-specific support for path.owner
 if os.name == 'nt':
     try:
         import win32security
     except ImportError:
         win32security = None
 else:
     try:
         import pwd
     except ImportError:
         pwd = None
 
 # Pre-2.3 support.  Are unicode filenames supported?
 _base = str
 _getcwd = os.getcwd
 try:
     if os.path.supports_unicode_filenames:
         _base = unicode
         _getcwd = os.getcwdu
 except AttributeError:
     pass
 
 # Pre-2.3 workaround for booleans
 try:
     True, False
 except NameError:
     True, False = 1, 0
 
 # Pre-2.3 workaround for basestring.
 try:
     basestring
 except NameError:
     basestring = (str, unicode)
 
 # Universal newline support
 _textmode = 'r'
 if hasattr(file, 'newlines'):
     _textmode = 'U'
 
 
 class TreeWalkWarning(Warning):
     pass
 
 class path(_base):
     """ Represents a filesystem path.
 
     For documentation on individual methods, consult their
     counterparts in os.path.
     """
 
     # --- Special Python methods.
 
     def __repr__(self):
         return 'path(%s)' % _base.__repr__(self)
 
     # Adding a path and a string yields a path.
     def __add__(self, more):
         try:
             resultStr = _base.__add__(self, more)
         except TypeError:  #Python bug
             resultStr = NotImplemented
         if resultStr is NotImplemented:
             return resultStr
         return self.__class__(resultStr)
 
     def __radd__(self, other):
         if isinstance(other, basestring):
             return self.__class__(other.__add__(self))
         else:
             return NotImplemented
 
     # The / operator joins paths.
     def __div__(self, rel):
         """ fp.__div__(rel) == fp / rel == fp.joinpath(rel)
 
         Join two path components, adding a separator character if
         needed.
         """
         return self.__class__(os.path.join(self, rel))
 
     # Make the / operator work even when true division is enabled.
     __truediv__ = __div__
 
     def getcwd(cls):
         """ Return the current working directory as a path object. """
         return cls(_getcwd())
     getcwd = classmethod(getcwd)
 
 
     # --- Operations on path strings.
 
     isabs = os.path.isabs
     def abspath(self):       return self.__class__(os.path.abspath(self))
     def normcase(self):      return self.__class__(os.path.normcase(self))
     def normpath(self):      return self.__class__(os.path.normpath(self))
     def realpath(self):      return self.__class__(os.path.realpath(self))
     def expanduser(self):    return self.__class__(os.path.expanduser(self))
     def expandvars(self):    return self.__class__(os.path.expandvars(self))
     def dirname(self):       return self.__class__(os.path.dirname(self))
     basename = os.path.basename
 
     def expand(self):
         """ Clean up a filename by calling expandvars(),
         expanduser(), and normpath() on it.
 
         This is commonly everything needed to clean up a filename
         read from a configuration file, for example.
         """
         return self.expandvars().expanduser().normpath()
 
     def _get_namebase(self):
         base, ext = os.path.splitext(self.name)
         return base
 
     def _get_ext(self):
         f, ext = os.path.splitext(_base(self))
         return ext
 
     def _get_drive(self):
         drive, r = os.path.splitdrive(self)
         return self.__class__(drive)
 
     parent = property(
         dirname, None, None,
         """ This path's parent directory, as a new path object.
 
         For example, path('/usr/local/lib/libpython.so').parent == path('/usr/local/lib')
         """)
 
     name = property(
         basename, None, None,
         """ The name of this file or directory without the full path.
 
         For example, path('/usr/local/lib/libpython.so').name == 'libpython.so'
         """)
 
     namebase = property(
         _get_namebase, None, None,
         """ The same as path.name, but with one file extension stripped off.
 
         For example, path('/home/guido/python.tar.gz').name     == 'python.tar.gz',
         but          path('/home/guido/python.tar.gz').namebase == 'python.tar'
         """)
 
     ext = property(
         _get_ext, None, None,
         """ The file extension, for example '.py'. """)
 
     drive = property(
         _get_drive, None, None,
         """ The drive specifier, for example 'C:'.
         This is always empty on systems that don't use drive specifiers.
         """)
 
     def splitpath(self):
         """ p.splitpath() -> Return (p.parent, p.name). """
         parent, child = os.path.split(self)
         return self.__class__(parent), child
 
     def splitdrive(self):
         """ p.splitdrive() -> Return (p.drive, <the rest of p>).
 
         Split the drive specifier from this path.  If there is
         no drive specifier, p.drive is empty, so the return value
         is simply (path(''), p).  This is always the case on Unix.
         """
         drive, rel = os.path.splitdrive(self)
         return self.__class__(drive), rel
 
     def splitext(self):
         """ p.splitext() -> Return (p.stripext(), p.ext).
 
         Split the filename extension from this path and return
         the two parts.  Either part may be empty.
 
         The extension is everything from '.' to the end of the
         last path segment.  This has the property that if
         (a, b) == p.splitext(), then a + b == p.
         """
         filename, ext = os.path.splitext(self)
         return self.__class__(filename), ext
 
     def stripext(self):
         """ p.stripext() -> Remove one file extension from the path.
 
         For example, path('/home/guido/python.tar.gz').stripext()
         returns path('/home/guido/python.tar').
         """
         return self.splitext()[0]
 
     if hasattr(os.path, 'splitunc'):
         def splitunc(self):
             unc, rest = os.path.splitunc(self)
             return self.__class__(unc), rest
 
         def _get_uncshare(self):
             unc, r = os.path.splitunc(self)
             return self.__class__(unc)
 
         uncshare = property(
             _get_uncshare, None, None,
             """ The UNC mount point for this path.
             This is empty for paths on local drives. """)
 
     def joinpath(self, *args):
         """ Join two or more path components, adding a separator
         character (os.sep) if needed.  Returns a new path
         object.
         """
         return self.__class__(os.path.join(self, *args))
 
     def splitall(self):
         r""" Return a list of the path components in this path.
 
         The first item in the list will be a path.  Its value will be
         either os.curdir, os.pardir, empty, or the root directory of
         this path (for example, '/' or 'C:\\').  The other items in
         the list will be strings.
 
         path.path.joinpath(*result) will yield the original path.
         """
         parts = []
         loc = self
         while loc != os.curdir and loc != os.pardir:
             prev = loc
             loc, child = prev.splitpath()
             if loc == prev:
                 break
             parts.append(child)
         parts.append(loc)
         parts.reverse()
         return parts
 
     def relpath(self):
         """ Return this path as a relative path,
         based from the current working directory.
         """
         cwd = self.__class__(os.getcwd())
         return cwd.relpathto(self)
 
     def relpathto(self, dest):
         """ Return a relative path from self to dest.
 
         If there is no relative path from self to dest, for example if
         they reside on different drives in Windows, then this returns
         dest.abspath().
         """
         origin = self.abspath()
         dest = self.__class__(dest).abspath()
 
         orig_list = origin.normcase().splitall()
         # Don't normcase dest!  We want to preserve the case.
         dest_list = dest.splitall()
 
         if orig_list[0] != os.path.normcase(dest_list[0]):
             # Can't get here from there.
             return dest
 
         # Find the location where the two paths start to differ.
         i = 0
         for start_seg, dest_seg in zip(orig_list, dest_list):
             if start_seg != os.path.normcase(dest_seg):
                 break
             i += 1
 
         # Now i is the point where the two paths diverge.
         # Need a certain number of "os.pardir"s to work up
         # from the origin to the point of divergence.
         segments = [os.pardir] * (len(orig_list) - i)
         # Need to add the diverging part of dest_list.
         segments += dest_list[i:]
         if len(segments) == 0:
             # If they happen to be identical, use os.curdir.
             relpath = os.curdir
         else:
             relpath = os.path.join(*segments)
         return self.__class__(relpath)
 
     # --- Listing, searching, walking, and matching
 
     def listdir(self, pattern=None):
         """ D.listdir() -> List of items in this directory.
 
         Use D.files() or D.dirs() instead if you want a listing
         of just files or just subdirectories.
 
         The elements of the list are path objects.
 
         With the optional 'pattern' argument, this only lists
         items whose names match the given pattern.
         """
         names = os.listdir(self)
         if pattern is not None:
             names = fnmatch.filter(names, pattern)
         return [self / child for child in names]
 
     def dirs(self, pattern=None):
         """ D.dirs() -> List of this directory's subdirectories.
 
         The elements of the list are path objects.
         This does not walk recursively into subdirectories
         (but see path.walkdirs).
 
         With the optional 'pattern' argument, this only lists
         directories whose names match the given pattern.  For
         example, d.dirs('build-*').
         """
         return [p for p in self.listdir(pattern) if p.isdir()]
 
     def files(self, pattern=None):
         """ D.files() -> List of the files in this directory.
 
         The elements of the list are path objects.
         This does not walk into subdirectories (see path.walkfiles).
 
         With the optional 'pattern' argument, this only lists files
         whose names match the given pattern.  For example,
         d.files('*.pyc').
         """
         
         return [p for p in self.listdir(pattern) if p.isfile()]
 
     def walk(self, pattern=None, errors='strict'):
         """ D.walk() -> iterator over files and subdirs, recursively.
 
         The iterator yields path objects naming each child item of
         this directory and its descendants.  This requires that
         D.isdir().
 
         This performs a depth-first traversal of the directory tree.
         Each directory is returned just before all its children.
 
         The errors= keyword argument controls behavior when an
         error occurs.  The default is 'strict', which causes an
         exception.  The other allowed values are 'warn', which
         reports the error via warnings.warn(), and 'ignore'.
         """
         if errors not in ('strict', 'warn', 'ignore'):
             raise ValueError("invalid errors parameter")
 
         try:
             childList = self.listdir()
         except Exception:
             if errors == 'ignore':
                 return
             elif errors == 'warn':
                 warnings.warn(
                     "Unable to list directory '%s': %s"
                     % (self, sys.exc_info()[1]),
                     TreeWalkWarning)
                 return
             else:
                 raise
 
         for child in childList:
             if pattern is None or child.fnmatch(pattern):
                 yield child
             try:
                 isdir = child.isdir()
             except Exception:
                 if errors == 'ignore':
                     isdir = False
                 elif errors == 'warn':
                     warnings.warn(
                         "Unable to access '%s': %s"
                         % (child, sys.exc_info()[1]),
                         TreeWalkWarning)
                     isdir = False
                 else:
                     raise
 
             if isdir:
                 for item in child.walk(pattern, errors):
                     yield item
 
     def walkdirs(self, pattern=None, errors='strict'):
         """ D.walkdirs() -> iterator over subdirs, recursively.
 
         With the optional 'pattern' argument, this yields only
         directories whose names match the given pattern.  For
         example, mydir.walkdirs('*test') yields only directories
         with names ending in 'test'.
 
         The errors= keyword argument controls behavior when an
         error occurs.  The default is 'strict', which causes an
         exception.  The other allowed values are 'warn', which
         reports the error via warnings.warn(), and 'ignore'.
         """
         if errors not in ('strict', 'warn', 'ignore'):
             raise ValueError("invalid errors parameter")
 
         try:
             dirs = self.dirs()
         except Exception:
             if errors == 'ignore':
                 return
             elif errors == 'warn':
                 warnings.warn(
                     "Unable to list directory '%s': %s"
                     % (self, sys.exc_info()[1]),
                     TreeWalkWarning)
                 return
             else:
                 raise
 
         for child in dirs:
             if pattern is None or child.fnmatch(pattern):
                 yield child
             for subsubdir in child.walkdirs(pattern, errors):
                 yield subsubdir
 
     def walkfiles(self, pattern=None, errors='strict'):
         """ D.walkfiles() -> iterator over files in D, recursively.
 
         The optional argument, pattern, limits the results to files
         with names that match the pattern.  For example,
         mydir.walkfiles('*.tmp') yields only files with the .tmp
         extension.
         """
         if errors not in ('strict', 'warn', 'ignore'):
             raise ValueError("invalid errors parameter")
 
         try:
             childList = self.listdir()
         except Exception:
             if errors == 'ignore':
                 return
             elif errors == 'warn':
                 warnings.warn(
                     "Unable to list directory '%s': %s"
                     % (self, sys.exc_info()[1]),
                     TreeWalkWarning)
                 return
             else:
                 raise
 
         for child in childList:
             try:
                 isfile = child.isfile()
                 isdir = not isfile and child.isdir()
             except:
                 if errors == 'ignore':
                     continue
                 elif errors == 'warn':
                     warnings.warn(
                         "Unable to access '%s': %s"
                         % (self, sys.exc_info()[1]),
                         TreeWalkWarning)
                     continue
                 else:
                     raise
 
             if isfile:
                 if pattern is None or child.fnmatch(pattern):
                     yield child
             elif isdir:
                 for f in child.walkfiles(pattern, errors):
                     yield f
 
     def fnmatch(self, pattern):
         """ Return True if self.name matches the given pattern.
 
         pattern - A filename pattern with wildcards,
             for example '*.py'.
         """
         return fnmatch.fnmatch(self.name, pattern)
 
     def glob(self, pattern):
         """ Return a list of path objects that match the pattern.
 
         pattern - a path relative to this directory, with wildcards.
 
         For example, path('/users').glob('*/bin/*') returns a list
         of all the files users have in their bin directories.
         """
         cls = self.__class__
         return [cls(s) for s in glob.glob(_base(self / pattern))]
 
 
     # --- Reading or writing an entire file at once.
 
     def open(self, mode='r'):
         """ Open this file.  Return a file object. """
         return file(self, mode)
 
     def bytes(self):
         """ Open this file, read all bytes, return them as a string. """
         f = self.open('rb')
         try:
             return f.read()
         finally:
             f.close()
 
     def write_bytes(self, bytes, append=False):
         """ Open this file and write the given bytes to it.
 
         Default behavior is to overwrite any existing file.
         Call p.write_bytes(bytes, append=True) to append instead.
         """
         if append:
             mode = 'ab'
         else:
             mode = 'wb'
         f = self.open(mode)
         try:
             f.write(bytes)
         finally:
             f.close()
 
     def text(self, encoding=None, errors='strict'):
         r""" Open this file, read it in, return the content as a string.
 
         This uses 'U' mode in Python 2.3 and later, so '\r\n' and '\r'
         are automatically translated to '\n'.
 
         Optional arguments:
 
         encoding - The Unicode encoding (or character set) of
             the file.  If present, the content of the file is
             decoded and returned as a unicode object; otherwise
             it is returned as an 8-bit str.
         errors - How to handle Unicode errors; see help(str.decode)
             for the options.  Default is 'strict'.
         """
         if encoding is None:
             # 8-bit
             f = self.open(_textmode)
             try:
                 return f.read()
             finally:
                 f.close()
         else:
             # Unicode
             f = codecs.open(self, 'r', encoding, errors)
             # (Note - Can't use 'U' mode here, since codecs.open
             # doesn't support 'U' mode, even in Python 2.3.)
             try:
                 t = f.read()
             finally:
                 f.close()
             return (t.replace(u'\r\n', u'\n')
                      .replace(u'\r\x85', u'\n')
                      .replace(u'\r', u'\n')
                      .replace(u'\x85', u'\n')
                      .replace(u'\u2028', u'\n'))
 
     def write_text(self, text, encoding=None, errors='strict', linesep=os.linesep, append=False):
         r""" Write the given text to this file.
 
         The default behavior is to overwrite any existing file;
         to append instead, use the 'append=True' keyword argument.
 
         There are two differences between path.write_text() and
         path.write_bytes(): newline handling and Unicode handling.
         See below.
 
         Parameters:
 
           - text - str/unicode - The text to be written.
 
           - encoding - str - The Unicode encoding that will be used.
             This is ignored if 'text' isn't a Unicode string.
 
           - errors - str - How to handle Unicode encoding errors.
             Default is 'strict'.  See help(unicode.encode) for the
             options.  This is ignored if 'text' isn't a Unicode
             string.
 
           - linesep - keyword argument - str/unicode - The sequence of
             characters to be used to mark end-of-line.  The default is
             os.linesep.  You can also specify None; this means to
             leave all newlines as they are in 'text'.
 
           - append - keyword argument - bool - Specifies what to do if
             the file already exists (True: append to the end of it;
             False: overwrite it.)  The default is False.
 
 
         --- Newline handling.
 
         write_text() converts all standard end-of-line sequences
         ('\n', '\r', and '\r\n') to your platform's default end-of-line
         sequence (see os.linesep; on Windows, for example, the
         end-of-line marker is '\r\n').
 
         If you don't like your platform's default, you can override it
         using the 'linesep=' keyword argument.  If you specifically want
         write_text() to preserve the newlines as-is, use 'linesep=None'.
 
         This applies to Unicode text the same as to 8-bit text, except
         there are three additional standard Unicode end-of-line sequences:
         u'\x85', u'\r\x85', and u'\u2028'.
 
         (This is slightly different from when you open a file for
         writing with fopen(filename, "w") in C or file(filename, 'w')
         in Python.)
 
 
         --- Unicode
 
         If 'text' isn't Unicode, then apart from newline handling, the
         bytes are written verbatim to the file.  The 'encoding' and
         'errors' arguments are not used and must be omitted.
 
         If 'text' is Unicode, it is first converted to bytes using the
         specified 'encoding' (or the default encoding if 'encoding'
         isn't specified).  The 'errors' argument applies only to this
         conversion.
 
         """
         if isinstance(text, unicode):
             if linesep is not None:
                 # Convert all standard end-of-line sequences to
                 # ordinary newline characters.
                 text = (text.replace(u'\r\n', u'\n')
                             .replace(u'\r\x85', u'\n')
                             .replace(u'\r', u'\n')
                             .replace(u'\x85', u'\n')
                             .replace(u'\u2028', u'\n'))
                 text = text.replace(u'\n', linesep)
             if encoding is None:
                 encoding = sys.getdefaultencoding()
             bytes = text.encode(encoding, errors)
         else:
             # It is an error to specify an encoding if 'text' is
             # an 8-bit string.
             assert encoding is None
 
             if linesep is not None:
                 text = (text.replace('\r\n', '\n')
                             .replace('\r', '\n'))
                 bytes = text.replace('\n', linesep)
 
         self.write_bytes(bytes, append)
 
     def lines(self, encoding=None, errors='strict', retain=True):
         r""" Open this file, read all lines, return them in a list.
 
         Optional arguments:
             encoding - The Unicode encoding (or character set) of
                 the file.  The default is None, meaning the content
                 of the file is read as 8-bit characters and returned
                 as a list of (non-Unicode) str objects.
             errors - How to handle Unicode errors; see help(str.decode)
                 for the options.  Default is 'strict'
             retain - If true, retain newline characters; but all newline
                 character combinations ('\r', '\n', '\r\n') are
                 translated to '\n'.  If false, newline characters are
                 stripped off.  Default is True.
 
         This uses 'U' mode in Python 2.3 and later.
         """
         if encoding is None and retain:
             f = self.open(_textmode)
             try:
                 return f.readlines()
             finally:
                 f.close()
         else:
             return self.text(encoding, errors).splitlines(retain)
 
     def write_lines(self, lines, encoding=None, errors='strict',
                     linesep=os.linesep, append=False):
         r""" Write the given lines of text to this file.
 
         By default this overwrites any existing file at this path.
 
         This puts a platform-specific newline sequence on every line.
         See 'linesep' below.
 
         lines - A list of strings.
 
         encoding - A Unicode encoding to use.  This applies only if
             'lines' contains any Unicode strings.
 
         errors - How to handle errors in Unicode encoding.  This
             also applies only to Unicode strings.
 
         linesep - The desired line-ending.  This line-ending is
             applied to every line.  If a line already has any
             standard line ending ('\r', '\n', '\r\n', u'\x85',
             u'\r\x85', u'\u2028'), that will be stripped off and
             this will be used instead.  The default is os.linesep,
             which is platform-dependent ('\r\n' on Windows, '\n' on
             Unix, etc.)  Specify None to write the lines as-is,
             like file.writelines().
 
         Use the keyword argument append=True to append lines to the
         file.  The default is to overwrite the file.  Warning:
         When you use this with Unicode data, if the encoding of the
         existing data in the file is different from the encoding
         you specify with the encoding= parameter, the result is
         mixed-encoding data, which can really confuse someone trying
         to read the file later.
         """
         if append:
             mode = 'ab'
         else:
             mode = 'wb'
         f = self.open(mode)
         try:
             for line in lines:
                 isUnicode = isinstance(line, unicode)
                 if linesep is not None:
                     # Strip off any existing line-end and add the
                     # specified linesep string.
                     if isUnicode:
                         if line[-2:] in (u'\r\n', u'\x0d\x85'):
                             line = line[:-2]
                         elif line[-1:] in (u'\r', u'\n',
                                            u'\x85', u'\u2028'):
                             line = line[:-1]
                     else:
                         if line[-2:] == '\r\n':
                             line = line[:-2]
                         elif line[-1:] in ('\r', '\n'):
                             line = line[:-1]
                     line += linesep
                 if isUnicode:
                     if encoding is None:
                         encoding = sys.getdefaultencoding()
                     line = line.encode(encoding, errors)
                 f.write(line)
         finally:
             f.close()
 
     def read_md5(self):
         """ Calculate the md5 hash for this file.
 
         This reads through the entire file.
         """
         f = self.open('rb')
         try:
             m = md5.new()
             while True:
                 d = f.read(8192)
                 if not d:
                     break
                 m.update(d)
         finally:
             f.close()
         return m.digest()
 
     # --- Methods for querying the filesystem.
 
     exists = os.path.exists
     isdir = os.path.isdir
     isfile = os.path.isfile
     islink = os.path.islink
     ismount = os.path.ismount
 
     if hasattr(os.path, 'samefile'):
         samefile = os.path.samefile
 
     getatime = os.path.getatime
     atime = property(
         getatime, None, None,
         """ Last access time of the file. """)
 
     getmtime = os.path.getmtime
     mtime = property(
         getmtime, None, None,
         """ Last-modified time of the file. """)
 
     if hasattr(os.path, 'getctime'):
         getctime = os.path.getctime
         ctime = property(
             getctime, None, None,
             """ Creation time of the file. """)
 
     getsize = os.path.getsize
     size = property(
         getsize, None, None,
         """ Size of the file, in bytes. """)
 
     if hasattr(os, 'access'):
         def access(self, mode):
             """ Return true if current user has access to this path.
 
             mode - One of the constants os.F_OK, os.R_OK, os.W_OK, os.X_OK
             """
             return os.access(self, mode)
 
     def stat(self):
         """ Perform a stat() system call on this path. """
         return os.stat(self)
 
     def lstat(self):
         """ Like path.stat(), but do not follow symbolic links. """
         return os.lstat(self)
 
     def get_owner(self):
         r""" Return the name of the owner of this file or directory.
 
         This follows symbolic links.
 
         On Windows, this returns a name of the form ur'DOMAIN\User Name'.
         On Windows, a group can own a file or directory.
         """
         if os.name == 'nt':
             if win32security is None:
                 raise Exception("path.owner requires win32all to be installed")
             desc = win32security.GetFileSecurity(
                 self, win32security.OWNER_SECURITY_INFORMATION)
             sid = desc.GetSecurityDescriptorOwner()
             account, domain, typecode = win32security.LookupAccountSid(None, sid)
             return domain + u'\\' + account
         else:
             if pwd is None:
                 raise NotImplementedError("path.owner is not implemented on this platform.")
             st = self.stat()
             return pwd.getpwuid(st.st_uid).pw_name
 
     owner = property(
         get_owner, None, None,
         """ Name of the owner of this file or directory. """)
 
     if hasattr(os, 'statvfs'):
         def statvfs(self):
             """ Perform a statvfs() system call on this path. """
             return os.statvfs(self)
 
     if hasattr(os, 'pathconf'):
         def pathconf(self, name):
             return os.pathconf(self, name)
 
 
     # --- Modifying operations on files and directories
 
     def utime(self, times):
         """ Set the access and modified times of this file. """
         os.utime(self, times)
 
     def chmod(self, mode):
         os.chmod(self, mode)
 
     if hasattr(os, 'chown'):
         def chown(self, uid, gid):
             os.chown(self, uid, gid)
 
     def rename(self, new):
         os.rename(self, new)
 
     def renames(self, new):
         os.renames(self, new)
 
 
     # --- Create/delete operations on directories
 
     def mkdir(self, mode=0777):
         os.mkdir(self, mode)
 
     def makedirs(self, mode=0777):
         os.makedirs(self, mode)
 
     def rmdir(self):
         os.rmdir(self)
 
     def removedirs(self):
         os.removedirs(self)
 
 
     # --- Modifying operations on files
 
     def touch(self):
         """ Set the access/modified times of this file to the current time.
         Create the file if it does not exist.
         """
         fd = os.open(self, os.O_WRONLY | os.O_CREAT, 0666)
         os.close(fd)
         os.utime(self, None)
 
     def remove(self):
         os.remove(self)
 
     def unlink(self):
         os.unlink(self)
 
 
     # --- Links
 
     if hasattr(os, 'link'):
         def link(self, newpath):
             """ Create a hard link at 'newpath', pointing to this file. """
             os.link(self, newpath)
 
     if hasattr(os, 'symlink'):
         def symlink(self, newlink):
             """ Create a symbolic link at 'newlink', pointing here. """
             os.symlink(self, newlink)
 
     if hasattr(os, 'readlink'):
         def readlink(self):
             """ Return the path to which this symbolic link points.
 
             The result may be an absolute or a relative path.
             """
             return self.__class__(os.readlink(self))
 
         def readlinkabs(self):
             """ Return the path to which this symbolic link points.
 
             The result is always an absolute path.
             """
             p = self.readlink()
             if p.isabs():
                 return p
             else:
                 return (self.parent / p).abspath()
 
 
     # --- High-level functions from shutil
 
     copyfile = shutil.copyfile
     copymode = shutil.copymode
     copystat = shutil.copystat
     copy = shutil.copy
     copy2 = shutil.copy2
     copytree = shutil.copytree
     if hasattr(shutil, 'move'):
         move = shutil.move
     rmtree = shutil.rmtree
 
 
     # --- Special stuff from os
 
     if hasattr(os, 'chroot'):
         def chroot(self):
             os.chroot(self)
 
     if hasattr(os, 'startfile'):
         def startfile(self):
             os.startfile(self)
 

IPython/genutils.py

@@ -1,2008 +1,2043 b''
 # -*- coding: utf-8 -*-
 """
 General purpose utilities.
 
 This is a grab-bag of stuff I find useful in most programs I write. Some of
 these things are also convenient when working at the command line.
 
 $Id: genutils.py 2998 2008-01-31 10:06:04Z vivainio $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Fernando']
 __license__ = Release.license
 
 #****************************************************************************
 # required modules from the Python standard library
 import __main__
 import commands
-import doctest
+try:
+    import doctest
+except ImportError:
+    pass
 import os
 import re
 import shlex
 import shutil
 import sys
 import tempfile
 import time
 import types
 import warnings
 
+# Curses and termios are Unix-only modules
+try:
+    import curses
+    # We need termios as well, so if its import happens to raise, we bail on
+    # using curses altogether.
+    import termios
+except ImportError:
+    USE_CURSES = False
+else:
+    # Curses on Solaris may not be complete, so we can't use it there
+    USE_CURSES = hasattr(curses,'initscr')
+
 # Other IPython utilities
 import IPython
 from IPython.Itpl import Itpl,itpl,printpl
 from IPython import DPyGetOpt, platutils
 from IPython.generics import result_display
 import IPython.ipapi
 from IPython.external.path import path
 if os.name == "nt":
     from IPython.winconsole import get_console_size
 
 try:
     set
 except:
     from sets import Set as set
 
 
 #****************************************************************************
 # Exceptions
 class Error(Exception):
     """Base class for exceptions in this module."""
     pass
 
 #----------------------------------------------------------------------------
 class IOStream:
     def __init__(self,stream,fallback):
         if not hasattr(stream,'write') or not hasattr(stream,'flush'):
             stream = fallback
         self.stream = stream
         self._swrite = stream.write
         self.flush = stream.flush
 
     def write(self,data):
         try:
             self._swrite(data)
         except:
             try:
                 # print handles some unicode issues which may trip a plain
                 # write() call.  Attempt to emulate write() by using a
                 # trailing comma
                 print >> self.stream, data,
             except:
                 # if we get here, something is seriously broken.
                 print >> sys.stderr, \
                       'ERROR - failed to write data to stream:', self.stream
 
     def close(self):
         pass
                 
         
 class IOTerm:
     """ Term holds the file or file-like objects for handling I/O operations.
 
     These are normally just sys.stdin, sys.stdout and sys.stderr but for
     Windows they can can replaced to allow editing the strings before they are
     displayed."""
 
     # In the future, having IPython channel all its I/O operations through
     # this class will make it easier to embed it into other environments which
     # are not a normal terminal (such as a GUI-based shell)
     def __init__(self,cin=None,cout=None,cerr=None):
         self.cin  = IOStream(cin,sys.stdin)
         self.cout = IOStream(cout,sys.stdout)
         self.cerr = IOStream(cerr,sys.stderr)
         
 # Global variable to be used for all I/O
 Term = IOTerm()
 
 import IPython.rlineimpl as readline
 # Remake Term to use the readline i/o facilities
 if sys.platform == 'win32' and readline.have_readline:
     
     Term = IOTerm(cout=readline._outputfile,cerr=readline._outputfile)
     
 
 #****************************************************************************
 # Generic warning/error printer, used by everything else
 def warn(msg,level=2,exit_val=1):
     """Standard warning printer. Gives formatting consistency.
 
     Output is sent to Term.cerr (sys.stderr by default).
     
     Options:
     
     -level(2): allows finer control:
       0 -> Do nothing, dummy function.
       1 -> Print message.
       2 -> Print 'WARNING:' + message. (Default level).
       3 -> Print 'ERROR:' + message.
       4 -> Print 'FATAL ERROR:' + message and trigger a sys.exit(exit_val).
 
     -exit_val (1): exit value returned by sys.exit() for a level 4
     warning. Ignored for all other levels."""
     
     if level>0:
         header = ['','','WARNING: ','ERROR: ','FATAL ERROR: ']
         print >> Term.cerr, '%s%s' % (header[level],msg)
         if level == 4:
             print >> Term.cerr,'Exiting.\n'
             sys.exit(exit_val)
 
 def info(msg):
     """Equivalent to warn(msg,level=1)."""
 
     warn(msg,level=1)
 
 def error(msg):
     """Equivalent to warn(msg,level=3)."""
 
     warn(msg,level=3)
 
 def fatal(msg,exit_val=1):
     """Equivalent to warn(msg,exit_val=exit_val,level=4)."""
 
     warn(msg,exit_val=exit_val,level=4)
 
 #---------------------------------------------------------------------------
 # Debugging routines
 # 
 def debugx(expr,pre_msg=''):
     """Print the value of an expression from the caller's frame.
     
     Takes an expression, evaluates it in the caller's frame and prints both
     the given expression and the resulting value (as well as a debug mark
     indicating the name of the calling function.  The input must be of a form
     suitable for eval().
 
     An optional message can be passed, which will be prepended to the printed
     expr->value pair."""
     
     cf = sys._getframe(1)
     print '[DBG:%s] %s%s -> %r' % (cf.f_code.co_name,pre_msg,expr,
                                    eval(expr,cf.f_globals,cf.f_locals))
 
 # deactivate it by uncommenting the following line, which makes it a no-op
 #def debugx(expr,pre_msg=''): pass
 
 #----------------------------------------------------------------------------
 StringTypes = types.StringTypes
 
 # Basic timing functionality
 
 # If possible (Unix), use the resource module instead of time.clock()
 try:
     import resource
     def clocku():
         """clocku() -> floating point number
 
         Return the *USER* CPU time in seconds since the start of the process.
         This is done via a call to resource.getrusage, so it avoids the
         wraparound problems in time.clock()."""
         
         return resource.getrusage(resource.RUSAGE_SELF)[0]
     
     def clocks():
         """clocks() -> floating point number
 
         Return the *SYSTEM* CPU time in seconds since the start of the process.
         This is done via a call to resource.getrusage, so it avoids the
         wraparound problems in time.clock()."""
         
         return resource.getrusage(resource.RUSAGE_SELF)[1]
     
     def clock():
         """clock() -> floating point number
 
         Return the *TOTAL USER+SYSTEM* CPU time in seconds since the start of
         the process.  This is done via a call to resource.getrusage, so it
         avoids the wraparound problems in time.clock()."""
 
         u,s = resource.getrusage(resource.RUSAGE_SELF)[:2] 
         return u+s
     
     def clock2():
         """clock2() -> (t_user,t_system)
 
         Similar to clock(), but return a tuple of user/system times."""
         return resource.getrusage(resource.RUSAGE_SELF)[:2]
 
 except ImportError:
     # There is no distinction of user/system time under windows, so we just use
     # time.clock() for everything...
     clocku = clocks = clock = time.clock
     def clock2():
         """Under windows, system CPU time can't be measured.
 
         This just returns clock() and zero."""
         return time.clock(),0.0
 
 def timings_out(reps,func,*args,**kw):
     """timings_out(reps,func,*args,**kw) -> (t_total,t_per_call,output)
 
     Execute a function reps times, return a tuple with the elapsed total
     CPU time in seconds, the time per call and the function's output.
 
     Under Unix, the return value is the sum of user+system time consumed by
     the process, computed via the resource module.  This prevents problems
     related to the wraparound effect which the time.clock() function has.
     
     Under Windows the return value is in wall clock seconds. See the
     documentation for the time module for more details."""
 
     reps = int(reps)
     assert reps >=1, 'reps must be >= 1'
     if reps==1:
         start = clock()
         out = func(*args,**kw)
         tot_time = clock()-start
     else:
         rng = xrange(reps-1) # the last time is executed separately to store output
         start = clock()
         for dummy in rng: func(*args,**kw)
         out = func(*args,**kw)  # one last time
         tot_time = clock()-start
     av_time = tot_time / reps
     return tot_time,av_time,out
 
 def timings(reps,func,*args,**kw):
     """timings(reps,func,*args,**kw) -> (t_total,t_per_call)
 
     Execute a function reps times, return a tuple with the elapsed total CPU
     time in seconds and the time per call. These are just the first two values
     in timings_out()."""
 
     return timings_out(reps,func,*args,**kw)[0:2]
 
 def timing(func,*args,**kw):
     """timing(func,*args,**kw) -> t_total
 
     Execute a function once, return the elapsed total CPU time in
     seconds. This is just the first value in timings_out()."""
 
     return timings_out(1,func,*args,**kw)[0]
 
 #****************************************************************************
 # file and system
 
 def arg_split(s,posix=False):
     """Split a command line's arguments in a shell-like manner.
 
     This is a modified version of the standard library's shlex.split()
     function, but with a default of posix=False for splitting, so that quotes
     in inputs are respected."""
 
     # XXX - there may be unicode-related problems here!!!  I'm not sure that
     # shlex is truly unicode-safe, so it might be necessary to do
     #
     # s = s.encode(sys.stdin.encoding)
     #
     # first, to ensure that shlex gets a normal string.  Input from anyone who
     # knows more about unicode and shlex than I would be good to have here...
     lex = shlex.shlex(s, posix=posix)
     lex.whitespace_split = True
     return list(lex)
 
 def system(cmd,verbose=0,debug=0,header=''):
     """Execute a system command, return its exit status.
 
     Options:
 
     - verbose (0): print the command to be executed.
     
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     stat = 0
     if verbose or debug: print header+cmd
     sys.stdout.flush()
     if not debug: stat = os.system(cmd)
     return stat
 
 def abbrev_cwd():
     """ Return abbreviated version of cwd, e.g. d:mydir """
     cwd = os.getcwd().replace('\\','/')
     drivepart = ''
     tail = cwd
     if sys.platform == 'win32':
         if len(cwd) < 4:
             return cwd
         drivepart,tail = os.path.splitdrive(cwd)
     
 
     parts = tail.split('/')    
     if len(parts) > 2:
         tail = '/'.join(parts[-2:])
         
     return (drivepart + (
         cwd == '/' and '/' or tail))
     
 
 # This function is used by ipython in a lot of places to make system calls.
 # We need it to be slightly different under win32, due to the vagaries of
 # 'network shares'.  A win32 override is below.
 
 def shell(cmd,verbose=0,debug=0,header=''):
     """Execute a command in the system shell, always return None.
 
     Options:
 
     - verbose (0): print the command to be executed.
     
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: this is similar to genutils.system(), but it returns None so it can
     be conveniently used in interactive loops without getting the return value
     (typically 0) printed many times."""
 
     stat = 0
     if verbose or debug: print header+cmd
     # flush stdout so we don't mangle python's buffering
     sys.stdout.flush()
     
     if not debug:
         platutils.set_term_title("IPy " + cmd)
         os.system(cmd)
         platutils.set_term_title("IPy " + abbrev_cwd())
 
 # override shell() for win32 to deal with network shares
 if os.name in ('nt','dos'):
 
     shell_ori = shell
 
     def shell(cmd,verbose=0,debug=0,header=''):
         if os.getcwd().startswith(r"\\"):
             path = os.getcwd()
             # change to c drive (cannot be on UNC-share when issuing os.system,
             # as cmd.exe cannot handle UNC addresses)
             os.chdir("c:")
             # issue pushd to the UNC-share and then run the command
             try:
                 shell_ori('"pushd %s&&"'%path+cmd,verbose,debug,header)
             finally:
                 os.chdir(path)
         else:
             shell_ori(cmd,verbose,debug,header)
 
     shell.__doc__ = shell_ori.__doc__
 
 def getoutput(cmd,verbose=0,debug=0,header='',split=0):
     """Dummy substitute for perl's backquotes.
 
     Executes a command and returns the output.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, the output is returned as a list split on newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class.
     
     This is pretty much deprecated and rarely used, 
     genutils.getoutputerror may be what you need.
     
     """
 
     if verbose or debug: print header+cmd
     if not debug:
         output = os.popen(cmd).read()
         # stipping last \n is here for backwards compat.
         if output.endswith('\n'):
             output = output[:-1]
         if split:
             return output.split('\n')
         else:
             return output
 
 def getoutputerror(cmd,verbose=0,debug=0,header='',split=0):
     """Return (standard output,standard error) of executing cmd in a shell.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, each of stdout/err is returned as a list split on
     newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     if verbose or debug: print header+cmd
     if not cmd:
         if split:
             return [],[]
         else:
             return '',''
     if not debug:
         pin,pout,perr = os.popen3(cmd)
         tout = pout.read().rstrip()
         terr = perr.read().rstrip()
         pin.close()
         pout.close()
         perr.close()
         if split:
             return tout.split('\n'),terr.split('\n')
         else:
             return tout,terr
 
 # for compatibility with older naming conventions
 xsys = system
 bq = getoutput
 
 class SystemExec:
     """Access the system and getoutput functions through a stateful interface.
 
     Note: here we refer to the system and getoutput functions from this
     library, not the ones from the standard python library.
     
     This class offers the system and getoutput functions as methods, but the
     verbose, debug and header parameters can be set for the instance (at
     creation time or later) so that they don't need to be specified on each
     call.
 
     For efficiency reasons, there's no way to override the parameters on a
     per-call basis other than by setting instance attributes. If you need
     local overrides, it's best to directly call system() or getoutput().
 
     The following names are provided as alternate options:
      - xsys: alias to system
      - bq: alias to getoutput
 
     An instance can then be created as:
      >>> sysexec = SystemExec(verbose=1,debug=0,header='Calling: ')
 
     And used as:
      >>> sysexec.xsys('pwd')
      >>> dirlist = sysexec.bq('ls -l')
     """
     
     def __init__(self,verbose=0,debug=0,header='',split=0):
         """Specify the instance's values for verbose, debug and header."""
         setattr_list(self,'verbose debug header split')
 
     def system(self,cmd):
         """Stateful interface to system(), with the same keyword parameters."""
 
         system(cmd,self.verbose,self.debug,self.header)
 
     def shell(self,cmd):
         """Stateful interface to shell(), with the same keyword parameters."""
 
         shell(cmd,self.verbose,self.debug,self.header)
 
     xsys = system  # alias
 
     def getoutput(self,cmd):
         """Stateful interface to getoutput()."""
 
         return getoutput(cmd,self.verbose,self.debug,self.header,self.split)
 
     def getoutputerror(self,cmd):
         """Stateful interface to getoutputerror()."""
 
         return getoutputerror(cmd,self.verbose,self.debug,self.header,self.split)
 
     bq = getoutput  # alias
 
 #-----------------------------------------------------------------------------
 def mutex_opts(dict,ex_op):
     """Check for presence of mutually exclusive keys in a dict.
 
     Call: mutex_opts(dict,[[op1a,op1b],[op2a,op2b]...]"""
     for op1,op2 in ex_op:
         if op1 in dict and op2 in dict:
             raise ValueError,'\n*** ERROR in Arguments *** '\
                   'Options '+op1+' and '+op2+' are mutually exclusive.'
 
 #-----------------------------------------------------------------------------
 def get_py_filename(name):
     """Return a valid python filename in the current directory.
 
     If the given name is not a file, it adds '.py' and searches again.
     Raises IOError with an informative message if the file isn't found."""
 
     name = os.path.expanduser(name)
     if not os.path.isfile(name) and not name.endswith('.py'):
         name += '.py'
     if os.path.isfile(name):
         return name
     else:
         raise IOError,'File `%s` not found.' % name
 
 #-----------------------------------------------------------------------------
 def filefind(fname,alt_dirs = None):
     """Return the given filename either in the current directory, if it
     exists, or in a specified list of directories.
 
     ~ expansion is done on all file and directory names.
 
     Upon an unsuccessful search, raise an IOError exception."""
 
     if alt_dirs is None:
         try:
             alt_dirs = get_home_dir()
         except HomeDirError:
             alt_dirs = os.getcwd()
     search = [fname] + list_strings(alt_dirs)
     search = map(os.path.expanduser,search)
     #print 'search list for',fname,'list:',search  # dbg
     fname = search[0]
     if os.path.isfile(fname):
         return fname
     for direc in search[1:]:
         testname = os.path.join(direc,fname)
         #print 'testname',testname  # dbg
         if os.path.isfile(testname):
             return testname
     raise IOError,'File' + `fname` + \
           ' not found in current or supplied directories:' + `alt_dirs`
 
 #----------------------------------------------------------------------------
 def file_read(filename):
     """Read a file and close it.  Returns the file source."""
     fobj = open(filename,'r');
     source = fobj.read();
     fobj.close()
     return source
 
 def file_readlines(filename):
     """Read a file and close it.  Returns the file source using readlines()."""
     fobj = open(filename,'r');
     lines = fobj.readlines();
     fobj.close()
     return lines
 
 #----------------------------------------------------------------------------
 def target_outdated(target,deps):
     """Determine whether a target is out of date.
 
     target_outdated(target,deps) -> 1/0
 
     deps: list of filenames which MUST exist.
     target: single filename which may or may not exist.
 
     If target doesn't exist or is older than any file listed in deps, return
     true, otherwise return false.
     """
     try:
         target_time = os.path.getmtime(target)
     except os.error:
         return 1
     for dep in deps:
         dep_time = os.path.getmtime(dep)
         if dep_time > target_time:
             #print "For target",target,"Dep failed:",dep # dbg
             #print "times (dep,tar):",dep_time,target_time # dbg
             return 1
     return 0
 
 #-----------------------------------------------------------------------------
 def target_update(target,deps,cmd):
     """Update a target with a given command given a list of dependencies.
 
     target_update(target,deps,cmd) -> runs cmd if target is outdated.
 
     This is just a wrapper around target_outdated() which calls the given
     command if target is outdated."""
 
     if target_outdated(target,deps):
         xsys(cmd)
 
 #----------------------------------------------------------------------------
 def unquote_ends(istr):
     """Remove a single pair of quotes from the endpoints of a string."""
 
     if not istr:
         return istr
     if (istr[0]=="'" and istr[-1]=="'") or \
        (istr[0]=='"' and istr[-1]=='"'):
         return istr[1:-1]
     else:
         return istr
 
 #----------------------------------------------------------------------------
 def process_cmdline(argv,names=[],defaults={},usage=''):
     """ Process command-line options and arguments.
 
     Arguments:
 
     - argv: list of arguments, typically sys.argv.
     
     - names: list of option names. See DPyGetOpt docs for details on options
     syntax.
 
     - defaults: dict of default values.
 
     - usage: optional usage notice to print if a wrong argument is passed.
 
     Return a dict of options and a list of free arguments."""
 
     getopt = DPyGetOpt.DPyGetOpt()
     getopt.setIgnoreCase(0)
     getopt.parseConfiguration(names)
 
     try:
         getopt.processArguments(argv)
     except DPyGetOpt.ArgumentError, exc:
         print usage
         warn('"%s"' % exc,level=4)
 
     defaults.update(getopt.optionValues)
     args = getopt.freeValues
     
     return defaults,args
 
 #----------------------------------------------------------------------------
 def optstr2types(ostr):
     """Convert a string of option names to a dict of type mappings.
 
     optstr2types(str) -> {None:'string_opts',int:'int_opts',float:'float_opts'}
 
     This is used to get the types of all the options in a string formatted
     with the conventions of DPyGetOpt. The 'type' None is used for options
     which are strings (they need no further conversion). This function's main
     use is to get a typemap for use with read_dict().
     """
 
     typeconv = {None:'',int:'',float:''}
     typemap = {'s':None,'i':int,'f':float}
     opt_re = re.compile(r'([\w]*)([^:=]*:?=?)([sif]?)')
 
     for w in ostr.split():
         oname,alias,otype = opt_re.match(w).groups()
         if otype == '' or alias == '!':   # simple switches are integers too
             otype = 'i'
         typeconv[typemap[otype]] += oname + ' '
     return typeconv
 
 #----------------------------------------------------------------------------
 def read_dict(filename,type_conv=None,**opt):
 
     """Read a dictionary of key=value pairs from an input file, optionally
     performing conversions on the resulting values.
 
     read_dict(filename,type_conv,**opt) -> dict
 
     Only one value per line is accepted, the format should be
      # optional comments are ignored
      key value\n
 
     Args:
 
       - type_conv: A dictionary specifying which keys need to be converted to
       which types. By default all keys are read as strings. This dictionary
       should have as its keys valid conversion functions for strings
       (int,long,float,complex, or your own).  The value for each key
       (converter) should be a whitespace separated string containing the names
       of all the entries in the file to be converted using that function. For
       keys to be left alone, use None as the conversion function (only needed
       with purge=1, see below).
 
       - opt: dictionary with extra options as below (default in parens)
 
         purge(0): if set to 1, all keys *not* listed in type_conv are purged out
         of the dictionary to be returned. If purge is going to be used, the
         set of keys to be left as strings also has to be explicitly specified
         using the (non-existent) conversion function None.
 
         fs(None): field separator. This is the key/value separator to be used
         when parsing the file. The None default means any whitespace [behavior
         of string.split()].
 
         strip(0): if 1, strip string values of leading/trailinig whitespace.
 
         warn(1): warning level if requested keys are not found in file.
           - 0: silently ignore.
           - 1: inform but proceed.
           - 2: raise KeyError exception.
 
         no_empty(0): if 1, remove keys with whitespace strings as a value.
 
         unique([]): list of keys (or space separated string) which can't be
         repeated. If one such key is found in the file, each new instance
         overwrites the previous one. For keys not listed here, the behavior is
         to make a list of all appearances.
 
     Example:
     If the input file test.ini has:
       i 3
       x 4.5
       y 5.5
       s hi ho
     Then:
 
     >>> type_conv={int:'i',float:'x',None:'s'}
     >>> read_dict('test.ini')
     {'i': '3', 's': 'hi ho', 'x': '4.5', 'y': '5.5'}
     >>> read_dict('test.ini',type_conv)
     {'i': 3, 's': 'hi ho', 'x': 4.5, 'y': '5.5'}
     >>> read_dict('test.ini',type_conv,purge=1)
     {'i': 3, 's': 'hi ho', 'x': 4.5}
     """
 
     # starting config
     opt.setdefault('purge',0)
     opt.setdefault('fs',None)  # field sep defaults to any whitespace
     opt.setdefault('strip',0)
     opt.setdefault('warn',1)
     opt.setdefault('no_empty',0)
     opt.setdefault('unique','')
     if type(opt['unique']) in StringTypes:
         unique_keys = qw(opt['unique'])
     elif type(opt['unique']) in (types.TupleType,types.ListType):
         unique_keys = opt['unique']
     else:
         raise ValueError, 'Unique keys must be given as a string, List or Tuple'
 
     dict = {}
     # first read in table of values as strings
     file = open(filename,'r')
     for line in file.readlines():
         line = line.strip()
         if len(line) and line[0]=='#': continue
         if len(line)>0:
             lsplit = line.split(opt['fs'],1)
             try:
                 key,val = lsplit
             except ValueError:
                 key,val = lsplit[0],''
             key = key.strip()
             if opt['strip']: val = val.strip()
             if val == "''" or val == '""': val = ''
             if opt['no_empty'] and (val=='' or val.isspace()):
                 continue
             # if a key is found more than once in the file, build a list
             # unless it's in the 'unique' list. In that case, last found in file
             # takes precedence. User beware.
             try:
                 if dict[key] and key in unique_keys:
                     dict[key] = val
                 elif type(dict[key]) is types.ListType:
                     dict[key].append(val)
                 else:
                     dict[key] = [dict[key],val]
             except KeyError:
                 dict[key] = val
     # purge if requested
     if opt['purge']:
         accepted_keys = qwflat(type_conv.values())
         for key in dict.keys():
             if key in accepted_keys: continue
             del(dict[key])
     # now convert if requested
     if type_conv==None: return dict
     conversions = type_conv.keys()
     try: conversions.remove(None)
     except: pass
     for convert in conversions:
         for val in qw(type_conv[convert]):
             try:
                 dict[val] = convert(dict[val])
             except KeyError,e:
                 if opt['warn'] == 0:
                     pass
                 elif opt['warn'] == 1:
                     print >>sys.stderr, 'Warning: key',val,\
                           'not found in file',filename
                 elif opt['warn'] == 2:
                     raise KeyError,e
                 else:
                     raise ValueError,'Warning level must be 0,1 or 2'
 
     return dict
 
 #----------------------------------------------------------------------------
 def flag_calls(func):
     """Wrap a function to detect and flag when it gets called.
 
     This is a decorator which takes a function and wraps it in a function with
     a 'called' attribute. wrapper.called is initialized to False.
 
     The wrapper.called attribute is set to False right before each call to the
     wrapped function, so if the call fails it remains False.  After the call
     completes, wrapper.called is set to True and the output is returned.
 
     Testing for truth in wrapper.called allows you to determine if a call to
     func() was attempted and succeeded."""
     
     def wrapper(*args,**kw):
         wrapper.called = False
         out = func(*args,**kw)
         wrapper.called = True
         return out
 
     wrapper.called = False
     wrapper.__doc__ = func.__doc__
     return wrapper
 
 #----------------------------------------------------------------------------
 def dhook_wrap(func,*a,**k):
     """Wrap a function call in a sys.displayhook controller.
 
     Returns a wrapper around func which calls func, with all its arguments and
     keywords unmodified, using the default sys.displayhook.  Since IPython
     modifies sys.displayhook, it breaks the behavior of certain systems that
     rely on the default behavior, notably doctest.
     """
 
     def f(*a,**k):
     
         dhook_s = sys.displayhook
         sys.displayhook = sys.__displayhook__
         try:
             out = func(*a,**k)
         finally:
             sys.displayhook = dhook_s
 
         return out
 
     f.__doc__ = func.__doc__
     return f
 
 #----------------------------------------------------------------------------
 def doctest_reload():
     """Properly reload doctest to reuse it interactively.
 
     This routine:
 
       - reloads doctest
       
       - resets its global 'master' attribute to None, so that multiple uses of
       the module interactively don't produce cumulative reports.
     
       - Monkeypatches its core test runner method to protect it from IPython's
       modified displayhook.  Doctest expects the default displayhook behavior
       deep down, so our modification breaks it completely.  For this reason, a
       hard monkeypatch seems like a reasonable solution rather than asking
       users to manually use a different doctest runner when under IPython."""
 
     import doctest
     reload(doctest)
     doctest.master=None
 
     try:
         doctest.DocTestRunner
     except AttributeError:
         # This is only for python 2.3 compatibility, remove once we move to
         # 2.4 only.
         pass
     else:
         doctest.DocTestRunner.run = dhook_wrap(doctest.DocTestRunner.run)
 
 #----------------------------------------------------------------------------
 class HomeDirError(Error):
     pass
 
 def get_home_dir():
     """Return the closest possible equivalent to a 'home' directory.
 
     We first try $HOME.  Absent that, on NT it's $HOMEDRIVE\$HOMEPATH.
 
     Currently only Posix and NT are implemented, a HomeDirError exception is
     raised for all other OSes. """
 
     isdir = os.path.isdir
     env = os.environ
     
     # first, check py2exe distribution root directory for _ipython.
     # This overrides all. Normally does not exist.
     
     if '\\library.zip\\' in IPython.__file__.lower():
         root, rest = IPython.__file__.lower().split('library.zip')
         if isdir(root + '_ipython'):
             os.environ["IPYKITROOT"] = root.rstrip('\\')
             return root
     
     try:
         homedir = env['HOME']
         if not isdir(homedir):
             # in case a user stuck some string which does NOT resolve to a
             # valid path, it's as good as if we hadn't foud it
             raise KeyError
         return homedir
     except KeyError:
         if os.name == 'posix':
             raise HomeDirError,'undefined $HOME, IPython can not proceed.'
         elif os.name == 'nt':
             # For some strange reason, win9x returns 'nt' for os.name.
             try:
                 homedir = os.path.join(env['HOMEDRIVE'],env['HOMEPATH'])
                 if not isdir(homedir):
                     homedir = os.path.join(env['USERPROFILE'])
                     if not isdir(homedir):
                         raise HomeDirError
                 return homedir
             except:
                 try:
                     # Use the registry to get the 'My Documents' folder.
                     import _winreg as wreg
                     key = wreg.OpenKey(wreg.HKEY_CURRENT_USER,
                                        "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
                     homedir = wreg.QueryValueEx(key,'Personal')[0]
                     key.Close()
                     if not isdir(homedir):
                         e = ('Invalid "Personal" folder registry key '
                              'typically "My Documents".\n'
                              'Value: %s\n'
                              'This is not a valid directory on your system.' %
                              homedir)
                         raise HomeDirError(e)
                     return homedir
                 except HomeDirError:
                     raise
                 except:
                     return 'C:\\'
         elif os.name == 'dos':
             # Desperate, may do absurd things in classic MacOS. May work under DOS.
             return 'C:\\'
         else:
             raise HomeDirError,'support for your operating system not implemented.'
 
 #****************************************************************************
 # strings and text
 
 class LSString(str):
     """String derivative with a special access attributes.
 
     These are normal strings, but with the special attributes:
 
         .l (or .list) : value as list (split on newlines).
         .n (or .nlstr): original value (the string itself).
         .s (or .spstr): value as whitespace-separated string.
         .p (or .paths): list of path objects
         
     Any values which require transformations are computed only once and
     cached.
 
     Such strings are very useful to efficiently interact with the shell, which
     typically only understands whitespace-separated options for commands."""
 
     def get_list(self):
         try:
             return self.__list
         except AttributeError:
             self.__list = self.split('\n')
             return self.__list
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = self.replace('\n',' ')
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         return self
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self.split('\n') if os.path.exists(p)]
             return self.__paths
     
     p = paths = property(get_paths)
 
 def print_lsstring(arg):
     """ Prettier (non-repr-like) and more informative printer for LSString """
     print "LSString (.p, .n, .l, .s available). Value:"
     print arg
     
 print_lsstring = result_display.when_type(LSString)(print_lsstring)
 
 #----------------------------------------------------------------------------
 class SList(list):
     """List derivative with a special access attributes.
 
     These are normal lists, but with the special attributes:
 
         .l (or .list) : value as list (the list itself).
         .n (or .nlstr): value as a string, joined on newlines.
         .s (or .spstr): value as a string, joined on spaces.
         .p (or .paths): list of path objects
         
     Any values which require transformations are computed only once and
     cached."""
 
     def get_list(self):
         return self
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = ' '.join(self)
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         try:
             return self.__nlstr
         except AttributeError:
             self.__nlstr = '\n'.join(self)
             return self.__nlstr
 
     n = nlstr = property(get_nlstr)
     
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self if os.path.exists(p)]
             return self.__paths
     
     p = paths = property(get_paths)
 
-    def grep(self, pattern, prune = False):
+    def grep(self, pattern, prune = False, field = None):
         """ Return all strings matching 'pattern' (a regex or callable) 
         
         This is case-insensitive. If prune is true, return all items
         NOT matching the pattern.
         
+        If field is specified, the match must occur in the specified 
+        whitespace-separated field.
+               
         Examples::
         
             a.grep( lambda x: x.startswith('C') )
             a.grep('Cha.*log', prune=1)
+            a.grep('chm', field=-1)
         """
+        
+        def match_target(s):
+            if field is None:
+                return s
+            parts = s.split()
+            try:
+                tgt = parts[field]
+                return tgt
+            except IndexError:
+                return ""
+        
         if isinstance(pattern, basestring):
             pred = lambda x : re.search(pattern, x, re.IGNORECASE)
         else:
             pred = pattern
         if not prune:
-            return SList([el for el in self if pred(el)])
+            return SList([el for el in self if pred(match_target(el))])
         else:
-            return SList([el for el in self if not pred(el)])
+            return SList([el for el in self if not pred(match_target(el))])
     def fields(self, *fields):
         """ Collect whitespace-separated fields from string list
         
         Allows quick awk-like usage of string lists.
         
         Example data (in var a, created by 'a = !ls -l')::
             -rwxrwxrwx  1 ville None      18 Dec 14  2006 ChangeLog
             drwxrwxrwx+ 6 ville None       0 Oct 24 18:05 IPython        
         
         a.fields(0) is ['-rwxrwxrwx', 'drwxrwxrwx+']
         a.fields(1,0) is ['1 -rwxrwxrwx', '6 drwxrwxrwx+']
         (note the joining by space).
+        a.fields(-1) is ['ChangeLog', 'IPython']
         
         IndexErrors are ignored.
         
         Without args, fields() just split()'s the strings.
         """
         if len(fields) == 0:
             return [el.split() for el in self]
         
         res = SList()
         for el in [f.split() for f in self]:
             lineparts = []
 
             for fd in fields:
                 try:
                     lineparts.append(el[fd])
                 except IndexError:
                     pass
             if lineparts:
                 res.append(" ".join(lineparts))
             
         return res        
                 
             
             
         
 
 def print_slist(arg):
     """ Prettier (non-repr-like) and more informative printer for SList """
     print "SList (.p, .n, .l, .s, .grep(), .fields() available). Value:"
     nlprint(arg)
     
 print_slist = result_display.when_type(SList)(print_slist)
 
 
 
 #----------------------------------------------------------------------------
 def esc_quotes(strng):
     """Return the input string with single and double quotes escaped out"""
 
     return strng.replace('"','\\"').replace("'","\\'")
 
 #----------------------------------------------------------------------------
 def make_quoted_expr(s):
     """Return string s in appropriate quotes, using raw string if possible.
     
     Effectively this turns string: cd \ao\ao\
     to: r"cd \ao\ao\_"[:-1]
     
     Note the use of raw string and padding at the end to allow trailing backslash.
     
     """
     
     tail = ''
     tailpadding = ''
     raw  = ''
     if "\\" in s:
         raw = 'r'
         if s.endswith('\\'):
             tail = '[:-1]'
             tailpadding = '_'
     if '"' not in s:
         quote = '"'
     elif "'" not in s:
         quote = "'"
     elif '"""' not in s and not s.endswith('"'):
         quote = '"""'
     elif "'''" not in s and not s.endswith("'"):
         quote = "'''"
     else:
         # give up, backslash-escaped string will do
         return '"%s"' % esc_quotes(s)
     res = raw + quote + s + tailpadding + quote + tail
     return res
 
 
 #----------------------------------------------------------------------------
 def raw_input_multi(header='', ps1='==> ', ps2='..> ',terminate_str = '.'):
     """Take multiple lines of input.
 
     A list with each line of input as a separate element is returned when a
     termination string is entered (defaults to a single '.'). Input can also
     terminate via EOF (^D in Unix, ^Z-RET in Windows).
 
     Lines of input which end in \\ are joined into single entries (and a
     secondary continuation prompt is issued as long as the user terminates
     lines with \\). This allows entering very long strings which are still
     meant to be treated as single entities.
     """
 
     try:
         if header:
             header += '\n'
         lines = [raw_input(header + ps1)]
     except EOFError:
         return []
     terminate = [terminate_str]
     try:
         while lines[-1:] != terminate:
             new_line = raw_input(ps1)
             while new_line.endswith('\\'):
                 new_line = new_line[:-1] + raw_input(ps2)
             lines.append(new_line)
                 
         return lines[:-1]  # don't return the termination command
     except EOFError:
         print
         return lines
 
 #----------------------------------------------------------------------------
 def raw_input_ext(prompt='',  ps2='... '):
     """Similar to raw_input(), but accepts extended lines if input ends with \\."""
 
     line = raw_input(prompt)
     while line.endswith('\\'):
         line = line[:-1] + raw_input(ps2)
     return line
 
 #----------------------------------------------------------------------------
 def ask_yes_no(prompt,default=None):
     """Asks a question and returns a boolean (y/n) answer.
 
     If default is given (one of 'y','n'), it is used if the user input is
     empty. Otherwise the question is repeated until an answer is given.
 
     An EOF is treated as the default answer.  If there is no default, an
     exception is raised to prevent infinite loops.
 
     Valid answers are: y/yes/n/no (match is not case sensitive)."""
 
     answers = {'y':True,'n':False,'yes':True,'no':False}
     ans = None
     while ans not in answers.keys():
         try:
             ans = raw_input(prompt+' ').lower()
             if not ans:  # response was an empty string
                 ans = default
         except KeyboardInterrupt:
             pass
         except EOFError:
             if default in answers.keys():
                 ans = default
                 print
             else:
                 raise
             
     return answers[ans]
 
 #----------------------------------------------------------------------------
 def marquee(txt='',width=78,mark='*'):
     """Return the input string centered in a 'marquee'."""
     if not txt:
         return (mark*width)[:width]
     nmark = (width-len(txt)-2)/len(mark)/2
     if nmark < 0: nmark =0
     marks = mark*nmark
     return '%s %s %s' % (marks,txt,marks)
 
 #----------------------------------------------------------------------------
 class EvalDict:
     """
     Emulate a dict which evaluates its contents in the caller's frame.
 
     Usage:
     >>>number = 19
     >>>text = "python"
     >>>print "%(text.capitalize())s %(number/9.0).1f rules!" % EvalDict()
     """
 
     # This version is due to [e-mail] on c.l.py, and is basically a
     # modified (shorter) version of:
     # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66018 by
     # Skip Montanaro ([e-mail]).
 
     def __getitem__(self, name):
         frame = sys._getframe(1)
         return eval(name, frame.f_globals, frame.f_locals)
 
 EvalString = EvalDict  # for backwards compatibility
 #----------------------------------------------------------------------------
 def qw(words,flat=0,sep=None,maxsplit=-1):
     """Similar to Perl's qw() operator, but with some more options.
 
     qw(words,flat=0,sep=' ',maxsplit=-1) -> words.split(sep,maxsplit)
 
     words can also be a list itself, and with flat=1, the output will be
     recursively flattened. Examples:
 
     >>> qw('1 2')
     ['1', '2']
     >>> qw(['a b','1 2',['m n','p q']])
     [['a', 'b'], ['1', '2'], [['m', 'n'], ['p', 'q']]]
     >>> qw(['a b','1 2',['m n','p q']],flat=1)
     ['a', 'b', '1', '2', 'm', 'n', 'p', 'q']    """
 
     if type(words) in StringTypes:
         return [word.strip() for word in words.split(sep,maxsplit)
                 if word and not word.isspace() ]
     if flat:
         return flatten(map(qw,words,[1]*len(words)))
     return map(qw,words)
 
 #----------------------------------------------------------------------------
 def qwflat(words,sep=None,maxsplit=-1):
     """Calls qw(words) in flat mode. It's just a convenient shorthand."""
     return qw(words,1,sep,maxsplit)
 
 #----------------------------------------------------------------------------
 def qw_lol(indata):
     """qw_lol('a b') -> [['a','b']],
     otherwise it's just a call to qw().
 
     We need this to make sure the modules_some keys *always* end up as a
     list of lists."""
 
     if type(indata) in StringTypes:
         return [qw(indata)]
     else:
         return qw(indata)
 
 #-----------------------------------------------------------------------------
 def list_strings(arg):
     """Always return a list of strings, given a string or list of strings
     as input."""
 
     if type(arg) in StringTypes: return [arg]
     else: return arg
 
 #----------------------------------------------------------------------------
 def grep(pat,list,case=1):
     """Simple minded grep-like function.
     grep(pat,list) returns occurrences of pat in list, None on failure.
 
     It only does simple string matching, with no support for regexps. Use the
     option case=0 for case-insensitive matching."""
 
     # This is pretty crude. At least it should implement copying only references
     # to the original data in case it's big. Now it copies the data for output.
     out=[]
     if case:
         for term in list:
             if term.find(pat)>-1: out.append(term)
     else:
         lpat=pat.lower()
         for term in list:
             if term.lower().find(lpat)>-1: out.append(term)
 
     if len(out): return out
     else: return None
 
 #----------------------------------------------------------------------------
 def dgrep(pat,*opts):
     """Return grep() on dir()+dir(__builtins__).
 
     A very common use of grep() when working interactively."""
 
     return grep(pat,dir(__main__)+dir(__main__.__builtins__),*opts)
 
 #----------------------------------------------------------------------------
 def idgrep(pat):
     """Case-insensitive dgrep()"""
 
     return dgrep(pat,0)
 
 #----------------------------------------------------------------------------
 def igrep(pat,list):
     """Synonym for case-insensitive grep."""
 
     return grep(pat,list,case=0)
 
 #----------------------------------------------------------------------------
 def indent(str,nspaces=4,ntabs=0):
     """Indent a string a given number of spaces or tabstops.
     
     indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
     """
     if str is None:
         return
     ind = '\t'*ntabs+' '*nspaces
     outstr = '%s%s' % (ind,str.replace(os.linesep,os.linesep+ind))
     if outstr.endswith(os.linesep+ind):
         return outstr[:-len(ind)]
     else:
         return outstr
     
 #-----------------------------------------------------------------------------
 def native_line_ends(filename,backup=1):
     """Convert (in-place) a file to line-ends native to the current OS.
 
     If the optional backup argument is given as false, no backup of the
     original file is left.  """
 
     backup_suffixes = {'posix':'~','dos':'.bak','nt':'.bak','mac':'.bak'}
 
     bak_filename = filename + backup_suffixes[os.name]
 
     original = open(filename).read()
     shutil.copy2(filename,bak_filename)
     try:
         new = open(filename,'wb')
         new.write(os.linesep.join(original.splitlines()))
         new.write(os.linesep) # ALWAYS put an eol at the end of the file
         new.close()
     except:
         os.rename(bak_filename,filename)
     if not backup:
         try:
             os.remove(bak_filename)
         except:
             pass
     
 #----------------------------------------------------------------------------
 def get_pager_cmd(pager_cmd = None):
     """Return a pager command.
 
     Makes some attempts at finding an OS-correct one."""
     
     if os.name == 'posix':
         default_pager_cmd = 'less -r'  # -r for color control sequences
     elif os.name in ['nt','dos']:
         default_pager_cmd = 'type'
 
     if pager_cmd is None:
         try:
             pager_cmd = os.environ['PAGER']
         except:
             pager_cmd = default_pager_cmd
     return pager_cmd
 
 #-----------------------------------------------------------------------------
 def get_pager_start(pager,start):
     """Return the string for paging files with an offset.
 
     This is the '+N' argument which less and more (under Unix) accept.
     """
 
     if pager in ['less','more']:
         if start:
             start_string = '+' + str(start)
         else:
             start_string = ''
     else:
         start_string = ''
     return start_string
 
 #----------------------------------------------------------------------------
 # (X)emacs on W32 doesn't like to be bypassed with msvcrt.getch()
 if os.name == 'nt' and os.environ.get('TERM','dumb') != 'emacs':
     import msvcrt
     def page_more():
         """ Smart pausing between pages
 
         @return:    True if need print more lines, False if quit
         """
         Term.cout.write('---Return to continue, q to quit--- ')
         ans = msvcrt.getch()
         if ans in ("q", "Q"):
             result = False
         else:
             result = True
         Term.cout.write("\b"*37 + " "*37 + "\b"*37)
         return result
 else:
     def page_more():
         ans = raw_input('---Return to continue, q to quit--- ')
         if ans.lower().startswith('q'):
             return False
         else:
             return True
 
 esc_re = re.compile(r"(\x1b[^m]+m)")
 
 def page_dumb(strng,start=0,screen_lines=25):
     """Very dumb 'pager' in Python, for when nothing else works.
 
     Only moves forward, same interface as page(), except for pager_cmd and
     mode."""
 
     out_ln  = strng.splitlines()[start:]
     screens = chop(out_ln,screen_lines-1)
     if len(screens) == 1:
         print >>Term.cout, os.linesep.join(screens[0])
     else:
         last_escape = ""
         for scr in screens[0:-1]:
             hunk = os.linesep.join(scr)
             print >>Term.cout, last_escape + hunk
             if not page_more():
                 return
             esc_list = esc_re.findall(hunk)
             if len(esc_list) > 0:
                 last_escape = esc_list[-1]
         print >>Term.cout, last_escape + os.linesep.join(screens[-1])
 
 #----------------------------------------------------------------------------
 def page(strng,start=0,screen_lines=0,pager_cmd = None):
     """Print a string, piping through a pager after a certain length.
 
     The screen_lines parameter specifies the number of *usable* lines of your
     terminal screen (total lines minus lines you need to reserve to show other
     information).
 
     If you set screen_lines to a number <=0, page() will try to auto-determine
     your screen size and will only use up to (screen_size+screen_lines) for
     printing, paging after that. That is, if you want auto-detection but need
     to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for
     auto-detection without any lines reserved simply use screen_lines = 0.
 
     If a string won't fit in the allowed lines, it is sent through the
     specified pager command. If none given, look for PAGER in the environment,
     and ultimately default to less.
 
     If no system pager works, the string is sent through a 'dumb pager'
     written in python, very simplistic.
     """
     
 
     # first, try the hook
     ip = IPython.ipapi.get()
     if ip:
         try:
             ip.IP.hooks.show_in_pager(strng)
             return
         except IPython.ipapi.TryNext:
             pass
         
     # Ugly kludge, but calling curses.initscr() flat out crashes in emacs
     TERM = os.environ.get('TERM','dumb')
     if TERM in ['dumb','emacs'] and os.name != 'nt':
         print strng
         return
     # chop off the topmost part of the string we don't want to see
     str_lines = strng.split(os.linesep)[start:]
     str_toprint = os.linesep.join(str_lines)
     num_newlines = len(str_lines)
     len_str = len(str_toprint)
     
     # Dumb heuristics to guesstimate number of on-screen lines the string
     # takes.  Very basic, but good enough for docstrings in reasonable
     # terminals. If someone later feels like refining it, it's not hard.
     numlines = max(num_newlines,int(len_str/80)+1)
 
     if os.name == "nt":
         screen_lines_def = get_console_size(defaulty=25)[1]
     else:
         screen_lines_def = 25 # default value if we can't auto-determine
 
     # auto-determine screen size
     if screen_lines <= 0:
         if TERM=='xterm':
-            try:
-                import curses
-                if hasattr(curses,'initscr'):
-                    use_curses = 1
-                else:
-                    use_curses = 0
-            except ImportError:
-                use_curses = 0
+            use_curses = USE_CURSES
         else:
             # curses causes problems on many terminals other than xterm.
-            use_curses = 0
+            use_curses = False
         if use_curses:
-                scr = curses.initscr()
-                screen_lines_real,screen_cols = scr.getmaxyx()
-                curses.endwin()
-                screen_lines += screen_lines_real
-                #print '***Screen size:',screen_lines_real,'lines x',\
-                #screen_cols,'columns.' # dbg
+            # There is a bug in curses, where *sometimes* it fails to properly
+            # initialize, and then after the endwin() call is made, the
+            # terminal is left in an unusable state.  Rather than trying to
+            # check everytime for this (by requesting and comparing termios
+            # flags each time), we just save the initial terminal state and
+            # unconditionally reset it every time.  It's cheaper than making
+            # the checks.
+            term_flags = termios.tcgetattr(sys.stdout)
+            scr = curses.initscr()
+            screen_lines_real,screen_cols = scr.getmaxyx()
+            curses.endwin()
+            # Restore terminal state in case endwin() didn't.
+            termios.tcsetattr(sys.stdout,termios.TCSANOW,term_flags)
+            # Now we have what we needed: the screen size in rows/columns
+            screen_lines += screen_lines_real
+            #print '***Screen size:',screen_lines_real,'lines x',\
+            #screen_cols,'columns.' # dbg
         else:
-                screen_lines += screen_lines_def
+            screen_lines += screen_lines_def
 
     #print 'numlines',numlines,'screenlines',screen_lines  # dbg
     if numlines <= screen_lines :
         #print '*** normal print'  # dbg
         print >>Term.cout, str_toprint
     else:
         # Try to open pager and default to internal one if that fails.
         # All failure modes are tagged as 'retval=1', to match the return
         # value of a failed system command.  If any intermediate attempt
         # sets retval to 1, at the end we resort to our own page_dumb() pager.
         pager_cmd = get_pager_cmd(pager_cmd)
         pager_cmd += ' ' + get_pager_start(pager_cmd,start)
         if os.name == 'nt':
             if pager_cmd.startswith('type'):
                 # The default WinXP 'type' command is failing on complex strings.
                 retval = 1
             else:
                 tmpname = tempfile.mktemp('.txt')
                 tmpfile = file(tmpname,'wt')
                 tmpfile.write(strng)
                 tmpfile.close()
                 cmd = "%s < %s" % (pager_cmd,tmpname)
                 if os.system(cmd):
                   retval = 1
                 else:
                   retval = None
                 os.remove(tmpname)
         else:
             try:
                 retval = None
                 # if I use popen4, things hang. No idea why.
                 #pager,shell_out = os.popen4(pager_cmd)
                 pager = os.popen(pager_cmd,'w')
                 pager.write(strng)
                 pager.close()
                 retval = pager.close()  # success returns None
             except IOError,msg:  # broken pipe when user quits
                 if msg.args == (32,'Broken pipe'):
                     retval = None
                 else:
                     retval = 1
             except OSError:
                 # Other strange problems, sometimes seen in Win2k/cygwin
                 retval = 1
         if retval is not None:
             page_dumb(strng,screen_lines=screen_lines)
 
 #----------------------------------------------------------------------------
 def page_file(fname,start = 0, pager_cmd = None):
     """Page a file, using an optional pager command and starting line.
     """
 
     pager_cmd = get_pager_cmd(pager_cmd)
     pager_cmd += ' ' + get_pager_start(pager_cmd,start)
 
     try:
         if os.environ['TERM'] in ['emacs','dumb']:
             raise EnvironmentError
         xsys(pager_cmd + ' ' + fname)
     except:
         try:
             if start > 0:
                 start -= 1
             page(open(fname).read(),start)
         except:
             print 'Unable to show file',`fname`
 
 
 #----------------------------------------------------------------------------
 def snip_print(str,width = 75,print_full = 0,header = ''):
     """Print a string snipping the midsection to fit in width.
 
     print_full: mode control:
       - 0: only snip long strings
       - 1: send to page() directly.
       - 2: snip long strings and ask for full length viewing with page()
     Return 1 if snipping was necessary, 0 otherwise."""
 
     if print_full == 1:
         page(header+str)
         return 0
 
     print header,
     if len(str) < width:
         print str
         snip = 0
     else:
         whalf = int((width -5)/2)
         print str[:whalf] + ' <...> ' + str[-whalf:]
         snip = 1
     if snip and print_full == 2:
         if raw_input(header+' Snipped. View (y/n)? [N]').lower() == 'y':
             page(str)
     return snip
 
 #****************************************************************************
 # lists, dicts and structures
 
 def belong(candidates,checklist):
     """Check whether a list of items appear in a given list of options.
 
     Returns a list of 1 and 0, one for each candidate given."""
 
     return [x in checklist for x in candidates]
 
 #----------------------------------------------------------------------------
 def uniq_stable(elems):
     """uniq_stable(elems) -> list
 
     Return from an iterable, a list of all the unique elements in the input,
     but maintaining the order in which they first appear.
 
     A naive solution to this problem which just makes a dictionary with the
     elements as keys fails to respect the stability condition, since
     dictionaries are unsorted by nature.
 
     Note: All elements in the input must be valid dictionary keys for this
     routine to work, as it internally uses a dictionary for efficiency
     reasons."""
     
     unique = []
     unique_dict = {}
     for nn in elems:
         if nn not in unique_dict:
             unique.append(nn)
             unique_dict[nn] = None
     return unique
 
 #----------------------------------------------------------------------------
 class NLprinter:
     """Print an arbitrarily nested list, indicating index numbers.
 
     An instance of this class called nlprint is available and callable as a
     function.
     
     nlprint(list,indent=' ',sep=': ') -> prints indenting each level by 'indent'
     and using 'sep' to separate the index from the value. """
 
     def __init__(self):
         self.depth = 0
     
     def __call__(self,lst,pos='',**kw):
         """Prints the nested list numbering levels."""
         kw.setdefault('indent',' ')
         kw.setdefault('sep',': ')
         kw.setdefault('start',0)
         kw.setdefault('stop',len(lst))
         # we need to remove start and stop from kw so they don't propagate
         # into a recursive call for a nested list.
         start = kw['start']; del kw['start']
         stop = kw['stop']; del kw['stop']
         if self.depth == 0 and 'header' in kw.keys():
             print kw['header']
             
         for idx in range(start,stop):
             elem = lst[idx]
             if type(elem)==type([]):
                 self.depth += 1
                 self.__call__(elem,itpl('$pos$idx,'),**kw)
                 self.depth -= 1
             else:
                 printpl(kw['indent']*self.depth+'$pos$idx$kw["sep"]$elem')
 
 nlprint = NLprinter()
 #----------------------------------------------------------------------------
 def all_belong(candidates,checklist):
     """Check whether a list of items ALL appear in a given list of options.
 
     Returns a single 1 or 0 value."""
 
     return 1-(0 in [x in checklist for x in candidates])
 
 #----------------------------------------------------------------------------
 def sort_compare(lst1,lst2,inplace = 1):
     """Sort and compare two lists.
 
     By default it does it in place, thus modifying the lists. Use inplace = 0
     to avoid that (at the cost of temporary copy creation)."""
     if not inplace:
         lst1 = lst1[:]
         lst2 = lst2[:]
     lst1.sort(); lst2.sort()
     return lst1 == lst2
 
 #----------------------------------------------------------------------------
 def mkdict(**kwargs):
     """Return a dict from a keyword list.
 
     It's just syntactic sugar for making ditcionary creation more convenient:
     # the standard way
     >>>data = { 'red' : 1, 'green' : 2, 'blue' : 3 }
     # a cleaner way
     >>>data = dict(red=1, green=2, blue=3)
 
     If you need more than this, look at the Struct() class."""
 
     return kwargs
 
 #----------------------------------------------------------------------------
 def list2dict(lst):
     """Takes a list of (key,value) pairs and turns it into a dict."""
 
     dic = {}
     for k,v in lst: dic[k] = v
     return dic
 
 #----------------------------------------------------------------------------
 def list2dict2(lst,default=''):
     """Takes a list and turns it into a dict.
     Much slower than list2dict, but more versatile. This version can take
     lists with sublists of arbitrary length (including sclars)."""
 
     dic = {}
     for elem in lst:
         if type(elem) in (types.ListType,types.TupleType):
             size = len(elem)
             if  size == 0:
                 pass
             elif size == 1:
                 dic[elem] = default
             else:
                 k,v = elem[0], elem[1:]
                 if len(v) == 1: v = v[0]
                 dic[k] = v
         else:
             dic[elem] = default
     return dic
 
 #----------------------------------------------------------------------------
 def flatten(seq):
     """Flatten a list of lists (NOT recursive, only works for 2d lists)."""
 
     return [x for subseq in seq for x in subseq]
 
 #----------------------------------------------------------------------------
 def get_slice(seq,start=0,stop=None,step=1):
     """Get a slice of a sequence with variable step. Specify start,stop,step."""
     if stop == None:
         stop = len(seq)
     item = lambda i: seq[i]
     return map(item,xrange(start,stop,step))
 
 #----------------------------------------------------------------------------
 def chop(seq,size):
     """Chop a sequence into chunks of the given size."""
     chunk = lambda i: seq[i:i+size]
     return map(chunk,xrange(0,len(seq),size))
 
 #----------------------------------------------------------------------------
 # with is a keyword as of python 2.5, so this function is renamed to withobj
 # from its old 'with' name.
 def with_obj(object, **args):
     """Set multiple attributes for an object, similar to Pascal's with.
 
     Example:
     with_obj(jim,
              born = 1960,
              haircolour = 'Brown',
              eyecolour = 'Green')
 
     Credit: Greg Ewing, in
     http://mail.python.org/pipermail/python-list/2001-May/040703.html.
 
     NOTE: up until IPython 0.7.2, this was called simply 'with', but 'with'
     has become a keyword for Python 2.5, so we had to rename it."""
 
     object.__dict__.update(args)
 
 #----------------------------------------------------------------------------
 def setattr_list(obj,alist,nspace = None):
     """Set a list of attributes for an object taken from a namespace.
 
     setattr_list(obj,alist,nspace) -> sets in obj all the attributes listed in
     alist with their values taken from nspace, which must be a dict (something
     like locals() will often do) If nspace isn't given, locals() of the
     *caller* is used, so in most cases you can omit it.
 
     Note that alist can be given as a string, which will be automatically
     split into a list on whitespace. If given as a list, it must be a list of
     *strings* (the variable names themselves), not of variables."""
 
     # this grabs the local variables from the *previous* call frame -- that is
     # the locals from the function that called setattr_list().
     # - snipped from weave.inline()
     if nspace is None:
         call_frame = sys._getframe().f_back
         nspace = call_frame.f_locals
 
     if type(alist) in StringTypes:
         alist = alist.split()
     for attr in alist:
         val = eval(attr,nspace)
         setattr(obj,attr,val)
 
 #----------------------------------------------------------------------------
 def getattr_list(obj,alist,*args):
     """getattr_list(obj,alist[, default]) -> attribute list.
 
     Get a list of named attributes for an object. When a default argument is
     given, it is returned when the attribute doesn't exist; without it, an
     exception is raised in that case.
 
     Note that alist can be given as a string, which will be automatically
     split into a list on whitespace. If given as a list, it must be a list of
     *strings* (the variable names themselves), not of variables."""
 
     if type(alist) in StringTypes:
         alist = alist.split()
     if args:
         if len(args)==1:
             default = args[0]
             return map(lambda attr: getattr(obj,attr,default),alist)
         else:
             raise ValueError,'getattr_list() takes only one optional argument'
     else:
         return map(lambda attr: getattr(obj,attr),alist)
     
 #----------------------------------------------------------------------------
 def map_method(method,object_list,*argseq,**kw):
     """map_method(method,object_list,*args,**kw) -> list
 
     Return a list of the results of applying the methods to the items of the
     argument sequence(s).  If more than one sequence is given, the method is
     called with an argument list consisting of the corresponding item of each
     sequence. All sequences must be of the same length.
 
     Keyword arguments are passed verbatim to all objects called.
 
     This is Python code, so it's not nearly as fast as the builtin map()."""
 
     out_list = []
     idx = 0
     for object in object_list:
         try:
             handler = getattr(object, method)
         except AttributeError:
             out_list.append(None)
         else:
             if argseq:
                 args = map(lambda lst:lst[idx],argseq)
                 #print 'ob',object,'hand',handler,'ar',args # dbg
                 out_list.append(handler(args,**kw))
             else:
                 out_list.append(handler(**kw))
         idx += 1
     return out_list
 
 #----------------------------------------------------------------------------
 def get_class_members(cls):
     ret = dir(cls)
     if hasattr(cls,'__bases__'):
         for base in cls.__bases__:
             ret.extend(get_class_members(base))
     return ret
 
 #----------------------------------------------------------------------------
 def dir2(obj):
     """dir2(obj) -> list of strings
 
     Extended version of the Python builtin dir(), which does a few extra
     checks, and supports common objects with unusual internals that confuse
     dir(), such as Traits and PyCrust.
 
     This version is guaranteed to return only a list of true strings, whereas
     dir() returns anything that objects inject into themselves, even if they
     are later not really valid for attribute access (many extension libraries
     have such bugs).
     """
     
     # Start building the attribute list via dir(), and then complete it
     # with a few extra special-purpose calls.
     words = dir(obj)
 
     if hasattr(obj,'__class__'):
         words.append('__class__')
         words.extend(get_class_members(obj.__class__))
     #if '__base__' in words: 1/0
 
     # Some libraries (such as traits) may introduce duplicates, we want to
     # track and clean this up if it happens
     may_have_dupes = False
 
     # this is the 'dir' function for objects with Enthought's traits
     if hasattr(obj, 'trait_names'):
         try:
             words.extend(obj.trait_names())
             may_have_dupes = True
         except TypeError:
             # This will happen if `obj` is a class and not an instance.
             pass
 
     # Support for PyCrust-style _getAttributeNames magic method.
     if hasattr(obj, '_getAttributeNames'):
         try:
             words.extend(obj._getAttributeNames())
             may_have_dupes = True
         except TypeError:
             # `obj` is a class and not an instance.  Ignore
             # this error.
             pass
 
     if may_have_dupes:
         # eliminate possible duplicates, as some traits may also
         # appear as normal attributes in the dir() call.
         words = list(set(words))
         words.sort()
 
     # filter out non-string attributes which may be stuffed by dir() calls
     # and poor coding in third-party modules
     return [w for w in words if isinstance(w, basestring)]
 
 #----------------------------------------------------------------------------
 def import_fail_info(mod_name,fns=None):
     """Inform load failure for a module."""
 
     if fns == None:
         warn("Loading of %s failed.\n" % (mod_name,))
     else:
         warn("Loading of %s from %s failed.\n" % (fns,mod_name))
 
 #----------------------------------------------------------------------------
 # Proposed popitem() extension, written as a method
 
 
 class NotGiven: pass
 
 def popkey(dct,key,default=NotGiven):
     """Return dct[key] and delete dct[key].
 
     If default is given, return it if dct[key] doesn't exist, otherwise raise
     KeyError.  """
 
     try:
         val = dct[key]
     except KeyError:
         if default is NotGiven:
             raise
         else:
             return default
     else:
         del dct[key]
         return val
 
 def wrap_deprecated(func, suggest = '<nothing>'):
     def newFunc(*args, **kwargs):
         warnings.warn("Call to deprecated function %s, use %s instead" % 
                       ( func.__name__, suggest),
                       category=DeprecationWarning,
                       stacklevel = 2)
         return func(*args, **kwargs)
     return newFunc
     
 #*************************** end of file <genutils.py> **********************