Upstream merges.

IPython/Extensions/ipy_completers.py

@@ -1,386 +1,391 b''
 
 """ Implementations for various useful completers
 
 See Extensions/ipy_stock_completers.py on examples of how to enable a completer,
 but the basic idea is to do:
 
 ip.set_hook('complete_command', svn_completer, str_key = 'svn')
 
 """
 import IPython.ipapi
 import glob,os,shlex,sys
 import inspect
 from time import time
 from zipimport import zipimporter
 ip = IPython.ipapi.get()
 
 try:
     set
 except:
     from sets import Set as set
 
 TIMEOUT_STORAGE = 3 #Time in seconds after which the rootmodules will be stored
 TIMEOUT_GIVEUP = 20 #Time in seconds after which we give up
 
 def quick_completer(cmd, completions):
     """ Easily create a trivial completer for a command.
 
     Takes either a list of completions, or all completions in string
     (that will be split on whitespace)
     
     Example::
     
         [d:\ipython]|1> import ipy_completers                                       
         [d:\ipython]|2> ipy_completers.quick_completer('foo', ['bar','baz'])        
         [d:\ipython]|3> foo b<TAB>
         bar baz                                                                     
         [d:\ipython]|3> foo ba
     """
     if isinstance(completions, basestring):
         
         completions = completions.split()
     def do_complete(self,event):
         return completions
     
     ip.set_hook('complete_command',do_complete, str_key = cmd)
     
 def getRootModules():
     """
     Returns a list containing the names of all the modules available in the
     folders of the pythonpath.
     """
     modules = []
     if ip.db.has_key('rootmodules'):
         return ip.db['rootmodules']
     t = time()
     store = False
     for path in sys.path:
         modules += moduleList(path)        
         if time() - t >= TIMEOUT_STORAGE and not store:
             store = True
             print "\nCaching the list of root modules, please wait!" 
             print "(This will only be done once - type '%rehashx' to " + \
             "reset cache!)"
             print
         if time() - t > TIMEOUT_GIVEUP:
             print "This is taking too long, we give up."
             print
             ip.db['rootmodules'] = []
             return []
     
     modules += sys.builtin_module_names
       
     modules = list(set(modules))
     if '__init__' in modules:
         modules.remove('__init__')
     modules = list(set(modules))
     if store:
         ip.db['rootmodules'] = modules
     return modules
 
 def moduleList(path):
     """
     Return the list containing the names of the modules available in the given
     folder.
     """
 
     if os.path.isdir(path):
         folder_list = os.listdir(path)
     elif path.endswith('.egg'):
         try:
             folder_list = [f for f in zipimporter(path)._files]
         except:
             folder_list = []
     else:
         folder_list = []
     #folder_list = glob.glob(os.path.join(path,'*'))
     folder_list = [p for p in folder_list  \
        if os.path.exists(os.path.join(path, p,'__init__.py'))\
            or p[-3:] in ('.py','.so')\
            or p[-4:] in ('.pyc','.pyo','.pyd')]
 
     folder_list = [os.path.basename(p).split('.')[0] for p in folder_list]
     return folder_list
 
 def moduleCompletion(line):
     """
     Returns a list containing the completion possibilities for an import line.
     The line looks like this :
     'import xml.d'
     'from xml.dom import'
     """
     def tryImport(mod, only_modules=False):
         def isImportable(module, attr):
             if only_modules:
                 return inspect.ismodule(getattr(module, attr))
             else:
                 return not(attr[:2] == '__' and attr[-2:] == '__')
         try:
             m = __import__(mod)
         except:
             return []
         mods = mod.split('.')
         for module in mods[1:]:
             m = getattr(m,module)
         if (not hasattr(m, '__file__')) or (not only_modules) or\
            (hasattr(m, '__file__') and '__init__' in m.__file__):
             completion_list = [attr for attr in dir(m) if isImportable(m, attr)]
         completion_list.extend(getattr(m,'__all__',[]))
         if hasattr(m, '__file__') and '__init__' in m.__file__:
             completion_list.extend(moduleList(os.path.dirname(m.__file__)))
         completion_list = list(set(completion_list))
         if '__init__' in completion_list:
             completion_list.remove('__init__')
         return completion_list
 
     words = line.split(' ')
     if len(words) == 3 and words[0] == 'from':
         return ['import ']
     if len(words) < 3 and (words[0] in ['import','from']) :
         if len(words) == 1:
             return getRootModules()
         mod = words[1].split('.')
         if len(mod) < 2:
             return getRootModules()
         completion_list = tryImport('.'.join(mod[:-1]), True)
         completion_list = ['.'.join(mod[:-1] + [el]) for el in completion_list]
         return completion_list
     if len(words) >= 3 and words[0] == 'from':
         mod = words[1]
         return tryImport(mod)
 
 def vcs_completer(commands, event):
     """ utility to make writing typical version control app completers easier
 
     VCS command line apps typically have the format:
 
     [sudo ]PROGNAME [help] [command] file file...
 
     """
 
 
     cmd_param = event.line.split()
     if event.line.endswith(' '):
         cmd_param.append('')
 
     if cmd_param[0] == 'sudo':
         cmd_param = cmd_param[1:]
 
     if len(cmd_param) == 2 or 'help' in cmd_param:
         return commands.split()
 
     return ip.IP.Completer.file_matches(event.symbol)
 
 
 pkg_cache = None
 
 def module_completer(self,event):
     """ Give completions after user has typed 'import ...' or 'from ...'"""
 
     # This works in all versions of python.  While 2.5 has
     # pkgutil.walk_packages(), that particular routine is fairly dangerous,
     # since it imports *EVERYTHING* on sys.path.  That is: a) very slow b) full
     # of possibly problematic side effects.
     # This search the folders in the sys.path for available modules.
 
     return moduleCompletion(event.line)
 
 
 svn_commands = """\
 add blame praise annotate ann cat checkout co cleanup commit ci copy
 cp delete del remove rm diff di export help ? h import info list ls
 lock log merge mkdir move mv rename ren propdel pdel pd propedit pedit
 pe propget pget pg proplist plist pl propset pset ps resolved revert
 status stat st switch sw unlock update
 """
 
 def svn_completer(self,event):
     return vcs_completer(svn_commands, event)
 
 
 hg_commands = """
 add addremove annotate archive backout branch branches bundle cat
 clone commit copy diff export grep heads help identify import incoming
 init locate log manifest merge outgoing parents paths pull push
 qapplied qclone qcommit qdelete qdiff qfold qguard qheader qimport
 qinit qnew qnext qpop qprev qpush qrefresh qrename qrestore qsave
 qselect qseries qtop qunapplied recover remove rename revert rollback
 root serve showconfig status strip tag tags tip unbundle update verify
 version
 """
 
 def hg_completer(self,event):
     """ Completer for mercurial commands """
 
     return vcs_completer(hg_commands, event)
 
 
 
 __bzr_commands = None
 
 def bzr_commands():
     global __bzr_commands
     if __bzr_commands is not None:
         return __bzr_commands
     out = os.popen('bzr help commands')
     __bzr_commands = [l.split()[0] for l in out]
     return __bzr_commands                
 
 def bzr_completer(self,event):
     """ Completer for bazaar commands """
     cmd_param = event.line.split()
     if event.line.endswith(' '):
         cmd_param.append('')
 
     if len(cmd_param) > 2:
         cmd = cmd_param[1]
         param = cmd_param[-1]
         output_file = (param == '--output=')
         if cmd == 'help':
             return bzr_commands()
         elif cmd in ['bundle-revisions','conflicts',
                      'deleted','nick','register-branch',
                      'serve','unbind','upgrade','version',
                      'whoami'] and not output_file:
             return []
         else:
             # the rest are probably file names
             return ip.IP.Completer.file_matches(event.symbol)
 
     return bzr_commands()
 
 
 def shlex_split(x):
     """Helper function to split lines into segments."""
     #shlex.split raise exception if syntax error in sh syntax
     #for example if no closing " is found. This function keeps dropping
     #the last character of the line until shlex.split does not raise
     #exception. Adds end of the line to the result of shlex.split
     #example: %run "c:/python  -> ['%run','"c:/python']
     endofline=[]
     while x!="":
         try:
             comps=shlex.split(x)
             if len(endofline)>=1:
                 comps.append("".join(endofline))
             return comps
         except ValueError:
             endofline=[x[-1:]]+endofline
             x=x[:-1]
     return ["".join(endofline)]
 
 def runlistpy(self, event):
     comps = shlex_split(event.line)
     relpath = (len(comps) > 1 and comps[-1] or '').strip("'\"")
 
     #print "\nev=",event  # dbg
     #print "rp=",relpath  # dbg
     #print 'comps=',comps  # dbg
 
     lglob = glob.glob
     isdir = os.path.isdir
     if relpath.startswith('~'):
         relpath = os.path.expanduser(relpath)
     dirs = [f.replace('\\','/') + "/" for f in lglob(relpath+'*')
             if isdir(f)]
 
     # Find if the user has already typed the first filename, after which we
     # should complete on all files, since after the first one other files may
     # be arguments to the input script.
     #filter(
     if filter(lambda f: f.endswith('.py') or f.endswith('.ipy') or
               f.endswith('.pyw'),comps):
         pys =  [f.replace('\\','/') for f in lglob('*')]
     else:
         pys =  [f.replace('\\','/')
                 for f in lglob(relpath+'*.py') + lglob(relpath+'*.ipy') +
                 lglob(relpath + '*.pyw')]
     return dirs + pys
 
 
+greedy_cd_completer = False
+
 def cd_completer(self, event):
     relpath = event.symbol
     #print event # dbg
     if '-b' in event.line:
         # return only bookmark completions
         bkms = self.db.get('bookmarks',{})
         return bkms.keys()
 
     
     if event.symbol == '-':
         width_dh = str(len(str(len(ip.user_ns['_dh']) + 1)))
         # jump in directory history by number
         fmt = '-%0' + width_dh +'d [%s]'
         ents = [ fmt % (i,s) for i,s in enumerate(ip.user_ns['_dh'])]
         if len(ents) > 1:
             return ents
         return []
 
     if relpath.startswith('~'):
         relpath = os.path.expanduser(relpath).replace('\\','/')
     found = []
     for d in [f.replace('\\','/') + '/' for f in glob.glob(relpath+'*')
               if os.path.isdir(f)]:
         if ' ' in d:
             # we don't want to deal with any of that, complex code
             # for this is elsewhere
             raise IPython.ipapi.TryNext
         found.append( d )
 
     if not found:
         if os.path.isdir(relpath):
             return [relpath]
         raise IPython.ipapi.TryNext
 
 
     def single_dir_expand(matches):
         "Recursively expand match lists containing a single dir."
         
         if len(matches) == 1 and os.path.isdir(matches[0]):
             # Takes care of links to directories also.  Use '/'
             # explicitly, even under Windows, so that name completions
             # don't end up escaped.
             d = matches[0]
             if d[-1] in ['/','\\']:
                 d = d[:-1]
 
             subdirs = [p for p in os.listdir(d) if os.path.isdir( d + '/' + p) and not p.startswith('.')]
             if subdirs:
                 matches = [ (d + '/' + p) for p in subdirs ]
                 return single_dir_expand(matches)
             else:
                 return matches
         else:
             return matches
 
-    return single_dir_expand(found)
+    if greedy_cd_completer:
+        return single_dir_expand(found)
+    else:
+        return found
 
 def apt_get_packages(prefix):
     out = os.popen('apt-cache pkgnames')
     for p in out:
         if p.startswith(prefix):
             yield p.rstrip()
     
     
 apt_commands = """\
 update upgrade install remove purge source build-dep dist-upgrade
 dselect-upgrade clean autoclean check"""
 
 def apt_completer(self, event):
     """ Completer for apt-get (uses apt-cache internally)
 
     """
 
 
     cmd_param = event.line.split()
     if event.line.endswith(' '):
         cmd_param.append('')
 
     if cmd_param[0] == 'sudo':
         cmd_param = cmd_param[1:]
 
     if len(cmd_param) == 2 or 'help' in cmd_param:
         return apt_commands.split()
 
     return list(apt_get_packages(event.symbol))
 

IPython/Extensions/ipy_profile_sh.py

@@ -1,251 +1,258 b''
 """Shell mode for IPython.
 
 Start ipython in shell mode by invoking "ipython -p sh"
 
 (the old version, "ipython -p pysh" still works but this is the more "modern" 
 shell mode and is recommended for users who don't care about pysh-mode
 compatibility)
 """
 
 from IPython import ipapi
 import os,textwrap
 
 # The import below effectively obsoletes your old-style ipythonrc[.ini],
 # so consider yourself warned!
 
 import ipy_defaults
 
 def main():    
     ip = ipapi.get()
     o = ip.options
     # autocall to "full" mode (smart mode is default, I like full mode)
     
     o.autocall = 2
     
     # Jason Orendorff's path class is handy to have in user namespace
     # if you are doing shell-like stuff
     try:
         ip.ex("from IPython.external.path import path" )
     except ImportError:
         pass
     
     # beefed up %env is handy in shell mode
     import envpersist
     
     # To see where mycmd resides (in path/aliases), do %which mycmd 
     import ipy_which
     
     # tab completers for hg, svn, ...
     import ipy_app_completers
     
     # To make executables foo and bar in mybin usable without PATH change, do:
     # %rehashdir c:/mybin
     # %store foo
     # %store bar
     import ipy_rehashdir
 
     # does not work without subprocess module!
     #import ipy_signals
     
     ip.ex('import os')
     ip.ex("def up(): os.chdir('..')")
     ip.user_ns['LA'] = LastArgFinder() 
     # Nice prompt
     
     o.prompt_in1= r'\C_LightBlue[\C_LightCyan\Y2\C_LightBlue]\C_Green|\#> '
     o.prompt_in2= r'\C_Green|\C_LightGreen\D\C_Green> '
     o.prompt_out= '<\#> '
     
     from IPython import Release
 
     import sys
     # Non-chatty banner
     o.banner = "IPython %s   [on Py %s]\n" % (Release.version,sys.version.split(None,1)[0])
     
     
     ip.IP.default_option('cd','-q')
     ip.IP.default_option('macro', '-r')
     # If you only rarely want to execute the things you %edit...  
     #ip.IP.default_option('edit','-x')
     
 
     o.prompts_pad_left="1"
     # Remove all blank lines in between prompts, like a normal shell.
     o.separate_in="0"
     o.separate_out="0"
     o.separate_out2="0"
     
     # now alias all syscommands
     
     db = ip.db
     
     syscmds = db.get("syscmdlist",[] )
     if not syscmds:
         print textwrap.dedent("""
         System command list not initialized, probably the first run...
         running %rehashx to refresh the command list. Run %rehashx
         again to refresh command list (after installing new software etc.)
         """)
         ip.magic('rehashx')
         syscmds = db.get("syscmdlist")
     
     # lowcase aliases on win32 only
     if os.name == 'posix':
         mapper = lambda s:s
     else:
         def mapper(s): return s.lower()
         
     for cmd in syscmds:
         # print "sys",cmd #dbg
         noext, ext = os.path.splitext(cmd)
         key = mapper(noext)
         if key not in ip.IP.alias_table:
             ip.defalias(key, cmd)
 
     # mglob combines 'find', recursion, exclusion... '%mglob?' to learn more
     ip.load("IPython.external.mglob")    
 
     # win32 is crippled w/o cygwin, try to help it a little bit
     if sys.platform == 'win32':
         if 'cygwin' in os.environ['PATH'].lower():          
             # use the colors of cygwin ls (recommended)
             ip.defalias('d', 'ls -F --color=auto')
         else:
             # get icp, imv, imkdir, igrep, irm,...
             ip.load('ipy_fsops')
             
             # and the next best thing to real 'ls -F'
             ip.defalias('d','dir /w /og /on')
     
+    ip.set_hook('input_prefilter', dotslash_prefilter_f)
     extend_shell_behavior(ip)
 
 class LastArgFinder:
     """ Allow $LA to work as "last argument of previous command", like $! in bash
     
     To call this in normal IPython code, do LA()
     """
     def __call__(self, hist_idx = None):
         ip = ipapi.get()
         if hist_idx is None:
             return str(self)
         return ip.IP.input_hist_raw[hist_idx].strip().split()[-1]
     def __str__(self):
         ip = ipapi.get()        
         for cmd in reversed(ip.IP.input_hist_raw):
             parts = cmd.strip().split()
             if len(parts) < 2 or parts[-1] in ['$LA', 'LA()']:
                 continue
             return parts[-1]
         return ""
 
-
-
-
+def dotslash_prefilter_f(self,line):
+    """ ./foo now runs foo as system command 
+    
+    Removes the need for doing !./foo
+    """
+    import IPython.genutils
+    if line.startswith("./"):
+        return "_ip.system(" + IPython.genutils.make_quoted_expr(line)+")"
+    raise ipapi.TryNext
 
 # XXX You do not need to understand the next function!
 # This should probably be moved out of profile
 
 def extend_shell_behavior(ip):
 
     # Instead of making signature a global variable tie it to IPSHELL.
     # In future if it is required to distinguish between different
     # shells we can assign a signature per shell basis
     ip.IP.__sig__ = 0xa005
     # mark the IPSHELL with this signature
     ip.IP.user_ns['__builtins__'].__dict__['__sig__'] = ip.IP.__sig__
 
     from IPython.Itpl import ItplNS
     from IPython.genutils import shell
     # utility to expand user variables via Itpl
     # xxx do something sensible with depth?
     ip.IP.var_expand = lambda cmd, lvars=None, depth=2: \
         str(ItplNS(cmd, ip.IP.user_ns, get_locals()))
 
     def get_locals():
         """ Substituting a variable through Itpl deep inside the IPSHELL stack
             requires the knowledge of all the variables in scope upto the last
             IPSHELL frame. This routine simply merges all the local variables
             on the IPSHELL stack without worrying about their scope rules
         """
         import sys
         # note lambda expression constitues a function call
         # hence fno should be incremented by one
         getsig = lambda fno: sys._getframe(fno+1).f_globals \
                              ['__builtins__'].__dict__['__sig__']
         getlvars = lambda fno: sys._getframe(fno+1).f_locals
         # trackback until we enter the IPSHELL
         frame_no = 1
         sig = ip.IP.__sig__
         fsig = ~sig
         while fsig != sig :
             try:
                 fsig = getsig(frame_no)
             except (AttributeError, KeyError):
                 frame_no += 1
             except ValueError:
                 # stack is depleted
                 # call did not originate from IPSHELL
                 return {}
         first_frame = frame_no
         # walk further back until we exit from IPSHELL or deplete stack
         try:
             while(sig == getsig(frame_no+1)):
                 frame_no += 1
         except (AttributeError, KeyError, ValueError):
             pass
         # merge the locals from top down hence overriding
         # any re-definitions of variables, functions etc.
         lvars = {}
         for fno in range(frame_no, first_frame-1, -1):
             lvars.update(getlvars(fno))
         #print '\n'*5, first_frame, frame_no, '\n', lvars, '\n'*5 #dbg
         return lvars
 
     def _runlines(lines):
         """Run a string of one or more lines of source.
 
         This method is capable of running a string containing multiple source
         lines, as if they had been entered at the IPython prompt.  Since it
         exposes IPython's processing machinery, the given strings can contain
         magic calls (%magic), special shell access (!cmd), etc."""
 
         # We must start with a clean buffer, in case this is run from an
         # interactive IPython session (via a magic, for example).
         ip.IP.resetbuffer()
         lines = lines.split('\n')
         more = 0
         command = ''
         for line in lines:
             # skip blank lines so we don't mess up the prompt counter, but do
             # NOT skip even a blank line if we are in a code block (more is
             # true)
             # if command is not empty trim the line
             if command != '' :
                 line = line.strip()
             # add the broken line to the command
             if line and line[-1] == '\\' :
                 command += line[0:-1] + ' '
                 more = True
                 continue
             else :
                 # add the last (current) line to the command
                 command += line
                 if command or more:
                     # push to raw history, so hist line numbers stay in sync
                     ip.IP.input_hist_raw.append("# " + command + "\n")
                     
                     more = ip.IP.push(ip.IP.prefilter(command,more))
                     command = ''
                     # IPython's runsource returns None if there was an error
                     # compiling the code.  This allows us to stop processing right
                     # away, so the user gets the error message at the right place.
                     if more is None:
                         break
         # final newline in case the input didn't have it, so that the code
         # actually does get executed
         if more:
             ip.IP.push('\n')
 
     ip.IP.runlines = _runlines
 
 main()

IPython/UserConfig/ipy_user_conf.py

@@ -1,107 +1,116 b''
 """ User configuration file for IPython
 
 This is a more flexible and safe way to configure ipython than *rc files
 (ipythonrc, ipythonrc-pysh etc.)
 
 This file is always imported on ipython startup. You can import the
 ipython extensions you need here (see IPython/Extensions directory).
 
 Feel free to edit this file to customize your ipython experience.
 
 Note that as such this file does nothing, for backwards compatibility.
 Consult e.g. file 'ipy_profile_sh.py' for an example of the things 
 you can do here.
 
 See http://ipython.scipy.org/moin/IpythonExtensionApi for detailed
 description on what you could do here.
 """
 
 # Most of your config files and extensions will probably start with this import
 
 import IPython.ipapi
 ip = IPython.ipapi.get()
 
 # You probably want to uncomment this if you did %upgrade -nolegacy
 # import ipy_defaults    
 
 import os   
 
 def main():   
 
     # uncomment if you want to get ipython -p sh behaviour
     # without having to use command line switches  
     # import ipy_profile_sh
 
     # Configure your favourite editor?
     # Good idea e.g. for %edit os.path.isfile
 
     #import ipy_editors
     
     # Choose one of these:
     
     #ipy_editors.scite()
     #ipy_editors.scite('c:/opt/scite/scite.exe')
     #ipy_editors.komodo()
     #ipy_editors.idle()
     # ... or many others, try 'ipy_editors??' after import to see them
     
     # Or roll your own:
     #ipy_editors.install_editor("c:/opt/jed +$line $file")
     
     
     o = ip.options
     # An example on how to set options
     #o.autocall = 1
     o.system_verbose = 0
     
     #import_all("os sys")
     #execf('~/_ipython/ns.py')
 
 
     # -- prompt
     # A different, more compact set of prompts from the default ones, that
     # always show your current location in the filesystem:
 
     #o.prompt_in1 = r'\C_LightBlue[\C_LightCyan\Y2\C_LightBlue]\C_Normal\n\C_Green|\#>'
     #o.prompt_in2 = r'.\D: '
     #o.prompt_out = r'[\#] '
     
     # Try one of these color settings if you can't read the text easily
     # autoexec is a list of IPython commands to execute on startup
     #o.autoexec.append('%colors LightBG')
     #o.autoexec.append('%colors NoColor')
     #o.autoexec.append('%colors Linux')
     
     # for sane integer division that converts to float (1/2 == 0.5)
     #o.autoexec.append('from __future__ import division')
     
     # For %tasks and %kill
     #import jobctrl 
     
     # For autoreloading of modules (%autoreload, %aimport)    
     #import ipy_autoreload
     
     # For winpdb support (%wdb)
     #import ipy_winpdb
     
     # For bzr completer, requires bzrlib (the python installation of bzr)
     #ip.load('ipy_bzr')
     
     # Tab completer that is not quite so picky (i.e. 
     # "foo".<TAB> and str(2).<TAB> will work). Complete 
     # at your own risk!
     #import ipy_greedycompleter
     
+    # If you are on Linux, you may be annoyed by
+    # "Display all N possibilities? (y or n)" on tab completion,
+    # as well as the paging through "more". Uncomment the following
+    # lines to disable that behaviour
+    #import readline
+    #readline.parse_and_bind('set completion-query-items 1000')
+    #readline.parse_and_bind('set page-completions no')
+    
+    
     
     
 # some config helper functions you can use 
 def import_all(modules):
     """ Usage: import_all("os sys") """ 
     for m in modules.split():
         ip.ex("from %s import *" % m)
         
 def execf(fname):
     """ Execute a file in user namespace """
     ip.ex('execfile("%s")' % os.path.expanduser(fname))
 
 main()

IPython/frontend/cocoa/cocoa_frontend.py

@@ -1,395 +1,425 b''
 # encoding: utf-8
-# -*- test-case-name: ipython1.frontend.cocoa.tests.test_cocoa_frontend -*-
+# -*- test-case-name: IPython.frontend.cocoa.tests.test_cocoa_frontend -*-
 
-"""PyObjC classes to provide a Cocoa frontend to the ipython1.kernel.engineservice.EngineService.
+"""PyObjC classes to provide a Cocoa frontend to the 
+IPython.kernel.engineservice.IEngineBase.
 
-The Cocoa frontend is divided into two classes:
-    - IPythonCocoaController
-    - IPythonCLITextViewDelegate
+To add an IPython interpreter to a cocoa app, instantiate an 
+IPythonCocoaController in a XIB and connect its textView outlet to an
+NSTextView instance in your UI. That's it.
 
-To add an IPython interpreter to a cocoa app, instantiate both of these classes in an XIB...[FINISH]
+Author: Barry Wark
 """
 
 __docformat__ = "restructuredtext en"
 
-#-------------------------------------------------------------------------------
-#       Copyright (C) 2008  Barry Wark <[e-mail]>
+#-----------------------------------------------------------------------------
+#       Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
 import objc
 import uuid
 
 from Foundation import NSObject, NSMutableArray, NSMutableDictionary,\
                         NSLog, NSNotificationCenter, NSMakeRange,\
                         NSLocalizedString, NSIntersectionRange
                         
 from AppKit import NSApplicationWillTerminateNotification, NSBeep,\
                     NSTextView, NSRulerView, NSVerticalRuler
 
 from pprint import saferepr
 
 import IPython
-from IPython.kernel.engineservice import EngineService, ThreadedEngineService
+from IPython.kernel.engineservice import ThreadedEngineService
 from IPython.frontend.frontendbase import FrontEndBase
 
 from twisted.internet.threads import blockingCallFromThread
 from twisted.python.failure import Failure
 
-#-------------------------------------------------------------------------------
+#------------------------------------------------------------------------------
 # Classes to implement the Cocoa frontend
-#-------------------------------------------------------------------------------
+#------------------------------------------------------------------------------
 
 # TODO: 
-#   1. use MultiEngineClient and out-of-process engine rather than ThreadedEngineService?
+#   1. use MultiEngineClient and out-of-process engine rather than 
+#       ThreadedEngineService?
 #   2. integrate Xgrid launching of engines
         
     
 
 
 class IPythonCocoaController(NSObject, FrontEndBase):
     userNS = objc.ivar() #mirror of engine.user_ns (key=>str(value))
     waitingForEngine = objc.ivar().bool()
     textView = objc.IBOutlet()
     
     def init(self):
         self = super(IPythonCocoaController, self).init()
         FrontEndBase.__init__(self, engine=ThreadedEngineService())
         if(self != None):
             self._common_init()
         
         return self
     
     def _common_init(self):
         """_common_init"""
         
         self.userNS = NSMutableDictionary.dictionary()
         self.waitingForEngine = False
         
         self.lines = {}
         self.tabSpaces = 4
         self.tabUsesSpaces = True
-        self.currentBlockID = self.nextBlockID()
+        self.currentBlockID = self.next_block_ID()
         self.blockRanges = {} # blockID=>NSRange
     
     
     def awakeFromNib(self):
         """awakeFromNib"""
         
         self._common_init()
         
         # Start the IPython engine
         self.engine.startService()
         NSLog('IPython engine started')
         
         # Register for app termination
-        NSNotificationCenter.defaultCenter().addObserver_selector_name_object_(self,
-                                                'appWillTerminate:',
-                                                 NSApplicationWillTerminateNotification,
-                                                  None)
+        nc = NSNotificationCenter.defaultCenter()
+        nc.addObserver_selector_name_object_(
+                                    self,
+                                    'appWillTerminate:',
+                                    NSApplicationWillTerminateNotification,
+                                    None)
         
         self.textView.setDelegate_(self)
         self.textView.enclosingScrollView().setHasVerticalRuler_(True)
-        self.verticalRulerView = NSRulerView.alloc().initWithScrollView_orientation_(
-            self.textView.enclosingScrollView(),
-            NSVerticalRuler)
+        r = NSRulerView.alloc().initWithScrollView_orientation_(
+                                        self.textView.enclosingScrollView(),
+                                        NSVerticalRuler)
+        self.verticalRulerView = r
         self.verticalRulerView.setClientView_(self.textView)
-        self.startCLIForTextView()
+        self._start_cli_banner()
     
     
     def appWillTerminate_(self, notification):
         """appWillTerminate"""
         
         self.engine.stopService()
     
     
     def complete(self, token):
         """Complete token in engine's user_ns
         
         Parameters
         ----------
         token : string
         
         Result
         ------
-        Deferred result of ipython1.kernel.engineservice.IEngineInteractive.complete
+        Deferred result of 
+        IPython.kernel.engineservice.IEngineBase.complete
         """
         
         return self.engine.complete(token)
     
     
     def execute(self, block, blockID=None):
         self.waitingForEngine = True
         self.willChangeValueForKey_('commandHistory')
         d = super(IPythonCocoaController, self).execute(block, blockID)
-        d.addBoth(self._engineDone)
-        d.addCallback(self._updateUserNS)
+        d.addBoth(self._engine_done)
+        d.addCallback(self._update_user_ns)
         
         return d
     
         
-    def _engineDone(self, x):
+    def _engine_done(self, x):
         self.waitingForEngine = False
         self.didChangeValueForKey_('commandHistory')
         return x
     
-    def _updateUserNS(self, result):
+    def _update_user_ns(self, result):
         """Update self.userNS from self.engine's namespace"""
         d = self.engine.keys()
-        d.addCallback(self._getEngineNamepsaceValuesForKeys)
+        d.addCallback(self._get_engine_namespace_values_for_keys)
         
         return result
     
     
-    def _getEngineNamepsaceValuesForKeys(self, keys):
+    def _get_engine_namespace_values_for_keys(self, keys):
         d = self.engine.pull(keys)
-        d.addCallback(self._storeEngineNamespaceValues, keys=keys)
+        d.addCallback(self._store_engine_namespace_values, keys=keys)
     
     
-    def _storeEngineNamespaceValues(self, values, keys=[]):
+    def _store_engine_namespace_values(self, values, keys=[]):
         assert(len(values) == len(keys))
         self.willChangeValueForKey_('userNS')
         for (k,v) in zip(keys,values):
             self.userNS[k] = saferepr(v)
         self.didChangeValueForKey_('userNS')
     
     
-    def startCLIForTextView(self):
+    def update_cell_prompt(self, result):
+        if(isinstance(result, Failure)):
+            blockID = result.blockID
+        else:
+            blockID = result['blockID']
+        
+        
+        self.insert_text(self.input_prompt(result=result),
+                textRange=NSMakeRange(self.blockRanges[blockID].location,0),
+                scrollToVisible=False
+                )
+        
+        return result
+    
+    
+    def render_result(self, result):
+        blockID = result['blockID']
+        inputRange = self.blockRanges[blockID]
+        del self.blockRanges[blockID]
+        
+        #print inputRange,self.current_block_range()
+        self.insert_text('\n' +
+                self.output_prompt(result) +
+                result.get('display',{}).get('pprint','') +
+                '\n\n',
+                textRange=NSMakeRange(inputRange.location+inputRange.length,
+                                    0))
+        return result
+    
+        
+    def render_error(self, failure):
+        self.insert_text('\n\n'+str(failure)+'\n\n')
+        self.start_new_block()
+        return failure
+    
+    
+    def _start_cli_banner(self):
         """Print banner"""
         
-        banner = """IPython1 %s -- An enhanced Interactive Python.""" % IPython.__version__
+        banner = """IPython1 %s -- An enhanced Interactive Python.""" % \
+                    IPython.__version__
         
         self.insert_text(banner + '\n\n')
     
-    # NSTextView/IPythonTextView delegate methods
+    
+    def start_new_block(self):
+        """"""
+        
+        self.currentBlockID = self.next_block_ID()
+    
+    
+    
+    def next_block_ID(self):
+        
+        return uuid.uuid4()
+    
+    def current_block_range(self):
+        return self.blockRanges.get(self.currentBlockID, 
+                        NSMakeRange(self.textView.textStorage().length(), 
+                        0))
+    
+    def current_block(self):
+        """The current block's text"""
+        
+        return self.text_for_range(self.current_block_range())
+    
+    def text_for_range(self, textRange):
+        """text_for_range"""
+        
+        ts = self.textView.textStorage()
+        return ts.string().substringWithRange_(textRange)
+    
+    def current_line(self):
+        block = self.text_for_range(self.current_block_range())
+        block = block.split('\n')
+        return block[-1]
+    
+    
+    def insert_text(self, string=None, textRange=None, scrollToVisible=True):
+        """Insert text into textView at textRange, updating blockRanges 
+        as necessary
+        """
+        
+        if(textRange == None):
+            #range for end of text
+            textRange = NSMakeRange(self.textView.textStorage().length(), 0) 
+        
+        for r in self.blockRanges.itervalues():
+            intersection = NSIntersectionRange(r,textRange)
+            if(intersection.length == 0): #ranges don't intersect
+                if r.location >= textRange.location:
+                    r.location += len(string)
+            else: #ranges intersect
+                if(r.location <= textRange.location):
+                    assert(intersection.length == textRange.length)
+                    r.length += textRange.length
+                else:
+                    r.location += intersection.length
+        
+        self.textView.replaceCharactersInRange_withString_(
+            textRange, string)
+        self.textView.setSelectedRange_(
+            NSMakeRange(textRange.location+len(string), 0))
+        if(scrollToVisible):
+            self.textView.scrollRangeToVisible_(textRange)
+        
+    
+    
+    
+    def replace_current_block_with_string(self, textView, string):
+        textView.replaceCharactersInRange_withString_(
+                                                self.current_block_range(),
+                                                string)
+        self.current_block_range().length = len(string)
+        r = NSMakeRange(textView.textStorage().length(), 0)
+        textView.scrollRangeToVisible_(r)
+        textView.setSelectedRange_(r)
+    
+    
+    def current_indent_string(self):
+        """returns string for indent or None if no indent"""
+        
+        if(len(self.current_block()) > 0):
+            lines = self.current_block().split('\n')
+            currentIndent = len(lines[-1]) - len(lines[-1])
+            if(currentIndent == 0):
+                currentIndent = self.tabSpaces
+        
+            if(self.tabUsesSpaces):
+                result = ' ' * currentIndent
+            else:
+                result = '\t' * (currentIndent/self.tabSpaces)
+        else:
+            result = None
+        
+        return result
+    
+    
+    # NSTextView delegate methods...
     def textView_doCommandBySelector_(self, textView, selector):
         assert(textView == self.textView)
         NSLog("textView_doCommandBySelector_: "+selector)
         
         
         if(selector == 'insertNewline:'):
-            indent = self.currentIndentString()
+            indent = self.current_indent_string()
             if(indent):
-                line = indent + self.currentLine()
+                line = indent + self.current_line()
             else:
-                line = self.currentLine()
+                line = self.current_line()
             
-            if(self.is_complete(self.currentBlock())):
-                self.execute(self.currentBlock(),
+            if(self.is_complete(self.current_block())):
+                self.execute(self.current_block(),
                                 blockID=self.currentBlockID)
-                self.startNewBlock()
+                self.start_new_block()
                 
                 return True
             
             return False
         
         elif(selector == 'moveUp:'):
-            prevBlock = self.get_history_previous(self.currentBlock())
+            prevBlock = self.get_history_previous(self.current_block())
             if(prevBlock != None):
-                self.replaceCurrentBlockWithString(textView, prevBlock)
+                self.replace_current_block_with_string(textView, prevBlock)
             else:
                 NSBeep()
             return True
         
         elif(selector == 'moveDown:'):
             nextBlock = self.get_history_next()
             if(nextBlock != None):
-                self.replaceCurrentBlockWithString(textView, nextBlock)
+                self.replace_current_block_with_string(textView, nextBlock)
             else:
                 NSBeep()
             return True
         
         elif(selector == 'moveToBeginningOfParagraph:'):
-            textView.setSelectedRange_(NSMakeRange(self.currentBlockRange().location, 0))
+            textView.setSelectedRange_(NSMakeRange(
+                                        self.current_block_range().location, 
+                                        0))
             return True
         elif(selector == 'moveToEndOfParagraph:'):
-            textView.setSelectedRange_(NSMakeRange(self.currentBlockRange().location + \
-                                                self.currentBlockRange().length, 0))
+            textView.setSelectedRange_(NSMakeRange(
+                                    self.current_block_range().location + \
+                                    self.current_block_range().length, 0))
             return True
         elif(selector == 'deleteToEndOfParagraph:'):
-            if(textView.selectedRange().location <= self.currentBlockRange().location):
+            if(textView.selectedRange().location <= \
+                self.current_block_range().location):
                 # Intersect the selected range with the current line range
-                if(self.currentBlockRange().length < 0):
+                if(self.current_block_range().length < 0):
                     self.blockRanges[self.currentBlockID].length = 0
             
                 r = NSIntersectionRange(textView.rangesForUserTextChange()[0],
-                                        self.currentBlockRange())
+                                        self.current_block_range())
                 
                 if(r.length > 0): #no intersection
                     textView.setSelectedRange_(r)
             
             return False # don't actually handle the delete
         
         elif(selector == 'insertTab:'):
-            if(len(self.currentLine().strip()) == 0): #only white space
+            if(len(self.current_line().strip()) == 0): #only white space
                 return False
             else:
                 self.textView.complete_(self)
                 return True
         
         elif(selector == 'deleteBackward:'):
             #if we're at the beginning of the current block, ignore
-            if(textView.selectedRange().location == self.currentBlockRange().location):
+            if(textView.selectedRange().location == \
+                self.current_block_range().location):
                 return True
             else:
-                self.currentBlockRange().length-=1
+                self.current_block_range().length-=1
                 return False
         return False
     
     
-    def textView_shouldChangeTextInRanges_replacementStrings_(self, textView, ranges, replacementStrings):
+    def textView_shouldChangeTextInRanges_replacementStrings_(self, 
+        textView, ranges, replacementStrings):
         """
         Delegate method for NSTextView.
         
-        Refuse change text in ranges not at end, but make those changes at end.
+        Refuse change text in ranges not at end, but make those changes at 
+        end.
         """
         
-        #print 'textView_shouldChangeTextInRanges_replacementStrings_:',ranges,replacementStrings
         assert(len(ranges) == len(replacementStrings))
         allow = True
         for r,s in zip(ranges, replacementStrings):
             r = r.rangeValue()
             if(textView.textStorage().length() > 0 and
-                    r.location < self.currentBlockRange().location):
+                    r.location < self.current_block_range().location):
                 self.insert_text(s)
                 allow = False
             
             
-            self.blockRanges.setdefault(self.currentBlockID, self.currentBlockRange()).length += len(s)
+            self.blockRanges.setdefault(self.currentBlockID, 
+                                        self.current_block_range()).length +=\
+                                         len(s)
         
         return allow
     
-    def textView_completions_forPartialWordRange_indexOfSelectedItem_(self, textView, words, charRange, index):
+    def textView_completions_forPartialWordRange_indexOfSelectedItem_(self, 
+        textView, words, charRange, index):
         try:
-            token = textView.textStorage().string().substringWithRange_(charRange)
+            ts = textView.textStorage()
+            token = ts.string().substringWithRange_(charRange)
             completions = blockingCallFromThread(self.complete, token)
         except:
             completions = objc.nil
             NSBeep()
         
         return (completions,0)
     
-    
-    def startNewBlock(self):
-        """"""
-        
-        self.currentBlockID = self.nextBlockID()
-    
-    
-    
-    def nextBlockID(self):
-        
-        return uuid.uuid4()
-    
-    def currentBlockRange(self):
-        return self.blockRanges.get(self.currentBlockID, NSMakeRange(self.textView.textStorage().length(), 0))
-    
-    def currentBlock(self):
-        """The current block's text"""
-        
-        return self.textForRange(self.currentBlockRange())
-    
-    def textForRange(self, textRange):
-        """textForRange"""
-        
-        return self.textView.textStorage().string().substringWithRange_(textRange)
-    
-    def currentLine(self):
-        block = self.textForRange(self.currentBlockRange())
-        block = block.split('\n')
-        return block[-1]
-    
-    def update_cell_prompt(self, result):
-        if(isinstance(result, Failure)):
-            blockID = result.blockID
-        else:
-            blockID = result['blockID']
-        
-        
-        self.insert_text(self.input_prompt(result=result),
-                        textRange=NSMakeRange(self.blockRanges[blockID].location,0),
-                        scrollToVisible=False
-                        )
-        
-        return result
-    
-    
-    def render_result(self, result):
-        blockID = result['blockID']
-        inputRange = self.blockRanges[blockID]
-        del self.blockRanges[blockID]
-        
-        #print inputRange,self.currentBlockRange()
-        self.insert_text('\n' +
-                        self.output_prompt(result) +
-                        result.get('display',{}).get('pprint','') +
-                        '\n\n',
-                        textRange=NSMakeRange(inputRange.location+inputRange.length, 0))
-        return result
-    
-        
-    def render_error(self, failure):
-        self.insert_text('\n\n'+str(failure)+'\n\n')
-        self.startNewBlock()
-        return failure
-    
-    
-    def insert_text(self, string=None, textRange=None, scrollToVisible=True):
-        """Insert text into textView at textRange, updating blockRanges as necessary"""
-        
-        if(textRange == None):
-            textRange = NSMakeRange(self.textView.textStorage().length(), 0) #range for end of text
-        
-        for r in self.blockRanges.itervalues():
-            intersection = NSIntersectionRange(r,textRange)
-            if(intersection.length == 0): #ranges don't intersect
-                if r.location >= textRange.location:
-                    r.location += len(string)
-            else: #ranges intersect
-                if(r.location <= textRange.location):
-                    assert(intersection.length == textRange.length)
-                    r.length += textRange.length
-                else:
-                    r.location += intersection.length
-        
-        self.textView.replaceCharactersInRange_withString_(textRange, string) #textStorage().string()
-        self.textView.setSelectedRange_(NSMakeRange(textRange.location+len(string), 0))
-        if(scrollToVisible):
-            self.textView.scrollRangeToVisible_(textRange)
-        
-    
-    
-    def replaceCurrentBlockWithString(self, textView, string):
-        textView.replaceCharactersInRange_withString_(self.currentBlockRange(),
-                                                        string)
-        self.currentBlockRange().length = len(string)
-        r = NSMakeRange(textView.textStorage().length(), 0)
-        textView.scrollRangeToVisible_(r)
-        textView.setSelectedRange_(r)
-    
-    
-    def currentIndentString(self):
-        """returns string for indent or None if no indent"""
-        
-        if(len(self.currentBlock()) > 0):
-            lines = self.currentBlock().split('\n')
-            currentIndent = len(lines[-1]) - len(lines[-1])
-            if(currentIndent == 0):
-                currentIndent = self.tabSpaces
-        
-            if(self.tabUsesSpaces):
-                result = ' ' * currentIndent
-            else:
-                result = '\t' * (currentIndent/self.tabSpaces)
-        else:
-            result = None
-        
-        return result
-    
 

IPython/frontend/cocoa/tests/test_cocoa_frontend.py

@@ -1,77 +1,72 b''
 # encoding: utf-8
-"""This file contains unittests for the ipython1.frontend.cocoa.cocoa_frontend module.
-
-Things that should be tested:
-
- - IPythonCocoaController instantiates an IEngineInteractive
- - IPythonCocoaController executes code on the engine
- - IPythonCocoaController mirrors engine's user_ns
+"""This file contains unittests for the 
+IPython.frontend.cocoa.cocoa_frontend module.
 """
 __docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#       Copyright (C) 2005  Fernando Perez <[e-mail]>
-#                           Brian E Granger <[e-mail]>
-#                           Benjamin Ragan-Kelley <[e-mail]>
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
-
-#-------------------------------------------------------------------------------
-# Imports
-#-------------------------------------------------------------------------------
-from IPython.kernel.core.interpreter import Interpreter
+                                                                            
+#---------------------------------------------------------------------------
+#       Copyright (C) 2005  The IPython Development Team    
+#                                                                           
+#  Distributed under the terms of the BSD License.  The full license is in  
+#  the file COPYING, distributed as part of this software.                  
+#---------------------------------------------------------------------------
+                                                                            
+#---------------------------------------------------------------------------
+# Imports                                                                   
+#---------------------------------------------------------------------------
+from IPython.kernel.core.interpreter import Interpreter                     
 import IPython.kernel.engineservice as es
 from IPython.testing.util import DeferredTestCase
 from twisted.internet.defer import succeed
 from IPython.frontend.cocoa.cocoa_frontend import IPythonCocoaController
                                                     
 from Foundation import NSMakeRect
 from AppKit import NSTextView, NSScrollView                                                
 
 class TestIPythonCocoaControler(DeferredTestCase):
     """Tests for IPythonCocoaController"""
     
     def setUp(self):
         self.controller = IPythonCocoaController.alloc().init()
         self.engine = es.EngineService()
         self.engine.startService()
     
     
     def tearDown(self):
         self.controller = None
         self.engine.stopService()
     
     def testControllerExecutesCode(self):
         code ="""5+5"""
         expected = Interpreter().execute(code)
         del expected['number']
         def removeNumberAndID(result):
             del result['number']
             del result['id']
             return result
-        self.assertDeferredEquals(self.controller.execute(code).addCallback(removeNumberAndID), expected)
+        self.assertDeferredEquals(
+            self.controller.execute(code).addCallback(removeNumberAndID), 
+            expected)
     
     def testControllerMirrorsUserNSWithValuesAsStrings(self):
         code = """userns1=1;userns2=2"""
         def testControllerUserNS(result):
             self.assertEquals(self.controller.userNS['userns1'], 1)
             self.assertEquals(self.controller.userNS['userns2'], 2)
         
         self.controller.execute(code).addCallback(testControllerUserNS)
     
     
     def testControllerInstantiatesIEngine(self):
         self.assert_(es.IEngineBase.providedBy(self.controller.engine))
     
     def testControllerCompletesToken(self):
         code = """longNameVariable=10"""
         def testCompletes(result):
             self.assert_("longNameVariable" in result)
         
         def testCompleteToken(result):
             self.controller.complete("longNa").addCallback(testCompletes)
         
         self.controller.execute(code).addCallback(testCompletes)
     

IPython/frontend/frontendbase.py

@@ -1,326 +1,352 b''
 # encoding: utf-8
+# -*- test-case-name: IPython.frontend.tests.test_frontendbase -*-
 """
-frontendbase provides an interface and base class for GUI frontends for IPython.kernel/IPython.kernel.core.
+frontendbase provides an interface and base class for GUI frontends for 
+IPython.kernel/IPython.kernel.core.
 
 Frontend implementations will likely want to subclass FrontEndBase. 
 
 Author: Barry Wark
 """
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 import string
 import uuid
 import _ast
 
 import zope.interface as zi
 
 from IPython.kernel.core.history import FrontEndHistory
 from IPython.kernel.core.util import Bunch
 from IPython.kernel.engineservice import IEngineCore
 
 from twisted.python.failure import Failure
 
 ##############################################################################
 # TEMPORARY!!! fake configuration, while we decide whether to use tconfig or
 # not
 
 rc = Bunch()
 rc.prompt_in1 = r'In [$number]:  '
 rc.prompt_in2 = r'...'
 rc.prompt_out = r'Out [$number]:  '
 
 ##############################################################################
 
 class IFrontEndFactory(zi.Interface):
     """Factory interface for frontends."""
     
     def __call__(engine=None, history=None):
         """
         Parameters:
         interpreter : IPython.kernel.engineservice.IEngineCore
         """
         
         pass
 
 
 
 class IFrontEnd(zi.Interface):
     """Interface for frontends. All methods return t.i.d.Deferred"""
     
-    zi.Attribute("input_prompt_template", "string.Template instance substituteable with execute result.")
-    zi.Attribute("output_prompt_template", "string.Template instance substituteable with execute result.")
-    zi.Attribute("continuation_prompt_template", "string.Template instance substituteable with execute result.")
+    zi.Attribute("input_prompt_template", "string.Template instance\
+                substituteable with execute result.")
+    zi.Attribute("output_prompt_template", "string.Template instance\
+                substituteable with execute result.")
+    zi.Attribute("continuation_prompt_template", "string.Template instance\
+                substituteable with execute result.")
     
     def update_cell_prompt(self, result):
         """Subclass may override to update the input prompt for a block. 
-        Since this method will be called as a twisted.internet.defer.Deferred's callback,
-        implementations should return result when finished."""
+        Since this method will be called as a 
+        twisted.internet.defer.Deferred's callback,
+        implementations should return result when finished.
+        
+        NB: result is a failure if the execute returned a failre. 
+        To get the blockID, you should do something like::
+            if(isinstance(result, twisted.python.failure.Failure)):
+                blockID = result.blockID
+            else:
+                blockID = result['blockID']
+        """
         
         pass
     
     def render_result(self, result):
-        """Render the result of an execute call. Implementors may choose the method of rendering.
-        For example, a notebook-style frontend might render a Chaco plot inline.
+        """Render the result of an execute call. Implementors may choose the
+         method of rendering.
+        For example, a notebook-style frontend might render a Chaco plot 
+        inline.
         
         Parameters:
             result : dict (result of IEngineBase.execute )
         
         Result:
             Output of frontend rendering
         """
         
         pass
     
     def render_error(self, failure):
-        """Subclasses must override to render the failure. Since this method will be called as a 
-        twisted.internet.defer.Deferred's callback, implementations should return result 
-        when finished."""
+        """Subclasses must override to render the failure. Since this method 
+        ill be called as a twisted.internet.defer.Deferred's callback, 
+        implementations should return result when finished.
+        """
         
         pass
     
     
     def input_prompt(result={}):
-        """Returns the input prompt by subsituting into self.input_prompt_template"""
+        """Returns the input prompt by subsituting into 
+        self.input_prompt_template
+        """
         pass
     
     def output_prompt(result):
-        """Returns the output prompt by subsituting into self.output_prompt_template"""
+        """Returns the output prompt by subsituting into 
+        self.output_prompt_template
+        """
         
         pass
     
     def continuation_prompt():
-        """Returns the continuation prompt by subsituting into self.continuation_prompt_template"""
+        """Returns the continuation prompt by subsituting into 
+        self.continuation_prompt_template
+        """
         
         pass
     
     def is_complete(block):
         """Returns True if block is complete, False otherwise."""
         
         pass
     
     def compile_ast(block):
         """Compiles block to an _ast.AST"""
         
         pass
     
     
     def get_history_previous(currentBlock):
         """Returns the block previous in  the history. Saves currentBlock if
         the history_cursor is currently at the end of the input history"""
         pass
     
     def get_history_next():
         """Returns the next block in the history."""
         
         pass
     
 
 class FrontEndBase(object):
     """
     FrontEndBase manages the state tasks for a CLI frontend:
         - Input and output history management
         - Input/continuation and output prompt generation
         
     Some issues (due to possibly unavailable engine):
         - How do we get the current cell number for the engine?
         - How do we handle completions?
     """
     
     zi.implements(IFrontEnd)
     zi.classProvides(IFrontEndFactory)
     
     history_cursor = 0
     
     current_indent_level = 0
     
     
     input_prompt_template = string.Template(rc.prompt_in1)
     output_prompt_template = string.Template(rc.prompt_out)
     continuation_prompt_template = string.Template(rc.prompt_in2)
     
     def __init__(self, engine=None, history=None):
         assert(engine==None or IEngineCore.providedBy(engine))
         self.engine = IEngineCore(engine)
         if history is None:
                 self.history = FrontEndHistory(input_cache=[''])
         else:
             self.history = history
         
     
     def input_prompt(self, result={}):
         """Returns the current input prompt
         
         It would be great to use ipython1.core.prompts.Prompt1 here
         """
         
         result.setdefault('number','')
         
         return self.input_prompt_template.safe_substitute(result)
     
     
     def continuation_prompt(self):
         """Returns the current continuation prompt"""
         
         return self.continuation_prompt_template.safe_substitute()
     
     def output_prompt(self, result):
         """Returns the output prompt for result"""
         
         return self.output_prompt_template.safe_substitute(result)
     
     
     def is_complete(self, block):
         """Determine if block is complete.
         
         Parameters
         block : string
         
         Result 
         True if block can be sent to the engine without compile errors.
         False otherwise.
         """
         
         try:
             ast = self.compile_ast(block)
         except:
             return False
         
         lines = block.split('\n')
         return (len(lines)==1 or str(lines[-1])=='')
     
     
     def compile_ast(self, block):
         """Compile block to an AST
         
         Parameters:
             block : str
         
         Result:
             AST
         
         Throws:
             Exception if block cannot be compiled
         """
         
         return compile(block, "<string>", "exec", _ast.PyCF_ONLY_AST)
     
     
     def execute(self, block, blockID=None):
         """Execute the block and return result.
         
         Parameters:
             block : {str, AST}
             blockID : any
-                Caller may provide an ID to identify this block. result['blockID'] := blockID
+                Caller may provide an ID to identify this block. 
+                result['blockID'] := blockID
         
         Result:
             Deferred result of self.interpreter.execute
         """
         
         if(not self.is_complete(block)):
             return Failure(Exception("Block is not compilable"))
         
         if(blockID == None):
             blockID = uuid.uuid4() #random UUID
         
         d = self.engine.execute(block)
         d.addCallback(self._add_history, block=block)
         d.addBoth(self._add_block_id, blockID)
         d.addBoth(self.update_cell_prompt)
         d.addCallbacks(self.render_result, errback=self.render_error)
         
         return d
     
     
     def _add_block_id(self, result, blockID):
-        """Add the blockID to result or failure. Unfortunatley, we have to treat failures
-        differently than result dicts
+        """Add the blockID to result or failure. Unfortunatley, we have to 
+        treat failures differently than result dicts.
         """
         
         if(isinstance(result, Failure)):
             result.blockID = blockID
         else:
             result['blockID'] = blockID
         
         return result
     
     def _add_history(self, result, block=None):
         """Add block to the history"""
         
         assert(block != None)
         self.history.add_items([block])
         self.history_cursor += 1
         
         return result
     
     
     def get_history_previous(self, currentBlock):
         """ Returns previous history string and decrement history cursor.
         """
         command = self.history.get_history_item(self.history_cursor - 1)
         
         if command is not None:
             if(self.history_cursor == len(self.history.input_cache)):
                 self.history.input_cache[self.history_cursor] = currentBlock
             self.history_cursor -= 1
         return command
     
     
     def get_history_next(self):
         """ Returns next history string and increment history cursor.
         """
         command = self.history.get_history_item(self.history_cursor+1)
         
         if command is not None:
             self.history_cursor += 1
         return command
     
     ###
     # Subclasses probably want to override these methods...
     ###
     
     def update_cell_prompt(self, result):
         """Subclass may override to update the input prompt for a block. 
-        Since this method will be called as a twisted.internet.defer.Deferred's callback,
-        implementations should return result when finished.
+        Since this method will be called as a 
+        twisted.internet.defer.Deferred's callback, implementations should 
+        return result when finished.
         
-        NP: result is a failure if the execute returned a failre. To get the blockID, you should
-        do something like::
+        NB: result is a failure if the execute returned a failre. 
+        To get the blockID, you should do something like::
             if(isinstance(result, twisted.python.failure.Failure)):
                 blockID = result.blockID
             else:
                 blockID = result['blockID']
             
         
         """
         
         return result
     
     
     def render_result(self, result):
-        """Subclasses must override to render result. Since this method will be called as a 
-        twisted.internet.defer.Deferred's callback, implementations should return result 
-        when finished."""
+        """Subclasses must override to render result. Since this method will
+        be called as a twisted.internet.defer.Deferred's callback, 
+        implementations should return result when finished.
+        """
         
         return result
     
     
     def render_error(self, failure):
-        """Subclasses must override to render the failure. Since this method will be called as a 
-        twisted.internet.defer.Deferred's callback, implementations should return result 
-        when finished."""
+        """Subclasses must override to render the failure. Since this method 
+        will be called as a twisted.internet.defer.Deferred's callback, 
+        implementations should return result when finished."""
         
         return failure
     
 
 

IPython/frontend/tests/test_frontendbase.py

@@ -1,149 +1,151 b''
 # encoding: utf-8
 
 """This file contains unittests for the frontendbase module."""
 
 __docformat__ = "restructuredtext en"
 
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
-
-#-------------------------------------------------------------------------------
-# Imports
-#-------------------------------------------------------------------------------
+#---------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team                         
+#                                                                           
+#  Distributed under the terms of the BSD License.  The full license is in  
+#  the file COPYING, distributed as part of this software.                  
+#---------------------------------------------------------------------------
+                                                                            
+#---------------------------------------------------------------------------
+# Imports                                                                   
+#---------------------------------------------------------------------------
 
 import unittest
 from IPython.frontend import frontendbase
 from IPython.kernel.engineservice import EngineService
 
 class FrontEndCallbackChecker(frontendbase.FrontEndBase):
     """FrontEndBase subclass for checking callbacks"""
     def __init__(self, engine=None, history=None):
-        super(FrontEndCallbackChecker, self).__init__(engine=engine, history=history)
+        super(FrontEndCallbackChecker, self).__init__(engine=engine, 
+                                                    history=history)
         self.updateCalled = False
         self.renderResultCalled = False
         self.renderErrorCalled = False
     
     def update_cell_prompt(self, result):
         self.updateCalled = True
         return result
     
     def render_result(self, result):
         self.renderResultCalled = True
         return result
     
     
     def render_error(self, failure):
         self.renderErrorCalled = True
         return failure
     
 
 
     
 class TestFrontendBase(unittest.TestCase):
     def setUp(self):
         """Setup the EngineService and FrontEndBase"""
         
         self.fb = FrontEndCallbackChecker(engine=EngineService())
     
     
     def test_implements_IFrontEnd(self):
-        assert(frontendbase.IFrontEnd.implementedBy(frontendbase.FrontEndBase))
+        assert(frontendbase.IFrontEnd.implementedBy(
+                                                frontendbase.FrontEndBase))
     
     
     def test_is_complete_returns_False_for_incomplete_block(self):
         """"""
         
         block = """def test(a):"""
         
         assert(self.fb.is_complete(block) == False)
     
     def test_is_complete_returns_True_for_complete_block(self):
         """"""
         
         block = """def test(a): pass"""
             
         assert(self.fb.is_complete(block))
         
         block = """a=3"""
         
         assert(self.fb.is_complete(block))
     
     
     def test_blockID_added_to_result(self):
         block = """3+3"""
         
         d = self.fb.execute(block, blockID='TEST_ID')
         
         d.addCallback(self.checkBlockID, expected='TEST_ID')
     
     def test_blockID_added_to_failure(self):
         block = "raise  Exception()"
         
         d = self.fb.execute(block,blockID='TEST_ID')
         d.addErrback(self.checkFailureID, expected='TEST_ID')
     
     def checkBlockID(self, result, expected=""):
         assert(result['blockID'] == expected)
     
     
     def checkFailureID(self, failure, expected=""):
         assert(failure.blockID == expected)
     
     
     def test_callbacks_added_to_execute(self):
         """test that
         update_cell_prompt
         render_result
         
         are added to execute request
         """
         
         d = self.fb.execute("10+10")
         d.addCallback(self.checkCallbacks)
     
     
     def checkCallbacks(self, result):
         assert(self.fb.updateCalled)
         assert(self.fb.renderResultCalled)
     
     
     def test_error_callback_added_to_execute(self):
         """test that render_error called on execution error"""
         
         d = self.fb.execute("raise Exception()")
         d.addCallback(self.checkRenderError)
     
     def checkRenderError(self, result):
         assert(self.fb.renderErrorCalled)
     
     def test_history_returns_expected_block(self):
         """Make sure history browsing doesn't fail"""
         
         blocks = ["a=1","a=2","a=3"]
         for b in blocks:
             d = self.fb.execute(b)
         
         # d is now the deferred for the last executed block
         d.addCallback(self.historyTests, blocks)
         
     
     def historyTests(self, result, blocks):
         """historyTests"""
         
         assert(len(blocks) >= 3)
         assert(self.fb.get_history_previous("") == blocks[-2])
         assert(self.fb.get_history_previous("") == blocks[-3])
         assert(self.fb.get_history_next() == blocks[-2])
     
     
     def test_history_returns_none_at_startup(self):
         """test_history_returns_none_at_startup"""
         
         assert(self.fb.get_history_previous("")==None)
         assert(self.fb.get_history_next()==None)
     
     

IPython/genutils.py

@@ -1,2046 +1,2097 b''
 # -*- coding: utf-8 -*-
 """
 General purpose utilities.
 
 This is a grab-bag of stuff I find useful in most programs I write. Some of
 these things are also convenient when working at the command line.
 
 $Id: genutils.py 2998 2008-01-31 10:06:04Z vivainio $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Fernando']
 __license__ = Release.license
 
 #****************************************************************************
 # required modules from the Python standard library
 import __main__
 import commands
 try:
     import doctest
 except ImportError:
     pass
 import os
+import platform
 import re
 import shlex
 import shutil
+import subprocess
 import sys
 import tempfile
 import time
 import types
 import warnings
 
 # Curses and termios are Unix-only modules
 try:
     import curses
     # We need termios as well, so if its import happens to raise, we bail on
     # using curses altogether.
     import termios
 except ImportError:
     USE_CURSES = False
 else:
     # Curses on Solaris may not be complete, so we can't use it there
     USE_CURSES = hasattr(curses,'initscr')
 
 # Other IPython utilities
 import IPython
 from IPython.Itpl import Itpl,itpl,printpl
 from IPython import DPyGetOpt, platutils
 from IPython.generics import result_display
 import IPython.ipapi
 from IPython.external.path import path
 if os.name == "nt":
     from IPython.winconsole import get_console_size
 
 try:
     set
 except:
     from sets import Set as set
 
 
 #****************************************************************************
 # Exceptions
 class Error(Exception):
     """Base class for exceptions in this module."""
     pass
 
 #----------------------------------------------------------------------------
 class IOStream:
     def __init__(self,stream,fallback):
         if not hasattr(stream,'write') or not hasattr(stream,'flush'):
             stream = fallback
         self.stream = stream
         self._swrite = stream.write
         self.flush = stream.flush
 
     def write(self,data):
         try:
             self._swrite(data)
         except:
             try:
                 # print handles some unicode issues which may trip a plain
                 # write() call.  Attempt to emulate write() by using a
                 # trailing comma
                 print >> self.stream, data,
             except:
                 # if we get here, something is seriously broken.
                 print >> sys.stderr, \
                       'ERROR - failed to write data to stream:', self.stream
 
     def close(self):
         pass
                 
         
 class IOTerm:
     """ Term holds the file or file-like objects for handling I/O operations.
 
     These are normally just sys.stdin, sys.stdout and sys.stderr but for
     Windows they can can replaced to allow editing the strings before they are
     displayed."""
 
     # In the future, having IPython channel all its I/O operations through
     # this class will make it easier to embed it into other environments which
     # are not a normal terminal (such as a GUI-based shell)
     def __init__(self,cin=None,cout=None,cerr=None):
         self.cin  = IOStream(cin,sys.stdin)
         self.cout = IOStream(cout,sys.stdout)
         self.cerr = IOStream(cerr,sys.stderr)
         
 # Global variable to be used for all I/O
 Term = IOTerm()
 
 import IPython.rlineimpl as readline
 # Remake Term to use the readline i/o facilities
 if sys.platform == 'win32' and readline.have_readline:
     
     Term = IOTerm(cout=readline._outputfile,cerr=readline._outputfile)
     
 
 #****************************************************************************
 # Generic warning/error printer, used by everything else
 def warn(msg,level=2,exit_val=1):
     """Standard warning printer. Gives formatting consistency.
 
     Output is sent to Term.cerr (sys.stderr by default).
     
     Options:
     
     -level(2): allows finer control:
       0 -> Do nothing, dummy function.
       1 -> Print message.
       2 -> Print 'WARNING:' + message. (Default level).
       3 -> Print 'ERROR:' + message.
       4 -> Print 'FATAL ERROR:' + message and trigger a sys.exit(exit_val).
 
     -exit_val (1): exit value returned by sys.exit() for a level 4
     warning. Ignored for all other levels."""
     
     if level>0:
         header = ['','','WARNING: ','ERROR: ','FATAL ERROR: ']
         print >> Term.cerr, '%s%s' % (header[level],msg)
         if level == 4:
             print >> Term.cerr,'Exiting.\n'
             sys.exit(exit_val)
 
 def info(msg):
     """Equivalent to warn(msg,level=1)."""
 
     warn(msg,level=1)
 
 def error(msg):
     """Equivalent to warn(msg,level=3)."""
 
     warn(msg,level=3)
 
 def fatal(msg,exit_val=1):
     """Equivalent to warn(msg,exit_val=exit_val,level=4)."""
 
     warn(msg,exit_val=exit_val,level=4)
 
 #---------------------------------------------------------------------------
 # Debugging routines
 # 
 def debugx(expr,pre_msg=''):
     """Print the value of an expression from the caller's frame.
     
     Takes an expression, evaluates it in the caller's frame and prints both
     the given expression and the resulting value (as well as a debug mark
     indicating the name of the calling function.  The input must be of a form
     suitable for eval().
 
     An optional message can be passed, which will be prepended to the printed
     expr->value pair."""
     
     cf = sys._getframe(1)
     print '[DBG:%s] %s%s -> %r' % (cf.f_code.co_name,pre_msg,expr,
                                    eval(expr,cf.f_globals,cf.f_locals))
 
 # deactivate it by uncommenting the following line, which makes it a no-op
 #def debugx(expr,pre_msg=''): pass
 
 #----------------------------------------------------------------------------
 StringTypes = types.StringTypes
 
 # Basic timing functionality
 
 # If possible (Unix), use the resource module instead of time.clock()
 try:
     import resource
     def clocku():
         """clocku() -> floating point number
 
         Return the *USER* CPU time in seconds since the start of the process.
         This is done via a call to resource.getrusage, so it avoids the
         wraparound problems in time.clock()."""
         
         return resource.getrusage(resource.RUSAGE_SELF)[0]
     
     def clocks():
         """clocks() -> floating point number
 
         Return the *SYSTEM* CPU time in seconds since the start of the process.
         This is done via a call to resource.getrusage, so it avoids the
         wraparound problems in time.clock()."""
         
         return resource.getrusage(resource.RUSAGE_SELF)[1]
     
     def clock():
         """clock() -> floating point number
 
         Return the *TOTAL USER+SYSTEM* CPU time in seconds since the start of
         the process.  This is done via a call to resource.getrusage, so it
         avoids the wraparound problems in time.clock()."""
 
         u,s = resource.getrusage(resource.RUSAGE_SELF)[:2] 
         return u+s
     
     def clock2():
         """clock2() -> (t_user,t_system)
 
         Similar to clock(), but return a tuple of user/system times."""
         return resource.getrusage(resource.RUSAGE_SELF)[:2]
 
 except ImportError:
     # There is no distinction of user/system time under windows, so we just use
     # time.clock() for everything...
     clocku = clocks = clock = time.clock
     def clock2():
         """Under windows, system CPU time can't be measured.
 
         This just returns clock() and zero."""
         return time.clock(),0.0
 
 def timings_out(reps,func,*args,**kw):
     """timings_out(reps,func,*args,**kw) -> (t_total,t_per_call,output)
 
     Execute a function reps times, return a tuple with the elapsed total
     CPU time in seconds, the time per call and the function's output.
 
     Under Unix, the return value is the sum of user+system time consumed by
     the process, computed via the resource module.  This prevents problems
     related to the wraparound effect which the time.clock() function has.
     
     Under Windows the return value is in wall clock seconds. See the
     documentation for the time module for more details."""
 
     reps = int(reps)
     assert reps >=1, 'reps must be >= 1'
     if reps==1:
         start = clock()
         out = func(*args,**kw)
         tot_time = clock()-start
     else:
         rng = xrange(reps-1) # the last time is executed separately to store output
         start = clock()
         for dummy in rng: func(*args,**kw)
         out = func(*args,**kw)  # one last time
         tot_time = clock()-start
     av_time = tot_time / reps
     return tot_time,av_time,out
 
 def timings(reps,func,*args,**kw):
     """timings(reps,func,*args,**kw) -> (t_total,t_per_call)
 
     Execute a function reps times, return a tuple with the elapsed total CPU
     time in seconds and the time per call. These are just the first two values
     in timings_out()."""
 
     return timings_out(reps,func,*args,**kw)[0:2]
 
 def timing(func,*args,**kw):
     """timing(func,*args,**kw) -> t_total
 
     Execute a function once, return the elapsed total CPU time in
     seconds. This is just the first value in timings_out()."""
 
     return timings_out(1,func,*args,**kw)[0]
 
 #****************************************************************************
 # file and system
 
 def arg_split(s,posix=False):
     """Split a command line's arguments in a shell-like manner.
 
     This is a modified version of the standard library's shlex.split()
     function, but with a default of posix=False for splitting, so that quotes
     in inputs are respected."""
 
     # XXX - there may be unicode-related problems here!!!  I'm not sure that
     # shlex is truly unicode-safe, so it might be necessary to do
     #
     # s = s.encode(sys.stdin.encoding)
     #
     # first, to ensure that shlex gets a normal string.  Input from anyone who
     # knows more about unicode and shlex than I would be good to have here...
     lex = shlex.shlex(s, posix=posix)
     lex.whitespace_split = True
     return list(lex)
 
 def system(cmd,verbose=0,debug=0,header=''):
     """Execute a system command, return its exit status.
 
     Options:
 
     - verbose (0): print the command to be executed.
     
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     stat = 0
     if verbose or debug: print header+cmd
     sys.stdout.flush()
     if not debug: stat = os.system(cmd)
     return stat
 
 def abbrev_cwd():
     """ Return abbreviated version of cwd, e.g. d:mydir """
     cwd = os.getcwd().replace('\\','/')
     drivepart = ''
     tail = cwd
     if sys.platform == 'win32':
         if len(cwd) < 4:
             return cwd
         drivepart,tail = os.path.splitdrive(cwd)
     
 
     parts = tail.split('/')    
     if len(parts) > 2:
         tail = '/'.join(parts[-2:])
         
     return (drivepart + (
         cwd == '/' and '/' or tail))
     
 
 # This function is used by ipython in a lot of places to make system calls.
 # We need it to be slightly different under win32, due to the vagaries of
 # 'network shares'.  A win32 override is below.
 
 def shell(cmd,verbose=0,debug=0,header=''):
     """Execute a command in the system shell, always return None.
 
     Options:
 
     - verbose (0): print the command to be executed.
     
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: this is similar to genutils.system(), but it returns None so it can
     be conveniently used in interactive loops without getting the return value
     (typically 0) printed many times."""
 
     stat = 0
     if verbose or debug: print header+cmd
     # flush stdout so we don't mangle python's buffering
     sys.stdout.flush()
     
     if not debug:
         platutils.set_term_title("IPy " + cmd)
         os.system(cmd)
         platutils.set_term_title("IPy " + abbrev_cwd())
 
 # override shell() for win32 to deal with network shares
 if os.name in ('nt','dos'):
 
     shell_ori = shell
 
     def shell(cmd,verbose=0,debug=0,header=''):
         if os.getcwd().startswith(r"\\"):
             path = os.getcwd()
             # change to c drive (cannot be on UNC-share when issuing os.system,
             # as cmd.exe cannot handle UNC addresses)
             os.chdir("c:")
             # issue pushd to the UNC-share and then run the command
             try:
                 shell_ori('"pushd %s&&"'%path+cmd,verbose,debug,header)
             finally:
                 os.chdir(path)
         else:
             shell_ori(cmd,verbose,debug,header)
 
     shell.__doc__ = shell_ori.__doc__
 
 def getoutput(cmd,verbose=0,debug=0,header='',split=0):
     """Dummy substitute for perl's backquotes.
 
     Executes a command and returns the output.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, the output is returned as a list split on newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class.
     
     This is pretty much deprecated and rarely used, 
     genutils.getoutputerror may be what you need.
     
     """
 
     if verbose or debug: print header+cmd
     if not debug:
         output = os.popen(cmd).read()
         # stipping last \n is here for backwards compat.
         if output.endswith('\n'):
             output = output[:-1]
         if split:
             return output.split('\n')
         else:
             return output
 
 def getoutputerror(cmd,verbose=0,debug=0,header='',split=0):
     """Return (standard output,standard error) of executing cmd in a shell.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, each of stdout/err is returned as a list split on
     newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     if verbose or debug: print header+cmd
     if not cmd:
         if split:
             return [],[]
         else:
             return '',''
     if not debug:
         pin,pout,perr = os.popen3(cmd)
         tout = pout.read().rstrip()
         terr = perr.read().rstrip()
         pin.close()
         pout.close()
         perr.close()
         if split:
             return tout.split('\n'),terr.split('\n')
         else:
             return tout,terr
 
 # for compatibility with older naming conventions
 xsys = system
 bq = getoutput
 
 class SystemExec:
     """Access the system and getoutput functions through a stateful interface.
 
     Note: here we refer to the system and getoutput functions from this
     library, not the ones from the standard python library.
     
     This class offers the system and getoutput functions as methods, but the
     verbose, debug and header parameters can be set for the instance (at
     creation time or later) so that they don't need to be specified on each
     call.
 
     For efficiency reasons, there's no way to override the parameters on a
     per-call basis other than by setting instance attributes. If you need
     local overrides, it's best to directly call system() or getoutput().
 
     The following names are provided as alternate options:
      - xsys: alias to system
      - bq: alias to getoutput
 
     An instance can then be created as:
      >>> sysexec = SystemExec(verbose=1,debug=0,header='Calling: ')
 
     And used as:
      >>> sysexec.xsys('pwd')
      >>> dirlist = sysexec.bq('ls -l')
     """
     
     def __init__(self,verbose=0,debug=0,header='',split=0):
         """Specify the instance's values for verbose, debug and header."""
         setattr_list(self,'verbose debug header split')
 
     def system(self,cmd):
         """Stateful interface to system(), with the same keyword parameters."""
 
         system(cmd,self.verbose,self.debug,self.header)
 
     def shell(self,cmd):
         """Stateful interface to shell(), with the same keyword parameters."""
 
         shell(cmd,self.verbose,self.debug,self.header)
 
     xsys = system  # alias
 
     def getoutput(self,cmd):
         """Stateful interface to getoutput()."""
 
         return getoutput(cmd,self.verbose,self.debug,self.header,self.split)
 
     def getoutputerror(self,cmd):
         """Stateful interface to getoutputerror()."""
 
         return getoutputerror(cmd,self.verbose,self.debug,self.header,self.split)
 
     bq = getoutput  # alias
 
 #-----------------------------------------------------------------------------
 def mutex_opts(dict,ex_op):
     """Check for presence of mutually exclusive keys in a dict.
 
     Call: mutex_opts(dict,[[op1a,op1b],[op2a,op2b]...]"""
     for op1,op2 in ex_op:
         if op1 in dict and op2 in dict:
             raise ValueError,'\n*** ERROR in Arguments *** '\
                   'Options '+op1+' and '+op2+' are mutually exclusive.'
 
 #-----------------------------------------------------------------------------
 def get_py_filename(name):
     """Return a valid python filename in the current directory.
 
     If the given name is not a file, it adds '.py' and searches again.
     Raises IOError with an informative message if the file isn't found."""
 
     name = os.path.expanduser(name)
     if not os.path.isfile(name) and not name.endswith('.py'):
         name += '.py'
     if os.path.isfile(name):
         return name
     else:
         raise IOError,'File `%s` not found.' % name
 
 #-----------------------------------------------------------------------------
 def filefind(fname,alt_dirs = None):
     """Return the given filename either in the current directory, if it
     exists, or in a specified list of directories.
 
     ~ expansion is done on all file and directory names.
 
     Upon an unsuccessful search, raise an IOError exception."""
 
     if alt_dirs is None:
         try:
             alt_dirs = get_home_dir()
         except HomeDirError:
             alt_dirs = os.getcwd()
     search = [fname] + list_strings(alt_dirs)
     search = map(os.path.expanduser,search)
     #print 'search list for',fname,'list:',search  # dbg
     fname = search[0]
     if os.path.isfile(fname):
         return fname
     for direc in search[1:]:
         testname = os.path.join(direc,fname)
         #print 'testname',testname  # dbg
         if os.path.isfile(testname):
             return testname
     raise IOError,'File' + `fname` + \
           ' not found in current or supplied directories:' + `alt_dirs`
 
 #----------------------------------------------------------------------------
 def file_read(filename):
     """Read a file and close it.  Returns the file source."""
     fobj = open(filename,'r');
     source = fobj.read();
     fobj.close()
     return source
 
 def file_readlines(filename):
     """Read a file and close it.  Returns the file source using readlines()."""
     fobj = open(filename,'r');
     lines = fobj.readlines();
     fobj.close()
     return lines
 
 #----------------------------------------------------------------------------
 def target_outdated(target,deps):
     """Determine whether a target is out of date.
 
     target_outdated(target,deps) -> 1/0
 
     deps: list of filenames which MUST exist.
     target: single filename which may or may not exist.
 
     If target doesn't exist or is older than any file listed in deps, return
     true, otherwise return false.
     """
     try:
         target_time = os.path.getmtime(target)
     except os.error:
         return 1
     for dep in deps:
         dep_time = os.path.getmtime(dep)
         if dep_time > target_time:
             #print "For target",target,"Dep failed:",dep # dbg
             #print "times (dep,tar):",dep_time,target_time # dbg
             return 1
     return 0
 
 #-----------------------------------------------------------------------------
 def target_update(target,deps,cmd):
     """Update a target with a given command given a list of dependencies.
 
     target_update(target,deps,cmd) -> runs cmd if target is outdated.
 
     This is just a wrapper around target_outdated() which calls the given
     command if target is outdated."""
 
     if target_outdated(target,deps):
         xsys(cmd)
 
 #----------------------------------------------------------------------------
 def unquote_ends(istr):
     """Remove a single pair of quotes from the endpoints of a string."""
 
     if not istr:
         return istr
     if (istr[0]=="'" and istr[-1]=="'") or \
        (istr[0]=='"' and istr[-1]=='"'):
         return istr[1:-1]
     else:
         return istr
 
 #----------------------------------------------------------------------------
 def process_cmdline(argv,names=[],defaults={},usage=''):
     """ Process command-line options and arguments.
 
     Arguments:
 
     - argv: list of arguments, typically sys.argv.
     
     - names: list of option names. See DPyGetOpt docs for details on options
     syntax.
 
     - defaults: dict of default values.
 
     - usage: optional usage notice to print if a wrong argument is passed.
 
     Return a dict of options and a list of free arguments."""
 
     getopt = DPyGetOpt.DPyGetOpt()
     getopt.setIgnoreCase(0)
     getopt.parseConfiguration(names)
 
     try:
         getopt.processArguments(argv)
     except DPyGetOpt.ArgumentError, exc:
         print usage
         warn('"%s"' % exc,level=4)
 
     defaults.update(getopt.optionValues)
     args = getopt.freeValues
     
     return defaults,args
 
 #----------------------------------------------------------------------------
 def optstr2types(ostr):
     """Convert a string of option names to a dict of type mappings.
 
     optstr2types(str) -> {None:'string_opts',int:'int_opts',float:'float_opts'}
 
     This is used to get the types of all the options in a string formatted
     with the conventions of DPyGetOpt. The 'type' None is used for options
     which are strings (they need no further conversion). This function's main
     use is to get a typemap for use with read_dict().
     """
 
     typeconv = {None:'',int:'',float:''}
     typemap = {'s':None,'i':int,'f':float}
     opt_re = re.compile(r'([\w]*)([^:=]*:?=?)([sif]?)')
 
     for w in ostr.split():
         oname,alias,otype = opt_re.match(w).groups()
         if otype == '' or alias == '!':   # simple switches are integers too
             otype = 'i'
         typeconv[typemap[otype]] += oname + ' '
     return typeconv
 
 #----------------------------------------------------------------------------
 def read_dict(filename,type_conv=None,**opt):
 
     """Read a dictionary of key=value pairs from an input file, optionally
     performing conversions on the resulting values.
 
     read_dict(filename,type_conv,**opt) -> dict
 
     Only one value per line is accepted, the format should be
      # optional comments are ignored
      key value\n
 
     Args:
 
       - type_conv: A dictionary specifying which keys need to be converted to
       which types. By default all keys are read as strings. This dictionary
       should have as its keys valid conversion functions for strings
       (int,long,float,complex, or your own).  The value for each key
       (converter) should be a whitespace separated string containing the names
       of all the entries in the file to be converted using that function. For
       keys to be left alone, use None as the conversion function (only needed
       with purge=1, see below).
 
       - opt: dictionary with extra options as below (default in parens)
 
         purge(0): if set to 1, all keys *not* listed in type_conv are purged out
         of the dictionary to be returned. If purge is going to be used, the
         set of keys to be left as strings also has to be explicitly specified
         using the (non-existent) conversion function None.
 
         fs(None): field separator. This is the key/value separator to be used
         when parsing the file. The None default means any whitespace [behavior
         of string.split()].
 
         strip(0): if 1, strip string values of leading/trailinig whitespace.
 
         warn(1): warning level if requested keys are not found in file.
           - 0: silently ignore.
           - 1: inform but proceed.
           - 2: raise KeyError exception.
 
         no_empty(0): if 1, remove keys with whitespace strings as a value.
 
         unique([]): list of keys (or space separated string) which can't be
         repeated. If one such key is found in the file, each new instance
         overwrites the previous one. For keys not listed here, the behavior is
         to make a list of all appearances.
 
     Example:
     If the input file test.ini has:
       i 3
       x 4.5
       y 5.5
       s hi ho
     Then:
 
     >>> type_conv={int:'i',float:'x',None:'s'}
     >>> read_dict('test.ini')
     {'i': '3', 's': 'hi ho', 'x': '4.5', 'y': '5.5'}
     >>> read_dict('test.ini',type_conv)
     {'i': 3, 's': 'hi ho', 'x': 4.5, 'y': '5.5'}
     >>> read_dict('test.ini',type_conv,purge=1)
     {'i': 3, 's': 'hi ho', 'x': 4.5}
     """
 
     # starting config
     opt.setdefault('purge',0)
     opt.setdefault('fs',None)  # field sep defaults to any whitespace
     opt.setdefault('strip',0)
     opt.setdefault('warn',1)
     opt.setdefault('no_empty',0)
     opt.setdefault('unique','')
     if type(opt['unique']) in StringTypes:
         unique_keys = qw(opt['unique'])
     elif type(opt['unique']) in (types.TupleType,types.ListType):
         unique_keys = opt['unique']
     else:
         raise ValueError, 'Unique keys must be given as a string, List or Tuple'
 
     dict = {}
     # first read in table of values as strings
     file = open(filename,'r')
     for line in file.readlines():
         line = line.strip()
         if len(line) and line[0]=='#': continue
         if len(line)>0:
             lsplit = line.split(opt['fs'],1)
             try:
                 key,val = lsplit
             except ValueError:
                 key,val = lsplit[0],''
             key = key.strip()
             if opt['strip']: val = val.strip()
             if val == "''" or val == '""': val = ''
             if opt['no_empty'] and (val=='' or val.isspace()):
                 continue
             # if a key is found more than once in the file, build a list
             # unless it's in the 'unique' list. In that case, last found in file
             # takes precedence. User beware.
             try:
                 if dict[key] and key in unique_keys:
                     dict[key] = val
                 elif type(dict[key]) is types.ListType:
                     dict[key].append(val)
                 else:
                     dict[key] = [dict[key],val]
             except KeyError:
                 dict[key] = val
     # purge if requested
     if opt['purge']:
         accepted_keys = qwflat(type_conv.values())
         for key in dict.keys():
             if key in accepted_keys: continue
             del(dict[key])
     # now convert if requested
     if type_conv==None: return dict
     conversions = type_conv.keys()
     try: conversions.remove(None)
     except: pass
     for convert in conversions:
         for val in qw(type_conv[convert]):
             try:
                 dict[val] = convert(dict[val])
             except KeyError,e:
                 if opt['warn'] == 0:
                     pass
                 elif opt['warn'] == 1:
                     print >>sys.stderr, 'Warning: key',val,\
                           'not found in file',filename
                 elif opt['warn'] == 2:
                     raise KeyError,e
                 else:
                     raise ValueError,'Warning level must be 0,1 or 2'
 
     return dict
 
 #----------------------------------------------------------------------------
 def flag_calls(func):
     """Wrap a function to detect and flag when it gets called.
 
     This is a decorator which takes a function and wraps it in a function with
     a 'called' attribute. wrapper.called is initialized to False.
 
     The wrapper.called attribute is set to False right before each call to the
     wrapped function, so if the call fails it remains False.  After the call
     completes, wrapper.called is set to True and the output is returned.
 
     Testing for truth in wrapper.called allows you to determine if a call to
     func() was attempted and succeeded."""
     
     def wrapper(*args,**kw):
         wrapper.called = False
         out = func(*args,**kw)
         wrapper.called = True
         return out
 
     wrapper.called = False
     wrapper.__doc__ = func.__doc__
     return wrapper
 
 #----------------------------------------------------------------------------
 def dhook_wrap(func,*a,**k):
     """Wrap a function call in a sys.displayhook controller.
 
     Returns a wrapper around func which calls func, with all its arguments and
     keywords unmodified, using the default sys.displayhook.  Since IPython
     modifies sys.displayhook, it breaks the behavior of certain systems that
     rely on the default behavior, notably doctest.
     """
 
     def f(*a,**k):
     
         dhook_s = sys.displayhook
         sys.displayhook = sys.__displayhook__
         try:
             out = func(*a,**k)
         finally:
             sys.displayhook = dhook_s
 
         return out
 
     f.__doc__ = func.__doc__
     return f
 
 #----------------------------------------------------------------------------
 def doctest_reload():
     """Properly reload doctest to reuse it interactively.
 
     This routine:
 
       - reloads doctest
       
       - resets its global 'master' attribute to None, so that multiple uses of
       the module interactively don't produce cumulative reports.
     
       - Monkeypatches its core test runner method to protect it from IPython's
       modified displayhook.  Doctest expects the default displayhook behavior
       deep down, so our modification breaks it completely.  For this reason, a
       hard monkeypatch seems like a reasonable solution rather than asking
       users to manually use a different doctest runner when under IPython."""
 
     import doctest
     reload(doctest)
     doctest.master=None
 
     try:
         doctest.DocTestRunner
     except AttributeError:
         # This is only for python 2.3 compatibility, remove once we move to
         # 2.4 only.
         pass
     else:
         doctest.DocTestRunner.run = dhook_wrap(doctest.DocTestRunner.run)
 
 #----------------------------------------------------------------------------
 class HomeDirError(Error):
     pass
 
 def get_home_dir():
     """Return the closest possible equivalent to a 'home' directory.
 
     We first try $HOME.  Absent that, on NT it's $HOMEDRIVE\$HOMEPATH.
 
     Currently only Posix and NT are implemented, a HomeDirError exception is
     raised for all other OSes. """
 
     isdir = os.path.isdir
     env = os.environ
     
     # first, check py2exe distribution root directory for _ipython.
     # This overrides all. Normally does not exist.
     
     if '\\library.zip\\' in IPython.__file__.lower():
         root, rest = IPython.__file__.lower().split('library.zip')
         if isdir(root + '_ipython'):
             os.environ["IPYKITROOT"] = root.rstrip('\\')
             return root
     
     try:
         homedir = env['HOME']
         if not isdir(homedir):
             # in case a user stuck some string which does NOT resolve to a
             # valid path, it's as good as if we hadn't foud it
             raise KeyError
         return homedir
     except KeyError:
         if os.name == 'posix':
             raise HomeDirError,'undefined $HOME, IPython can not proceed.'
         elif os.name == 'nt':
             # For some strange reason, win9x returns 'nt' for os.name.
             try:
                 homedir = os.path.join(env['HOMEDRIVE'],env['HOMEPATH'])
                 if not isdir(homedir):
                     homedir = os.path.join(env['USERPROFILE'])
                     if not isdir(homedir):
                         raise HomeDirError
                 return homedir
             except:
                 try:
                     # Use the registry to get the 'My Documents' folder.
                     import _winreg as wreg
                     key = wreg.OpenKey(wreg.HKEY_CURRENT_USER,
                                        "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
                     homedir = wreg.QueryValueEx(key,'Personal')[0]
                     key.Close()
                     if not isdir(homedir):
                         e = ('Invalid "Personal" folder registry key '
                              'typically "My Documents".\n'
                              'Value: %s\n'
                              'This is not a valid directory on your system.' %
                              homedir)
                         raise HomeDirError(e)
                     return homedir
                 except HomeDirError:
                     raise
                 except:
                     return 'C:\\'
         elif os.name == 'dos':
             # Desperate, may do absurd things in classic MacOS. May work under DOS.
             return 'C:\\'
         else:
             raise HomeDirError,'support for your operating system not implemented.'
 
 #****************************************************************************
 # strings and text
 
 class LSString(str):
     """String derivative with a special access attributes.
 
     These are normal strings, but with the special attributes:
 
         .l (or .list) : value as list (split on newlines).
         .n (or .nlstr): original value (the string itself).
         .s (or .spstr): value as whitespace-separated string.
         .p (or .paths): list of path objects
         
     Any values which require transformations are computed only once and
     cached.
 
     Such strings are very useful to efficiently interact with the shell, which
     typically only understands whitespace-separated options for commands."""
 
     def get_list(self):
         try:
             return self.__list
         except AttributeError:
             self.__list = self.split('\n')
             return self.__list
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = self.replace('\n',' ')
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         return self
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self.split('\n') if os.path.exists(p)]
             return self.__paths
     
     p = paths = property(get_paths)
 
 def print_lsstring(arg):
     """ Prettier (non-repr-like) and more informative printer for LSString """
     print "LSString (.p, .n, .l, .s available). Value:"
     print arg
     
 print_lsstring = result_display.when_type(LSString)(print_lsstring)
 
 #----------------------------------------------------------------------------
 class SList(list):
     """List derivative with a special access attributes.
 
     These are normal lists, but with the special attributes:
 
         .l (or .list) : value as list (the list itself).
         .n (or .nlstr): value as a string, joined on newlines.
         .s (or .spstr): value as a string, joined on spaces.
         .p (or .paths): list of path objects
         
     Any values which require transformations are computed only once and
     cached."""
 
     def get_list(self):
         return self
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = ' '.join(self)
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         try:
             return self.__nlstr
         except AttributeError:
             self.__nlstr = '\n'.join(self)
             return self.__nlstr
 
     n = nlstr = property(get_nlstr)
     
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self if os.path.exists(p)]
             return self.__paths
     
     p = paths = property(get_paths)
 
     def grep(self, pattern, prune = False, field = None):
         """ Return all strings matching 'pattern' (a regex or callable) 
         
         This is case-insensitive. If prune is true, return all items
         NOT matching the pattern.
         
         If field is specified, the match must occur in the specified 
         whitespace-separated field.
                
         Examples::
         
             a.grep( lambda x: x.startswith('C') )
             a.grep('Cha.*log', prune=1)
             a.grep('chm', field=-1)
         """
         
         def match_target(s):
             if field is None:
                 return s
             parts = s.split()
             try:
                 tgt = parts[field]
                 return tgt
             except IndexError:
                 return ""
         
         if isinstance(pattern, basestring):
             pred = lambda x : re.search(pattern, x, re.IGNORECASE)
         else:
             pred = pattern
         if not prune:
             return SList([el for el in self if pred(match_target(el))])
         else:
             return SList([el for el in self if not pred(match_target(el))])
     def fields(self, *fields):
         """ Collect whitespace-separated fields from string list
         
         Allows quick awk-like usage of string lists.
         
         Example data (in var a, created by 'a = !ls -l')::
             -rwxrwxrwx  1 ville None      18 Dec 14  2006 ChangeLog
             drwxrwxrwx+ 6 ville None       0 Oct 24 18:05 IPython        
         
         a.fields(0) is ['-rwxrwxrwx', 'drwxrwxrwx+']
         a.fields(1,0) is ['1 -rwxrwxrwx', '6 drwxrwxrwx+']
         (note the joining by space).
         a.fields(-1) is ['ChangeLog', 'IPython']
         
         IndexErrors are ignored.
         
         Without args, fields() just split()'s the strings.
         """
         if len(fields) == 0:
             return [el.split() for el in self]
         
         res = SList()
         for el in [f.split() for f in self]:
             lineparts = []
 
             for fd in fields:
                 try:
                     lineparts.append(el[fd])
                 except IndexError:
                     pass
             if lineparts:
                 res.append(" ".join(lineparts))
             
         return res        
                 
             
             
         
 
 def print_slist(arg):
     """ Prettier (non-repr-like) and more informative printer for SList """
     print "SList (.p, .n, .l, .s, .grep(), .fields() available). Value:"
     nlprint(arg)
     
 print_slist = result_display.when_type(SList)(print_slist)
 
 
 
 #----------------------------------------------------------------------------
 def esc_quotes(strng):
     """Return the input string with single and double quotes escaped out"""
 
     return strng.replace('"','\\"').replace("'","\\'")
 
 #----------------------------------------------------------------------------
 def make_quoted_expr(s):
     """Return string s in appropriate quotes, using raw string if possible.
     
     Effectively this turns string: cd \ao\ao\
     to: r"cd \ao\ao\_"[:-1]
     
     Note the use of raw string and padding at the end to allow trailing backslash.
     
     """
     
     tail = ''
     tailpadding = ''
     raw  = ''
     if "\\" in s:
         raw = 'r'
         if s.endswith('\\'):
             tail = '[:-1]'
             tailpadding = '_'
     if '"' not in s:
         quote = '"'
     elif "'" not in s:
         quote = "'"
     elif '"""' not in s and not s.endswith('"'):
         quote = '"""'
     elif "'''" not in s and not s.endswith("'"):
         quote = "'''"
     else:
         # give up, backslash-escaped string will do
         return '"%s"' % esc_quotes(s)
     res = raw + quote + s + tailpadding + quote + tail
     return res
 
 
 #----------------------------------------------------------------------------
 def raw_input_multi(header='', ps1='==> ', ps2='..> ',terminate_str = '.'):
     """Take multiple lines of input.
 
     A list with each line of input as a separate element is returned when a
     termination string is entered (defaults to a single '.'). Input can also
     terminate via EOF (^D in Unix, ^Z-RET in Windows).
 
     Lines of input which end in \\ are joined into single entries (and a
     secondary continuation prompt is issued as long as the user terminates
     lines with \\). This allows entering very long strings which are still
     meant to be treated as single entities.
     """
 
     try:
         if header:
             header += '\n'
         lines = [raw_input(header + ps1)]
     except EOFError:
         return []
     terminate = [terminate_str]
     try:
         while lines[-1:] != terminate:
             new_line = raw_input(ps1)
             while new_line.endswith('\\'):
                 new_line = new_line[:-1] + raw_input(ps2)
             lines.append(new_line)
                 
         return lines[:-1]  # don't return the termination command
     except EOFError:
         print
         return lines
 
 #----------------------------------------------------------------------------
 def raw_input_ext(prompt='',  ps2='... '):
     """Similar to raw_input(), but accepts extended lines if input ends with \\."""
 
     line = raw_input(prompt)
     while line.endswith('\\'):
         line = line[:-1] + raw_input(ps2)
     return line
 
 #----------------------------------------------------------------------------
 def ask_yes_no(prompt,default=None):
     """Asks a question and returns a boolean (y/n) answer.
 
     If default is given (one of 'y','n'), it is used if the user input is
     empty. Otherwise the question is repeated until an answer is given.
 
     An EOF is treated as the default answer.  If there is no default, an
     exception is raised to prevent infinite loops.
 
     Valid answers are: y/yes/n/no (match is not case sensitive)."""
 
     answers = {'y':True,'n':False,'yes':True,'no':False}
     ans = None
     while ans not in answers.keys():
         try:
             ans = raw_input(prompt+' ').lower()
             if not ans:  # response was an empty string
                 ans = default
         except KeyboardInterrupt:
             pass
         except EOFError:
             if default in answers.keys():
                 ans = default
                 print
             else:
                 raise
             
     return answers[ans]
 
 #----------------------------------------------------------------------------
 def marquee(txt='',width=78,mark='*'):
     """Return the input string centered in a 'marquee'."""
     if not txt:
         return (mark*width)[:width]
     nmark = (width-len(txt)-2)/len(mark)/2
     if nmark < 0: nmark =0
     marks = mark*nmark
     return '%s %s %s' % (marks,txt,marks)
 
 #----------------------------------------------------------------------------
 class EvalDict:
     """
     Emulate a dict which evaluates its contents in the caller's frame.
 
     Usage:
     >>>number = 19
     >>>text = "python"
     >>>print "%(text.capitalize())s %(number/9.0).1f rules!" % EvalDict()
     """
 
     # This version is due to [e-mail] on c.l.py, and is basically a
     # modified (shorter) version of:
     # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66018 by
     # Skip Montanaro ([e-mail]).
 
     def __getitem__(self, name):
         frame = sys._getframe(1)
         return eval(name, frame.f_globals, frame.f_locals)
 
 EvalString = EvalDict  # for backwards compatibility
 #----------------------------------------------------------------------------
 def qw(words,flat=0,sep=None,maxsplit=-1):
     """Similar to Perl's qw() operator, but with some more options.
 
     qw(words,flat=0,sep=' ',maxsplit=-1) -> words.split(sep,maxsplit)
 
     words can also be a list itself, and with flat=1, the output will be
     recursively flattened. Examples:
 
     >>> qw('1 2')
     ['1', '2']
     >>> qw(['a b','1 2',['m n','p q']])
     [['a', 'b'], ['1', '2'], [['m', 'n'], ['p', 'q']]]
     >>> qw(['a b','1 2',['m n','p q']],flat=1)
     ['a', 'b', '1', '2', 'm', 'n', 'p', 'q']    """
 
     if type(words) in StringTypes:
         return [word.strip() for word in words.split(sep,maxsplit)
                 if word and not word.isspace() ]
     if flat:
         return flatten(map(qw,words,[1]*len(words)))
     return map(qw,words)
 
 #----------------------------------------------------------------------------
 def qwflat(words,sep=None,maxsplit=-1):
     """Calls qw(words) in flat mode. It's just a convenient shorthand."""
     return qw(words,1,sep,maxsplit)
 
 #----------------------------------------------------------------------------
 def qw_lol(indata):
     """qw_lol('a b') -> [['a','b']],
     otherwise it's just a call to qw().
 
     We need this to make sure the modules_some keys *always* end up as a
     list of lists."""
 
     if type(indata) in StringTypes:
         return [qw(indata)]
     else:
         return qw(indata)
 
 #-----------------------------------------------------------------------------
 def list_strings(arg):
     """Always return a list of strings, given a string or list of strings
     as input."""
 
     if type(arg) in StringTypes: return [arg]
     else: return arg
 
 #----------------------------------------------------------------------------
 def grep(pat,list,case=1):
     """Simple minded grep-like function.
     grep(pat,list) returns occurrences of pat in list, None on failure.
 
     It only does simple string matching, with no support for regexps. Use the
     option case=0 for case-insensitive matching."""
 
     # This is pretty crude. At least it should implement copying only references
     # to the original data in case it's big. Now it copies the data for output.
     out=[]
     if case:
         for term in list:
             if term.find(pat)>-1: out.append(term)
     else:
         lpat=pat.lower()
         for term in list:
             if term.lower().find(lpat)>-1: out.append(term)
 
     if len(out): return out
     else: return None
 
 #----------------------------------------------------------------------------
 def dgrep(pat,*opts):
     """Return grep() on dir()+dir(__builtins__).
 
     A very common use of grep() when working interactively."""
 
     return grep(pat,dir(__main__)+dir(__main__.__builtins__),*opts)
 
 #----------------------------------------------------------------------------
 def idgrep(pat):
     """Case-insensitive dgrep()"""
 
     return dgrep(pat,0)
 
 #----------------------------------------------------------------------------
 def igrep(pat,list):
     """Synonym for case-insensitive grep."""
 
     return grep(pat,list,case=0)
 
 #----------------------------------------------------------------------------
 def indent(str,nspaces=4,ntabs=0):
     """Indent a string a given number of spaces or tabstops.
     
     indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
     """
     if str is None:
         return
     ind = '\t'*ntabs+' '*nspaces
     outstr = '%s%s' % (ind,str.replace(os.linesep,os.linesep+ind))
     if outstr.endswith(os.linesep+ind):
         return outstr[:-len(ind)]
     else:
         return outstr
     
 #-----------------------------------------------------------------------------
 def native_line_ends(filename,backup=1):
     """Convert (in-place) a file to line-ends native to the current OS.
 
     If the optional backup argument is given as false, no backup of the
     original file is left.  """
 
     backup_suffixes = {'posix':'~','dos':'.bak','nt':'.bak','mac':'.bak'}
 
     bak_filename = filename + backup_suffixes[os.name]
 
     original = open(filename).read()
     shutil.copy2(filename,bak_filename)
     try:
         new = open(filename,'wb')
         new.write(os.linesep.join(original.splitlines()))
         new.write(os.linesep) # ALWAYS put an eol at the end of the file
         new.close()
     except:
         os.rename(bak_filename,filename)
     if not backup:
         try:
             os.remove(bak_filename)
         except:
             pass
     
 #----------------------------------------------------------------------------
 def get_pager_cmd(pager_cmd = None):
     """Return a pager command.
 
     Makes some attempts at finding an OS-correct one."""
     
     if os.name == 'posix':
         default_pager_cmd = 'less -r'  # -r for color control sequences
     elif os.name in ['nt','dos']:
         default_pager_cmd = 'type'
 
     if pager_cmd is None:
         try:
             pager_cmd = os.environ['PAGER']
         except:
             pager_cmd = default_pager_cmd
     return pager_cmd
 
 #-----------------------------------------------------------------------------
 def get_pager_start(pager,start):
     """Return the string for paging files with an offset.
 
     This is the '+N' argument which less and more (under Unix) accept.
     """
 
     if pager in ['less','more']:
         if start:
             start_string = '+' + str(start)
         else:
             start_string = ''
     else:
         start_string = ''
     return start_string
 
 #----------------------------------------------------------------------------
 # (X)emacs on W32 doesn't like to be bypassed with msvcrt.getch()
 if os.name == 'nt' and os.environ.get('TERM','dumb') != 'emacs':
     import msvcrt
     def page_more():
         """ Smart pausing between pages
 
         @return:    True if need print more lines, False if quit
         """
         Term.cout.write('---Return to continue, q to quit--- ')
         ans = msvcrt.getch()
         if ans in ("q", "Q"):
             result = False
         else:
             result = True
         Term.cout.write("\b"*37 + " "*37 + "\b"*37)
         return result
 else:
     def page_more():
         ans = raw_input('---Return to continue, q to quit--- ')
         if ans.lower().startswith('q'):
             return False
         else:
             return True
 
 esc_re = re.compile(r"(\x1b[^m]+m)")
 
 def page_dumb(strng,start=0,screen_lines=25):
     """Very dumb 'pager' in Python, for when nothing else works.
 
     Only moves forward, same interface as page(), except for pager_cmd and
     mode."""
 
     out_ln  = strng.splitlines()[start:]
     screens = chop(out_ln,screen_lines-1)
     if len(screens) == 1:
         print >>Term.cout, os.linesep.join(screens[0])
     else:
         last_escape = ""
         for scr in screens[0:-1]:
             hunk = os.linesep.join(scr)
             print >>Term.cout, last_escape + hunk
             if not page_more():
                 return
             esc_list = esc_re.findall(hunk)
             if len(esc_list) > 0:
                 last_escape = esc_list[-1]
         print >>Term.cout, last_escape + os.linesep.join(screens[-1])
 
 #----------------------------------------------------------------------------
 def page(strng,start=0,screen_lines=0,pager_cmd = None):
     """Print a string, piping through a pager after a certain length.
 
     The screen_lines parameter specifies the number of *usable* lines of your
     terminal screen (total lines minus lines you need to reserve to show other
     information).
 
     If you set screen_lines to a number <=0, page() will try to auto-determine
     your screen size and will only use up to (screen_size+screen_lines) for
     printing, paging after that. That is, if you want auto-detection but need
     to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for
     auto-detection without any lines reserved simply use screen_lines = 0.
 
     If a string won't fit in the allowed lines, it is sent through the
     specified pager command. If none given, look for PAGER in the environment,
     and ultimately default to less.
 
     If no system pager works, the string is sent through a 'dumb pager'
     written in python, very simplistic.
     """
 
     # Some routines may auto-compute start offsets incorrectly and pass a
     # negative value.  Offset to 0 for robustness.
     start = max(0,start)
 
     # first, try the hook
     ip = IPython.ipapi.get()
     if ip:
         try:
             ip.IP.hooks.show_in_pager(strng)
             return
         except IPython.ipapi.TryNext:
             pass
         
     # Ugly kludge, but calling curses.initscr() flat out crashes in emacs
     TERM = os.environ.get('TERM','dumb')
     if TERM in ['dumb','emacs'] and os.name != 'nt':
         print strng
         return
     # chop off the topmost part of the string we don't want to see
     str_lines = strng.split(os.linesep)[start:]
     str_toprint = os.linesep.join(str_lines)
     num_newlines = len(str_lines)
     len_str = len(str_toprint)
     
     # Dumb heuristics to guesstimate number of on-screen lines the string
     # takes.  Very basic, but good enough for docstrings in reasonable
     # terminals. If someone later feels like refining it, it's not hard.
     numlines = max(num_newlines,int(len_str/80)+1)
 
     if os.name == "nt":
         screen_lines_def = get_console_size(defaulty=25)[1]
     else:
         screen_lines_def = 25 # default value if we can't auto-determine
 
     # auto-determine screen size
     if screen_lines <= 0:
         if TERM=='xterm':
             use_curses = USE_CURSES
         else:
             # curses causes problems on many terminals other than xterm.
             use_curses = False
         if use_curses:
             # There is a bug in curses, where *sometimes* it fails to properly
             # initialize, and then after the endwin() call is made, the
             # terminal is left in an unusable state.  Rather than trying to
             # check everytime for this (by requesting and comparing termios
             # flags each time), we just save the initial terminal state and
             # unconditionally reset it every time.  It's cheaper than making
             # the checks.
             term_flags = termios.tcgetattr(sys.stdout)
             scr = curses.initscr()
             screen_lines_real,screen_cols = scr.getmaxyx()
             curses.endwin()
             # Restore terminal state in case endwin() didn't.
             termios.tcsetattr(sys.stdout,termios.TCSANOW,term_flags)
             # Now we have what we needed: the screen size in rows/columns
             screen_lines += screen_lines_real
             #print '***Screen size:',screen_lines_real,'lines x',\
             #screen_cols,'columns.' # dbg
         else:
             screen_lines += screen_lines_def
 
     #print 'numlines',numlines,'screenlines',screen_lines  # dbg
     if numlines <= screen_lines :
         #print '*** normal print'  # dbg
         print >>Term.cout, str_toprint
     else:
         # Try to open pager and default to internal one if that fails.
         # All failure modes are tagged as 'retval=1', to match the return
         # value of a failed system command.  If any intermediate attempt
         # sets retval to 1, at the end we resort to our own page_dumb() pager.
         pager_cmd = get_pager_cmd(pager_cmd)
         pager_cmd += ' ' + get_pager_start(pager_cmd,start)
         if os.name == 'nt':
             if pager_cmd.startswith('type'):
                 # The default WinXP 'type' command is failing on complex strings.
                 retval = 1
             else:
                 tmpname = tempfile.mktemp('.txt')
                 tmpfile = file(tmpname,'wt')
                 tmpfile.write(strng)
                 tmpfile.close()
                 cmd = "%s < %s" % (pager_cmd,tmpname)
                 if os.system(cmd):
                   retval = 1
                 else:
                   retval = None
                 os.remove(tmpname)
         else:
             try:
                 retval = None
                 # if I use popen4, things hang. No idea why.
                 #pager,shell_out = os.popen4(pager_cmd)
                 pager = os.popen(pager_cmd,'w')
                 pager.write(strng)
                 pager.close()
                 retval = pager.close()  # success returns None
             except IOError,msg:  # broken pipe when user quits
                 if msg.args == (32,'Broken pipe'):
                     retval = None
                 else:
                     retval = 1
             except OSError:
                 # Other strange problems, sometimes seen in Win2k/cygwin
                 retval = 1
         if retval is not None:
             page_dumb(strng,screen_lines=screen_lines)
 
 #----------------------------------------------------------------------------
 def page_file(fname,start = 0, pager_cmd = None):
     """Page a file, using an optional pager command and starting line.
     """
 
     pager_cmd = get_pager_cmd(pager_cmd)
     pager_cmd += ' ' + get_pager_start(pager_cmd,start)
 
     try:
         if os.environ['TERM'] in ['emacs','dumb']:
             raise EnvironmentError
         xsys(pager_cmd + ' ' + fname)
     except:
         try:
             if start > 0:
                 start -= 1
             page(open(fname).read(),start)
         except:
             print 'Unable to show file',`fname`
 
 
 #----------------------------------------------------------------------------
 def snip_print(str,width = 75,print_full = 0,header = ''):
     """Print a string snipping the midsection to fit in width.
 
     print_full: mode control:
       - 0: only snip long strings
       - 1: send to page() directly.
       - 2: snip long strings and ask for full length viewing with page()
     Return 1 if snipping was necessary, 0 otherwise."""
 
     if print_full == 1:
         page(header+str)
         return 0
 
     print header,
     if len(str) < width:
         print str
         snip = 0
     else:
         whalf = int((width -5)/2)
         print str[:whalf] + ' <...> ' + str[-whalf:]
         snip = 1
     if snip and print_full == 2:
         if raw_input(header+' Snipped. View (y/n)? [N]').lower() == 'y':
             page(str)
     return snip
 
 #****************************************************************************
 # lists, dicts and structures
 
 def belong(candidates,checklist):
     """Check whether a list of items appear in a given list of options.
 
     Returns a list of 1 and 0, one for each candidate given."""
 
     return [x in checklist for x in candidates]
 
 #----------------------------------------------------------------------------
 def uniq_stable(elems):
     """uniq_stable(elems) -> list
 
     Return from an iterable, a list of all the unique elements in the input,
     but maintaining the order in which they first appear.
 
     A naive solution to this problem which just makes a dictionary with the
     elements as keys fails to respect the stability condition, since
     dictionaries are unsorted by nature.
 
     Note: All elements in the input must be valid dictionary keys for this
     routine to work, as it internally uses a dictionary for efficiency
     reasons."""
     
     unique = []
     unique_dict = {}
     for nn in elems:
         if nn not in unique_dict:
             unique.append(nn)
             unique_dict[nn] = None
     return unique
 
 #----------------------------------------------------------------------------
 class NLprinter:
     """Print an arbitrarily nested list, indicating index numbers.
 
     An instance of this class called nlprint is available and callable as a
     function.
     
     nlprint(list,indent=' ',sep=': ') -> prints indenting each level by 'indent'
     and using 'sep' to separate the index from the value. """
 
     def __init__(self):
         self.depth = 0
     
     def __call__(self,lst,pos='',**kw):
         """Prints the nested list numbering levels."""
         kw.setdefault('indent',' ')
         kw.setdefault('sep',': ')
         kw.setdefault('start',0)
         kw.setdefault('stop',len(lst))
         # we need to remove start and stop from kw so they don't propagate
         # into a recursive call for a nested list.
         start = kw['start']; del kw['start']
         stop = kw['stop']; del kw['stop']
         if self.depth == 0 and 'header' in kw.keys():
             print kw['header']
             
         for idx in range(start,stop):
             elem = lst[idx]
             if type(elem)==type([]):
                 self.depth += 1
                 self.__call__(elem,itpl('$pos$idx,'),**kw)
                 self.depth -= 1
             else:
                 printpl(kw['indent']*self.depth+'$pos$idx$kw["sep"]$elem')
 
 nlprint = NLprinter()
 #----------------------------------------------------------------------------
 def all_belong(candidates,checklist):
     """Check whether a list of items ALL appear in a given list of options.
 
     Returns a single 1 or 0 value."""
 
     return 1-(0 in [x in checklist for x in candidates])
 
 #----------------------------------------------------------------------------
 def sort_compare(lst1,lst2,inplace = 1):
     """Sort and compare two lists.
 
     By default it does it in place, thus modifying the lists. Use inplace = 0
     to avoid that (at the cost of temporary copy creation)."""
     if not inplace:
         lst1 = lst1[:]
         lst2 = lst2[:]
     lst1.sort(); lst2.sort()
     return lst1 == lst2
 
 #----------------------------------------------------------------------------
 def mkdict(**kwargs):
     """Return a dict from a keyword list.
 
     It's just syntactic sugar for making ditcionary creation more convenient:
     # the standard way
     >>>data = { 'red' : 1, 'green' : 2, 'blue' : 3 }
     # a cleaner way
     >>>data = dict(red=1, green=2, blue=3)
 
     If you need more than this, look at the Struct() class."""
 
     return kwargs
 
 #----------------------------------------------------------------------------
 def list2dict(lst):
     """Takes a list of (key,value) pairs and turns it into a dict."""
 
     dic = {}
     for k,v in lst: dic[k] = v
     return dic
 
 #----------------------------------------------------------------------------
 def list2dict2(lst,default=''):
     """Takes a list and turns it into a dict.
     Much slower than list2dict, but more versatile. This version can take
     lists with sublists of arbitrary length (including sclars)."""
 
     dic = {}
     for elem in lst:
         if type(elem) in (types.ListType,types.TupleType):
             size = len(elem)
             if  size == 0:
                 pass
             elif size == 1:
                 dic[elem] = default
             else:
                 k,v = elem[0], elem[1:]
                 if len(v) == 1: v = v[0]
                 dic[k] = v
         else:
             dic[elem] = default
     return dic
 
 #----------------------------------------------------------------------------
 def flatten(seq):
     """Flatten a list of lists (NOT recursive, only works for 2d lists)."""
 
     return [x for subseq in seq for x in subseq]
 
 #----------------------------------------------------------------------------
 def get_slice(seq,start=0,stop=None,step=1):
     """Get a slice of a sequence with variable step. Specify start,stop,step."""
     if stop == None:
         stop = len(seq)
     item = lambda i: seq[i]
     return map(item,xrange(start,stop,step))
 
 #----------------------------------------------------------------------------
 def chop(seq,size):
     """Chop a sequence into chunks of the given size."""
     chunk = lambda i: seq[i:i+size]
     return map(chunk,xrange(0,len(seq),size))
 
 #----------------------------------------------------------------------------
 # with is a keyword as of python 2.5, so this function is renamed to withobj
 # from its old 'with' name.
 def with_obj(object, **args):
     """Set multiple attributes for an object, similar to Pascal's with.
 
     Example:
     with_obj(jim,
              born = 1960,
              haircolour = 'Brown',
              eyecolour = 'Green')
 
     Credit: Greg Ewing, in
     http://mail.python.org/pipermail/python-list/2001-May/040703.html.
 
     NOTE: up until IPython 0.7.2, this was called simply 'with', but 'with'
     has become a keyword for Python 2.5, so we had to rename it."""
 
     object.__dict__.update(args)
 
 #----------------------------------------------------------------------------
 def setattr_list(obj,alist,nspace = None):
     """Set a list of attributes for an object taken from a namespace.
 
     setattr_list(obj,alist,nspace) -> sets in obj all the attributes listed in
     alist with their values taken from nspace, which must be a dict (something
     like locals() will often do) If nspace isn't given, locals() of the
     *caller* is used, so in most cases you can omit it.
 
     Note that alist can be given as a string, which will be automatically
     split into a list on whitespace. If given as a list, it must be a list of
     *strings* (the variable names themselves), not of variables."""
 
     # this grabs the local variables from the *previous* call frame -- that is
     # the locals from the function that called setattr_list().
     # - snipped from weave.inline()
     if nspace is None:
         call_frame = sys._getframe().f_back
         nspace = call_frame.f_locals
 
     if type(alist) in StringTypes:
         alist = alist.split()
     for attr in alist:
         val = eval(attr,nspace)
         setattr(obj,attr,val)
 
 #----------------------------------------------------------------------------
 def getattr_list(obj,alist,*args):
     """getattr_list(obj,alist[, default]) -> attribute list.
 
     Get a list of named attributes for an object. When a default argument is
     given, it is returned when the attribute doesn't exist; without it, an
     exception is raised in that case.
 
     Note that alist can be given as a string, which will be automatically
     split into a list on whitespace. If given as a list, it must be a list of
     *strings* (the variable names themselves), not of variables."""
 
     if type(alist) in StringTypes:
         alist = alist.split()
     if args:
         if len(args)==1:
             default = args[0]
             return map(lambda attr: getattr(obj,attr,default),alist)
         else:
             raise ValueError,'getattr_list() takes only one optional argument'
     else:
         return map(lambda attr: getattr(obj,attr),alist)
     
 #----------------------------------------------------------------------------
 def map_method(method,object_list,*argseq,**kw):
     """map_method(method,object_list,*args,**kw) -> list
 
     Return a list of the results of applying the methods to the items of the
     argument sequence(s).  If more than one sequence is given, the method is
     called with an argument list consisting of the corresponding item of each
     sequence. All sequences must be of the same length.
 
     Keyword arguments are passed verbatim to all objects called.
 
     This is Python code, so it's not nearly as fast as the builtin map()."""
 
     out_list = []
     idx = 0
     for object in object_list:
         try:
             handler = getattr(object, method)
         except AttributeError:
             out_list.append(None)
         else:
             if argseq:
                 args = map(lambda lst:lst[idx],argseq)
                 #print 'ob',object,'hand',handler,'ar',args # dbg
                 out_list.append(handler(args,**kw))
             else:
                 out_list.append(handler(**kw))
         idx += 1
     return out_list
 
 #----------------------------------------------------------------------------
 def get_class_members(cls):
     ret = dir(cls)
     if hasattr(cls,'__bases__'):
         for base in cls.__bases__:
             ret.extend(get_class_members(base))
     return ret
 
 #----------------------------------------------------------------------------
 def dir2(obj):
     """dir2(obj) -> list of strings
 
     Extended version of the Python builtin dir(), which does a few extra
     checks, and supports common objects with unusual internals that confuse
     dir(), such as Traits and PyCrust.
 
     This version is guaranteed to return only a list of true strings, whereas
     dir() returns anything that objects inject into themselves, even if they
     are later not really valid for attribute access (many extension libraries
     have such bugs).
     """
     
     # Start building the attribute list via dir(), and then complete it
     # with a few extra special-purpose calls.
     words = dir(obj)
 
     if hasattr(obj,'__class__'):
         words.append('__class__')
         words.extend(get_class_members(obj.__class__))
     #if '__base__' in words: 1/0
 
     # Some libraries (such as traits) may introduce duplicates, we want to
     # track and clean this up if it happens
     may_have_dupes = False
 
     # this is the 'dir' function for objects with Enthought's traits
     if hasattr(obj, 'trait_names'):
         try:
             words.extend(obj.trait_names())
             may_have_dupes = True
         except TypeError:
             # This will happen if `obj` is a class and not an instance.
             pass
 
     # Support for PyCrust-style _getAttributeNames magic method.
     if hasattr(obj, '_getAttributeNames'):
         try:
             words.extend(obj._getAttributeNames())
             may_have_dupes = True
         except TypeError:
             # `obj` is a class and not an instance.  Ignore
             # this error.
             pass
 
     if may_have_dupes:
         # eliminate possible duplicates, as some traits may also
         # appear as normal attributes in the dir() call.
         words = list(set(words))
         words.sort()
 
     # filter out non-string attributes which may be stuffed by dir() calls
     # and poor coding in third-party modules
     return [w for w in words if isinstance(w, basestring)]
 
 #----------------------------------------------------------------------------
 def import_fail_info(mod_name,fns=None):
     """Inform load failure for a module."""
 
     if fns == None:
         warn("Loading of %s failed.\n" % (mod_name,))
     else:
         warn("Loading of %s from %s failed.\n" % (fns,mod_name))
 
 #----------------------------------------------------------------------------
 # Proposed popitem() extension, written as a method
 
 
 class NotGiven: pass
 
 def popkey(dct,key,default=NotGiven):
     """Return dct[key] and delete dct[key].
 
     If default is given, return it if dct[key] doesn't exist, otherwise raise
     KeyError.  """
 
     try:
         val = dct[key]
     except KeyError:
         if default is NotGiven:
             raise
         else:
             return default
     else:
         del dct[key]
         return val
 
 def wrap_deprecated(func, suggest = '<nothing>'):
     def newFunc(*args, **kwargs):
         warnings.warn("Call to deprecated function %s, use %s instead" % 
                       ( func.__name__, suggest),
                       category=DeprecationWarning,
                       stacklevel = 2)
         return func(*args, **kwargs)
     return newFunc
-    
-#*************************** end of file <genutils.py> **********************
 
+
+def _num_cpus_unix():
+    """Return the number of active CPUs on a Unix system."""
+    return os.sysconf("SC_NPROCESSORS_ONLN")
+
+
+def _num_cpus_darwin():
+    """Return the number of active CPUs on a Darwin system."""
+    p = subprocess.Popen(['sysctl','-n','hw.ncpu'],stdout=subprocess.PIPE)
+    return p.stdout.read()
+
+
+def _num_cpus_windows():
+    """Return the number of active CPUs on a Windows system."""
+    return os.environ.get("NUMBER_OF_PROCESSORS")
+
+
+def num_cpus():
+   """Return the effective number of CPUs in the system as an integer.
+
+   This cross-platform function makes an attempt at finding the total number of
+   available CPUs in the system, as returned by various underlying system and
+   python calls.
+
+   If it can't find a sensible answer, it returns 1 (though an error *may* make
+   it return a large positive number that's actually incorrect).
+   """
+   
+   # Many thanks to the Parallel Python project (http://www.parallelpython.com)
+   # for the names of the keys we needed to look up for this function.  This
+   # code was inspired by their equivalent function.
+
+   ncpufuncs = {'Linux':_num_cpus_unix,
+                'Darwin':_num_cpus_darwin,
+                'Windows':_num_cpus_windows,
+                # On Vista, python < 2.5.2 has a bug and returns 'Microsoft'
+                # See http://bugs.python.org/issue1082 for details.
+                'Microsoft':_num_cpus_windows,
+                }
+
+   ncpufunc = ncpufuncs.get(platform.system(),
+                            # default to unix version (Solaris, AIX, etc)
+                            _num_cpus_unix)
+
+   try:
+       ncpus = max(1,int(ncpufunc()))
+   except:
+       ncpus = 1
+   return ncpus
+
+#*************************** end of file <genutils.py> **********************

IPython/ipstruct.py

@@ -1,394 +1,406 b''
 # -*- coding: utf-8 -*-
 """Mimic C structs with lots of extra functionality.
 
 $Id: ipstruct.py 1950 2006-11-28 19:15:35Z vivainio $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2004 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Fernando']
 __license__ = Release.license
 
 __all__ = ['Struct']
 
 import types
 import pprint
 
 from IPython.genutils import list2dict2
 
 class Struct:
     """Class to mimic C structs but also provide convenient dictionary-like
     functionality.
 
     Instances can be initialized with a dictionary, a list of key=value pairs
     or both. If both are present, the dictionary must come first.
 
     Because Python classes provide direct assignment to their members, it's
     easy to overwrite normal methods (S.copy = 1 would destroy access to
     S.copy()). For this reason, all builtin method names are protected and
     can't be assigned to. An attempt to do s.copy=1 or s['copy']=1 will raise
     a KeyError exception. If you really want to, you can bypass this
     protection by directly assigning to __dict__: s.__dict__['copy']=1 will
     still work. Doing this will break functionality, though. As in most of
     Python, namespace protection is weakly enforced, so feel free to shoot
     yourself if you really want to.
 
     Note that this class uses more memory and is *much* slower than a regular
     dictionary, so be careful in situations where memory or performance are
     critical. But for day to day use it should behave fine. It is particularly
     convenient for storing configuration data in programs.
 
     +,+=,- and -= are implemented. +/+= do merges (non-destructive updates),
     -/-= remove keys from the original. See the method descripitions.
 
     This class allows a quick access syntax: both s.key and s['key'] are
     valid.  This syntax has a limitation: each 'key' has to be explicitly
     accessed by its original name. The normal s.key syntax doesn't provide
     access to the keys via variables whose values evaluate to the desired
     keys. An example should clarify this:
 
     Define a dictionary and initialize both with dict and k=v pairs:
     >>> d={'a':1,'b':2}
     >>> s=Struct(d,hi=10,ho=20)
+
     The return of __repr__ can be used to create a new instance:
     >>> s
-    Struct({'ho': 20, 'b': 2, 'hi': 10, 'a': 1})
+    Struct({'__allownew': True, 'a': 1, 'b': 2, 'hi': 10, 'ho': 20})
+
+    Note: the special '__allownew' key is used for internal purposes.
+    
     __str__ (called by print) shows it's not quite a regular dictionary:
     >>> print s
-    Struct {a: 1, b: 2, hi: 10, ho: 20}
+    Struct({'__allownew': True, 'a': 1, 'b': 2, 'hi': 10, 'ho': 20})
+
     Access by explicitly named key with dot notation:
     >>> s.a
     1
+
     Or like a dictionary:
     >>> s['a']
     1
+
     If you want a variable to hold the key value, only dictionary access works:
     >>> key='hi'
     >>> s.key
     Traceback (most recent call last):
       File "<stdin>", line 1, in ?
     AttributeError: Struct instance has no attribute 'key'
+
     >>> s[key]
     10
 
     Another limitation of the s.key syntax (and Struct(key=val)
     initialization): keys can't be numbers. But numeric keys can be used and
     accessed using the dictionary syntax. Again, an example:
 
     This doesn't work:
-    >>> s=Struct(4='hi')
+    >>> s=Struct(4='hi')  #doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
     SyntaxError: keyword can't be an expression
+
     But this does:
     >>> s=Struct()
     >>> s[4]='hi'
     >>> s
-    Struct({4: 'hi'})
+    Struct({4: 'hi', '__allownew': True})
     >>> s[4]
     'hi'
     """
 
     # Attributes to which __setitem__ and __setattr__ will block access.
     # Note: much of this will be moot in Python 2.2 and will be done in a much
     # cleaner way.
     __protected = ('copy dict dictcopy get has_attr has_key items keys '
                    'merge popitem setdefault update values '
                    '__make_dict __dict_invert ').split()
 
     def __init__(self,dict=None,**kw):
         """Initialize with a dictionary, another Struct, or by giving
         explicitly the list of attributes.
 
         Both can be used, but the dictionary must come first:
         Struct(dict), Struct(k1=v1,k2=v2) or Struct(dict,k1=v1,k2=v2).
         """
         self.__dict__['__allownew'] = True
         if dict is None:
             dict = {}
         if isinstance(dict,Struct):
             dict = dict.dict()
         elif dict and  type(dict) is not types.DictType:
             raise TypeError,\
                   'Initialize with a dictionary or key=val pairs.'
         dict.update(kw)
         # do the updating by hand to guarantee that we go through the
         # safety-checked __setitem__
         for k,v in dict.items():
             self[k] = v
         
 
     def __setitem__(self,key,value):
         """Used when struct[key] = val calls are made."""
         if key in Struct.__protected:
             raise KeyError,'Key '+`key`+' is a protected key of class Struct.'
         if not self['__allownew'] and key not in self.__dict__:
             raise KeyError(
             "Can't create unknown attribute %s - Check for typos, or use allow_new_attr to create new attributes!" %
             key)
             
         self.__dict__[key] = value
 
     def __setattr__(self, key, value):
         """Used when struct.key = val calls are made."""
         self.__setitem__(key,value)
 
     def __str__(self):
         """Gets called by print."""
         
         return 'Struct('+ pprint.pformat(self.__dict__)+')'
 
     def __repr__(self):
         """Gets called by repr.
         
         A Struct can be recreated with S_new=eval(repr(S_old))."""
         return self.__str__()
 
     def __getitem__(self,key):
         """Allows struct[key] access."""
         return self.__dict__[key]
 
     def __contains__(self,key):
         """Allows use of the 'in' operator."""
         return self.__dict__.has_key(key)
 
     def __iadd__(self,other):
         """S += S2 is a shorthand for S.merge(S2)."""
         self.merge(other)
         return self
 
     def __add__(self,other):
         """S + S2 -> New Struct made form S and S.merge(S2)"""
         Sout = self.copy()
         Sout.merge(other)
         return Sout
 
     def __sub__(self,other):
         """Return S1-S2, where all keys in S2 have been deleted (if present)
         from S1."""
         Sout = self.copy()
         Sout -= other
         return Sout
 
     def __isub__(self,other):
         """Do in place S = S - S2, meaning all keys in S2 have been deleted
         (if present) from S1."""
 
         for k in other.keys():
             if self.has_key(k):
                 del self.__dict__[k]
 
     def __make_dict(self,__loc_data__,**kw):
         "Helper function for update and merge. Return a dict from data."
 
         if __loc_data__ == None:
             dict = {}
         elif type(__loc_data__) is types.DictType:
             dict = __loc_data__
         elif isinstance(__loc_data__,Struct):
             dict = __loc_data__.__dict__
         else:
             raise TypeError, 'Update with a dict, a Struct or key=val pairs.'
         if kw:
             dict.update(kw)
         return dict
 
     def __dict_invert(self,dict):
         """Helper function for merge. Takes a dictionary whose values are
         lists and returns a dict. with the elements of each list as keys and
         the original keys as values."""
 
         outdict = {}
         for k,lst in dict.items():
             if type(lst) is types.StringType:
                 lst = lst.split()
             for entry in lst:
                 outdict[entry] = k
         return outdict
 
     def clear(self):
         """Clear all attributes."""
         self.__dict__.clear()
 
     def copy(self):
         """Return a (shallow) copy of a Struct."""
         return Struct(self.__dict__.copy())
 
     def dict(self):
         """Return the Struct's dictionary."""
         return self.__dict__
 
     def dictcopy(self):
         """Return a (shallow) copy of the Struct's dictionary."""
         return self.__dict__.copy()
 
     def popitem(self):
         """S.popitem() -> (k, v), remove and return some (key, value) pair as
         a 2-tuple; but raise KeyError if S is empty."""
         return self.__dict__.popitem()
 
     def update(self,__loc_data__=None,**kw):
         """Update (merge) with data from another Struct or from a dictionary.
         Optionally, one or more key=value pairs can be given at the end for
         direct update."""
 
         # The funny name __loc_data__ is to prevent a common variable name which
         # could be a fieled of a Struct to collide with this parameter. The problem
         # would arise if the function is called with a keyword with this same name
         # that a user means to add as a Struct field.
         newdict = Struct.__make_dict(self,__loc_data__,**kw)
         for k,v in newdict.items():
             self[k] = v
 
     def merge(self,__loc_data__=None,__conflict_solve=None,**kw):
         """S.merge(data,conflict,k=v1,k=v2,...) -> merge data and k=v into S.
 
         This is similar to update(), but much more flexible.  First, a dict is
         made from data+key=value pairs. When merging this dict with the Struct
         S, the optional dictionary 'conflict' is used to decide what to do.
 
         If conflict is not given, the default behavior is to preserve any keys
         with their current value (the opposite of the update method's
         behavior).
 
         conflict is a dictionary of binary functions which will be used to
         solve key conflicts. It must have the following structure:
 
           conflict == { fn1 : [Skey1,Skey2,...], fn2 : [Skey3], etc }
 
         Values must be lists or whitespace separated strings which are
         automatically converted to lists of strings by calling string.split().
 
         Each key of conflict is a function which defines a policy for
         resolving conflicts when merging with the input data. Each fn must be
         a binary function which returns the desired outcome for a key
         conflict. These functions will be called as fn(old,new).
 
         An example is probably in order. Suppose you are merging the struct S
         with a dict D and the following conflict policy dict:
 
             S.merge(D,{fn1:['a','b',4], fn2:'key_c key_d'})
 
         If the key 'a' is found in both S and D, the merge method will call:
 
             S['a'] = fn1(S['a'],D['a'])
 
         As a convenience, merge() provides five (the most commonly needed)
         pre-defined policies: preserve, update, add, add_flip and add_s. The
         easiest explanation is their implementation:
 
           preserve = lambda old,new: old
           update   = lambda old,new: new
           add      = lambda old,new: old + new
           add_flip = lambda old,new: new + old  # note change of order!
           add_s    = lambda old,new: old + ' ' + new  # only works for strings!
 
         You can use those four words (as strings) as keys in conflict instead
         of defining them as functions, and the merge method will substitute
         the appropriate functions for you. That is, the call
 
           S.merge(D,{'preserve':'a b c','add':[4,5,'d'],my_function:[6]})
 
         will automatically substitute the functions preserve and add for the
         names 'preserve' and 'add' before making any function calls.
 
         For more complicated conflict resolution policies, you still need to
         construct your own functions. """
 
         data_dict = Struct.__make_dict(self,__loc_data__,**kw)
 
         # policies for conflict resolution: two argument functions which return
         # the value that will go in the new struct
         preserve = lambda old,new: old
         update   = lambda old,new: new
         add      = lambda old,new: old + new
         add_flip = lambda old,new: new + old  # note change of order!
         add_s    = lambda old,new: old + ' ' + new
 
         # default policy is to keep current keys when there's a conflict
         conflict_solve = list2dict2(self.keys(),default = preserve)
 
         # the conflict_solve dictionary is given by the user 'inverted': we
         # need a name-function mapping, it comes as a function -> names
         # dict. Make a local copy (b/c we'll make changes), replace user
         # strings for the three builtin policies and invert it.
         if __conflict_solve:
             inv_conflict_solve_user = __conflict_solve.copy()
             for name, func in [('preserve',preserve), ('update',update),
-                               ('add',add), ('add_flip',add_flip), ('add_s',add_s)]:
+                               ('add',add), ('add_flip',add_flip),
+                               ('add_s',add_s)]:
                 if name in inv_conflict_solve_user.keys():
                     inv_conflict_solve_user[func] = inv_conflict_solve_user[name]
                     del inv_conflict_solve_user[name]
             conflict_solve.update(Struct.__dict_invert(self,inv_conflict_solve_user))
         #print 'merge. conflict_solve: '; pprint(conflict_solve) # dbg
         #print '*'*50,'in merger. conflict_solver:';  pprint(conflict_solve)
         for key in data_dict:
             if key not in self:
                 self[key] = data_dict[key]
             else:
                 self[key] = conflict_solve[key](self[key],data_dict[key])
 
     def has_key(self,key):
         """Like has_key() dictionary method."""
         return self.__dict__.has_key(key)
 
     def hasattr(self,key):
         """hasattr function available as a method.
 
         Implemented like has_key, to make sure that all available keys in the
         internal dictionary of the Struct appear also as attributes (even
         numeric keys)."""
         return self.__dict__.has_key(key)
 
     def items(self):
         """Return the items in the Struct's dictionary, in the same format
         as a call to {}.items()."""
         return self.__dict__.items()
 
     def keys(self):
         """Return the keys in the Struct's dictionary, in the same format
         as a call to {}.keys()."""
         return self.__dict__.keys()
 
     def values(self,keys=None):
         """Return the values in the Struct's dictionary, in the same format
         as a call to {}.values().
 
         Can be called with an optional argument keys, which must be a list or
         tuple of keys. In this case it returns only the values corresponding
         to those keys (allowing a form of 'slicing' for Structs)."""
         if not keys:
             return self.__dict__.values()
         else:
             ret=[]
             for k in keys:
                 ret.append(self[k])
             return ret
 
     def get(self,attr,val=None):
-        """S.get(k[,d]) -> S[k] if S.has_key(k), else d.  d defaults to None."""
+        """S.get(k[,d]) -> S[k] if k in S, else d.  d defaults to None."""
         try:
             return self[attr]
         except KeyError:
             return val
 
     def setdefault(self,attr,val=None):
-        """S.setdefault(k[,d]) -> S.get(k,d), also set S[k]=d if not S.has_key(k)"""
+        """S.setdefault(k[,d]) -> S.get(k,d), also set S[k]=d if k not in S"""
         if not self.has_key(attr):
             self[attr] = val
         return self.get(attr,val)
     
     def allow_new_attr(self, allow = True):
         """ Set whether new attributes can be created inside struct
         
-        This can be used to catch typos by verifying that the attribute user tries to 
-        change already exists in this Struct.
+        This can be used to catch typos by verifying that the attribute user
+        tries to change already exists in this Struct.
         """
         self['__allownew'] = allow
         
         
 # end class Struct
 

IPython/kernel/engineservice.py

@@ -1,874 +1,876 b''
 # encoding: utf-8
 # -*- test-case-name: IPython.kernel.tests.test_engineservice -*-
 
 """A Twisted Service Representation of the IPython core.
 
 The IPython Core exposed to the network is called the Engine.  Its
 representation in Twisted in the EngineService.  Interfaces and adapters
 are used to abstract out the details of the actual network protocol used.
 The EngineService is an Engine that knows nothing about the actual protocol
 used.
 
 The EngineService is exposed with various network protocols in modules like:
 
 enginepb.py
 enginevanilla.py
 
 As of 12/12/06 the classes in this module have been simplified greatly.  It was 
 felt that we had over-engineered things.  To improve the maintainability of the
 code we have taken out the ICompleteEngine interface and the completeEngine
 method that automatically added methods to engines.
 
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import os, sys, copy
 import cPickle as pickle
 from new import instancemethod
 
 from twisted.application import service
 from twisted.internet import defer, reactor
 from twisted.python import log, failure, components
 import zope.interface as zi
 
 from IPython.kernel.core.interpreter import Interpreter
 from IPython.kernel import newserialized, error, util
 from IPython.kernel.util import printer
 from IPython.kernel.twistedutil import gatherBoth, DeferredList
 from IPython.kernel import codeutil
 
 
 #-------------------------------------------------------------------------------
 # Interface specification for the Engine
 #-------------------------------------------------------------------------------
 
 class IEngineCore(zi.Interface):
     """The minimal required interface for the IPython Engine.
     
     This interface provides a formal specification of the IPython core.
     All these methods should return deferreds regardless of what side of a
     network connection they are on.
     
     In general, this class simply wraps a shell class and wraps its return
     values as Deferred objects.  If the underlying shell class method raises
     an exception, this class should convert it to a twisted.failure.Failure
     that will be propagated along the Deferred's errback chain.
     
     In addition, Failures are aggressive.  By this, we mean that if a method
     is performing multiple actions (like pulling multiple object) if any
     single one fails, the entire method will fail with that Failure.  It is
     all or nothing. 
     """
     
     id = zi.interface.Attribute("the id of the Engine object")
     properties = zi.interface.Attribute("A dict of properties of the Engine")
     
     def execute(lines):
         """Execute lines of Python code.
         
         Returns a dictionary with keys (id, number, stdin, stdout, stderr)
         upon success.
         
         Returns a failure object if the execution of lines raises an exception.
         """
     
     def push(namespace):
         """Push dict namespace into the user's namespace.
         
         Returns a deferred to None or a failure.
         """
     
     def pull(keys):
         """Pulls values out of the user's namespace by keys.
         
         Returns a deferred to a tuple objects or a single object.
         
         Raises NameError if any one of objects doess not exist.
         """
     
     def push_function(namespace):
         """Push a dict of key, function pairs into the user's namespace.
         
         Returns a deferred to None or a failure."""
     
     def pull_function(keys):
         """Pulls functions out of the user's namespace by keys.
         
         Returns a deferred to a tuple of functions or a single function.
         
         Raises NameError if any one of the functions does not exist.
         """
     
     def get_result(i=None):
         """Get the stdin/stdout/stderr of command i.
         
         Returns a deferred to a dict with keys
         (id, number, stdin, stdout, stderr).
         
         Raises IndexError if command i does not exist.
         Raises TypeError if i in not an int.
         """
     
     def reset():
         """Reset the shell.
         
         This clears the users namespace.  Won't cause modules to be
         reloaded.  Should also re-initialize certain variables like id.
         """
     
     def kill():
         """Kill the engine by stopping the reactor."""
     
     def keys():
         """Return the top level variables in the users namspace.
         
         Returns a deferred to a dict."""
     
 
 class IEngineSerialized(zi.Interface):
     """Push/Pull methods that take Serialized objects.  
     
     All methods should return deferreds.
     """
     
     def push_serialized(namespace):
         """Push a dict of keys and Serialized objects into the user's namespace."""
     
     def pull_serialized(keys):
         """Pull objects by key from the user's namespace as Serialized.
         
         Returns a list of or one Serialized.
         
         Raises NameError is any one of the objects does not exist.
         """
     
 
 class IEngineProperties(zi.Interface):
     """Methods for access to the properties object of an Engine"""
     
     properties = zi.Attribute("A StrictDict object, containing the properties")
     
     def set_properties(properties):
         """set properties by key and value"""
     
     def get_properties(keys=None):
         """get a list of properties by `keys`, if no keys specified, get all"""
     
     def del_properties(keys):
         """delete properties by `keys`"""
     
     def has_properties(keys):
         """get a list of bool values for whether `properties` has `keys`"""
 
     def clear_properties():
         """clear the properties dict"""
     
 class IEngineBase(IEngineCore, IEngineSerialized, IEngineProperties):
     """The basic engine interface that EngineService will implement.
     
     This exists so it is easy to specify adapters that adapt to and from the
     API that the basic EngineService implements.
     """
     pass
 
 class IEngineQueued(IEngineBase):
     """Interface for adding a queue to an IEngineBase.  
     
     This interface extends the IEngineBase interface to add methods for managing
     the engine's queue.  The implicit details of this interface are that the 
     execution of all methods declared in IEngineBase should appropriately be
     put through a queue before execution. 
     
     All methods should return deferreds.
     """
     
     def clear_queue():
         """Clear the queue."""
     
     def queue_status():
         """Get the queued and pending commands in the queue."""
     
     def register_failure_observer(obs):
         """Register an observer of pending Failures.
         
         The observer must implement IFailureObserver.
         """
     
     def unregister_failure_observer(obs):
         """Unregister an observer of pending Failures."""
     
 
 class IEngineThreaded(zi.Interface):
     """A place holder for threaded commands.  
     
     All methods should return deferreds.
     """
     pass
 
 
 #-------------------------------------------------------------------------------
 # Functions and classes to implement the EngineService
 #-------------------------------------------------------------------------------
 
 
 class StrictDict(dict):
     """This is a strict copying dictionary for use as the interface to the 
     properties of an Engine.
     :IMPORTANT:
         This object copies the values you set to it, and returns copies to you
         when you request them.  The only way to change properties os explicitly
         through the setitem and getitem of the dictionary interface.
         Example:
         >>> e = kernel.get_engine(id)
         >>> L = someList
         >>> e.properties['L'] = L
         >>> L == e.properties['L']
         ... True
         >>> L.append(something Else)
         >>> L == e.properties['L']
         ... False
         
         getitem copies, so calls to methods of objects do not affect the
         properties, as in the following example:
         >>> e.properties[1] = range(2)
         >>> print e.properties[1]
         ... [0, 1]
         >>> e.properties[1].append(2)
         >>> print e.properties[1]
         ... [0, 1]
         
     """
     def __init__(self, *args, **kwargs):
         dict.__init__(self, *args, **kwargs)
         self.modified = True
     
     def __getitem__(self, key):
         return copy.deepcopy(dict.__getitem__(self, key))
     
     def __setitem__(self, key, value):
         # check if this entry is valid for transport around the network
         # and copying
         try:
             pickle.dumps(key, 2)
             pickle.dumps(value, 2)
             newvalue = copy.deepcopy(value)
         except:
             raise error.InvalidProperty(value)
         dict.__setitem__(self, key, newvalue)
         self.modified = True
     
     def __delitem__(self, key):
         dict.__delitem__(self, key)
         self.modified = True
     
     def update(self, dikt):
         for k,v in dikt.iteritems():
             self[k] = v
     
     def pop(self, key):
         self.modified = True
         return dict.pop(self, key)
     
     def popitem(self):
         self.modified = True
         return dict.popitem(self)
     
     def clear(self):
         self.modified = True
         dict.clear(self)
     
     def subDict(self, *keys):
         d = {}
         for key in keys:
             d[key] = self[key]
         return d
     
 
 
 class EngineAPI(object):
     """This is the object through which the user can edit the `properties`
     attribute of an Engine.
     The Engine Properties object copies all object in and out of itself.
     See the EngineProperties object for details.
     """
     _fix=False
     def __init__(self, id):
         self.id = id
         self.properties = StrictDict()
         self._fix=True
     
     def __setattr__(self, k,v):
         if self._fix:
             raise error.KernelError("I am protected!")
         else:
             object.__setattr__(self, k, v)
     
     def __delattr__(self, key):
         raise error.KernelError("I am protected!")
     
 
 _apiDict = {}
 
 def get_engine(id):
     """Get the Engine API object, whcih currently just provides the properties 
     object, by ID"""
     global _apiDict
     if not _apiDict.get(id):
         _apiDict[id] = EngineAPI(id)
     return _apiDict[id]
 
 def drop_engine(id):
     """remove an engine"""
     global _apiDict
     if _apiDict.has_key(id):
         del _apiDict[id]
 
 class EngineService(object, service.Service):
     """Adapt a IPython shell into a IEngine implementing Twisted Service."""
     
     zi.implements(IEngineBase)
     name = 'EngineService'
     
     def __init__(self, shellClass=Interpreter, mpi=None):
         """Create an EngineService.
         
         shellClass: something that implements IInterpreter or core1
         mpi:        an mpi module that has rank and size attributes
         """
         self.shellClass = shellClass
         self.shell = self.shellClass()
         self.mpi = mpi
         self.id = None
         self.properties = get_engine(self.id).properties
         if self.mpi is not None:
             log.msg("MPI started with rank = %i and size = %i" % 
                 (self.mpi.rank, self.mpi.size))
             self.id = self.mpi.rank
         self._seedNamespace()
     
     # Make id a property so that the shell can get the updated id
         
     def _setID(self, id):
         self._id = id
         self.properties = get_engine(id).properties
         self.shell.push({'id': id})
     
     def _getID(self):
         return self._id
     
     id = property(_getID, _setID)
     
     def _seedNamespace(self):
         self.shell.push({'mpi': self.mpi, 'id' : self.id})
     
     def executeAndRaise(self, msg, callable, *args, **kwargs):
         """Call a method of self.shell and wrap any exception."""
         d = defer.Deferred()
         try:
             result = callable(*args, **kwargs)
         except:
             # This gives the following:
             # et=exception class
             # ev=exception class instance
             # tb=traceback object
             et,ev,tb = sys.exc_info()
             # This call adds attributes to the exception value
             et,ev,tb = self.shell.formatTraceback(et,ev,tb,msg)
             # Add another attribute
             ev._ipython_engine_info = msg
             f = failure.Failure(ev,et,None)
             d.errback(f)
         else:
             d.callback(result)
             
         return d
     
     # The IEngine methods.  See the interface for documentation.
     
     def execute(self, lines):
         msg = {'engineid':self.id,
                'method':'execute',
                'args':[lines]}
         d = self.executeAndRaise(msg, self.shell.execute, lines)
         d.addCallback(self.addIDToResult)
         return d
     
     def addIDToResult(self, result):
         result['id'] = self.id
         return result
     
     def push(self, namespace):
         msg = {'engineid':self.id,
                'method':'push',
                'args':[repr(namespace.keys())]}
         d = self.executeAndRaise(msg, self.shell.push, namespace)
         return d
     
     def pull(self, keys):
         msg = {'engineid':self.id,
                'method':'pull',
                'args':[repr(keys)]}
         d = self.executeAndRaise(msg, self.shell.pull, keys)
         return d
     
     def push_function(self, namespace):
         msg = {'engineid':self.id,
                'method':'push_function',
                'args':[repr(namespace.keys())]}
         d = self.executeAndRaise(msg, self.shell.push_function, namespace)
         return d
     
     def pull_function(self, keys):
         msg = {'engineid':self.id,
                'method':'pull_function',
                'args':[repr(keys)]}
         d = self.executeAndRaise(msg, self.shell.pull_function, keys)
         return d
     
     def get_result(self, i=None):
         msg = {'engineid':self.id,
                'method':'get_result',
                'args':[repr(i)]}
         d = self.executeAndRaise(msg, self.shell.getCommand, i)
         d.addCallback(self.addIDToResult)
         return d
     
     def reset(self):
         msg = {'engineid':self.id,
                'method':'reset',
                'args':[]}
         del self.shell
         self.shell = self.shellClass()
         self.properties.clear()
         d = self.executeAndRaise(msg, self._seedNamespace)
         return d
     
     def kill(self):
         drop_engine(self.id)
         try:
             reactor.stop()
         except RuntimeError:
             log.msg('The reactor was not running apparently.')
             return defer.fail()
         else:
             return defer.succeed(None)
     
     def keys(self):
         """Return a list of variables names in the users top level namespace.
         
         This used to return a dict of all the keys/repr(values) in the 
         user's namespace.  This was too much info for the ControllerService
         to handle so it is now just a list of keys.
         """
         
         remotes = []
         for k in self.shell.user_ns.iterkeys():
             if k not in ['__name__', '_ih', '_oh', '__builtins__',
                          'In', 'Out', '_', '__', '___', '__IP', 'input', 'raw_input']:
                 remotes.append(k)
         return defer.succeed(remotes)
     
     def set_properties(self, properties):
         msg = {'engineid':self.id,
                'method':'set_properties',
                'args':[repr(properties.keys())]}
         return self.executeAndRaise(msg, self.properties.update, properties)
     
     def get_properties(self, keys=None):
         msg = {'engineid':self.id,
                'method':'get_properties',
                'args':[repr(keys)]}
         if keys is None:
             keys = self.properties.keys()
         return self.executeAndRaise(msg, self.properties.subDict, *keys)
     
     def _doDel(self, keys):
         for key in keys:
             del self.properties[key]
     
     def del_properties(self, keys):
         msg = {'engineid':self.id,
                'method':'del_properties',
                'args':[repr(keys)]}
         return self.executeAndRaise(msg, self._doDel, keys)
     
     def _doHas(self, keys):
         return [self.properties.has_key(key) for key in keys]
     
     def has_properties(self, keys):
         msg = {'engineid':self.id,
                'method':'has_properties',
                'args':[repr(keys)]}
         return self.executeAndRaise(msg, self._doHas, keys)
     
     def clear_properties(self):
         msg = {'engineid':self.id,
                'method':'clear_properties',
                'args':[]}
         return self.executeAndRaise(msg, self.properties.clear)
     
     def push_serialized(self, sNamespace):
         msg = {'engineid':self.id,
                'method':'push_serialized',
                'args':[repr(sNamespace.keys())]}       
         ns = {}
         for k,v in sNamespace.iteritems():
             try:
                 unserialized = newserialized.IUnSerialized(v)
                 ns[k] = unserialized.getObject()
             except:
                 return defer.fail()
         return self.executeAndRaise(msg, self.shell.push, ns)
     
     def pull_serialized(self, keys):
         msg = {'engineid':self.id,
                'method':'pull_serialized',
                'args':[repr(keys)]}
         if isinstance(keys, str):
             keys = [keys]
         if len(keys)==1:
             d = self.executeAndRaise(msg, self.shell.pull, keys)
             d.addCallback(newserialized.serialize)
             return d
         elif len(keys)>1:
             d = self.executeAndRaise(msg, self.shell.pull, keys)         
             @d.addCallback
             def packThemUp(values):
                 serials = []
                 for v in values:
                     try:
                         serials.append(newserialized.serialize(v))
                     except:
                         return defer.fail(failure.Failure())
                 return serials
             return packThemUp
 
 
 def queue(methodToQueue):
     def queuedMethod(this, *args, **kwargs):
         name = methodToQueue.__name__
         return this.submitCommand(Command(name, *args, **kwargs))
     return queuedMethod
     
 class QueuedEngine(object):
     """Adapt an IEngineBase to an IEngineQueued by wrapping it.
     
     The resulting object will implement IEngineQueued which extends
     IEngineCore which extends (IEngineBase, IEngineSerialized). 
     
     This seems like the best way of handling it, but I am not sure.  The
     other option is to have the various base interfaces be used like
     mix-in intefaces.  The problem I have with this is adpatation is
     more difficult and complicated because there can be can multiple
     original and final Interfaces. 
     """
     
     zi.implements(IEngineQueued)
     
     def __init__(self, engine):
         """Create a QueuedEngine object from an engine
         
         engine:       An implementor of IEngineCore and IEngineSerialized
         keepUpToDate: whether to update the remote status when the 
                       queue is empty.  Defaults to False.
         """
         
         # This is the right way to do these tests rather than 
         # IEngineCore in list(zi.providedBy(engine)) which will only 
         # picks of the interfaces that are directly declared by engine.
         assert IEngineBase.providedBy(engine), \
             "engine passed to QueuedEngine doesn't provide IEngineBase"
             
         self.engine = engine
         self.id = engine.id
         self.queued = []
         self.history = {}
         self.engineStatus = {}
         self.currentCommand = None
         self.failureObservers = []
     
     def _get_properties(self):
         return self.engine.properties
     
     properties = property(_get_properties, lambda self, _: None)
     # Queue management methods.  You should not call these directly
     
     def submitCommand(self, cmd):
         """Submit command to queue."""
         
         d = defer.Deferred()
         cmd.setDeferred(d)
         if self.currentCommand is not None:
             if self.currentCommand.finished:
                 # log.msg("Running command immediately: %r" % cmd)
                 self.currentCommand = cmd
                 self.runCurrentCommand()
             else:  # command is still running  
                 # log.msg("Command is running: %r" % self.currentCommand)
                 # log.msg("Queueing: %r" % cmd)
                 self.queued.append(cmd)
         else:
             # log.msg("No current commands, running: %r" % cmd)
             self.currentCommand = cmd
             self.runCurrentCommand()
         return d
     
     def runCurrentCommand(self):
         """Run current command."""
         
         cmd = self.currentCommand
         f = getattr(self.engine, cmd.remoteMethod, None)
         if f:
             d = f(*cmd.args, **cmd.kwargs)
             if cmd.remoteMethod is 'execute':
                 d.addCallback(self.saveResult)
             d.addCallback(self.finishCommand)
             d.addErrback(self.abortCommand)
         else:
             return defer.fail(AttributeError(cmd.remoteMethod))
     
     def _flushQueue(self):
         """Pop next command in queue and run it."""
         
         if len(self.queued) > 0:
             self.currentCommand = self.queued.pop(0)
             self.runCurrentCommand()
     
     def saveResult(self, result):
         """Put the result in the history."""
         self.history[result['number']] = result
         return result
     
     def finishCommand(self, result):
         """Finish currrent command."""
         
         # The order of these commands is absolutely critical.
         self.currentCommand.handleResult(result)
         self.currentCommand.finished = True
         self._flushQueue()
         return result
     
     def abortCommand(self, reason):
         """Abort current command.
         
         This eats the Failure but first passes it onto the Deferred that the 
         user has.
         
         It also clear out the queue so subsequence commands don't run.
         """
 
         # The order of these 3 commands is absolutely critical.  The currentCommand
         # must first be marked as finished BEFORE the queue is cleared and before
         # the current command is sent the failure.
         # Also, the queue must be cleared BEFORE the current command is sent the Failure
         # otherwise the errback chain could trigger new commands to be added to the 
         # queue before we clear it.  We should clear ONLY the commands that were in
         # the queue when the error occured. 
         self.currentCommand.finished = True
         s = "%r %r %r" % (self.currentCommand.remoteMethod, self.currentCommand.args, self.currentCommand.kwargs)
         self.clear_queue(msg=s)
         self.currentCommand.handleError(reason)
         
         return None
     
     #---------------------------------------------------------------------------
     # IEngineCore methods
     #---------------------------------------------------------------------------
     
     @queue
     def execute(self, lines):
         pass
 
     @queue
     def push(self, namespace):
         pass      
     
     @queue
     def pull(self, keys):
         pass
         
     @queue
     def push_function(self, namespace):
         pass      
     
     @queue
     def pull_function(self, keys):
         pass        
 
     def get_result(self, i=None):
         if i is None:
             i = max(self.history.keys()+[None])
 
         cmd = self.history.get(i, None)
         # Uncomment this line to disable chaching of results
         #cmd = None
         if cmd is None:
             return self.submitCommand(Command('get_result', i))
         else:
             return defer.succeed(cmd)
         
     def reset(self):
         self.clear_queue()
         self.history = {}  # reset the cache - I am not sure we should do this
         return self.submitCommand(Command('reset'))
     
     def kill(self):
         self.clear_queue()
         return self.submitCommand(Command('kill'))
     
     @queue
     def keys(self):
         pass
     
     #---------------------------------------------------------------------------
     # IEngineSerialized methods
     #---------------------------------------------------------------------------
 
     @queue
     def push_serialized(self, namespace):
         pass
         
     @queue
     def pull_serialized(self, keys):
         pass
     
     #---------------------------------------------------------------------------
     # IEngineProperties methods
     #---------------------------------------------------------------------------
 
     @queue
     def set_properties(self, namespace):
         pass
         
     @queue
     def get_properties(self, keys=None):
         pass
     
     @queue
     def del_properties(self, keys):
         pass
     
     @queue
     def has_properties(self, keys):
         pass
     
     @queue
     def clear_properties(self):
         pass
     
     #---------------------------------------------------------------------------
     # IQueuedEngine methods
     #---------------------------------------------------------------------------
     
     def clear_queue(self, msg=''):
         """Clear the queue, but doesn't cancel the currently running commmand."""
         
         for cmd in self.queued:
             cmd.deferred.errback(failure.Failure(error.QueueCleared(msg)))
         self.queued = []
         return defer.succeed(None)
     
     def queue_status(self):
         if self.currentCommand is not None:
             if self.currentCommand.finished:
                 pending = repr(None)
             else:
                 pending = repr(self.currentCommand)
         else:
             pending = repr(None)
         dikt = {'queue':map(repr,self.queued), 'pending':pending}
         return defer.succeed(dikt)
         
     def register_failure_observer(self, obs):
         self.failureObservers.append(obs)
     
     def unregister_failure_observer(self, obs):
         self.failureObservers.remove(obs)
     
 
 # Now register QueuedEngine as an adpater class that makes an IEngineBase into a
 # IEngineQueued.  
 components.registerAdapter(QueuedEngine, IEngineBase, IEngineQueued)
     
 
 class Command(object):
     """A command object that encapslates queued commands.
     
     This class basically keeps track of a command that has been queued
     in a QueuedEngine.  It manages the deferreds and hold the method to be called
     and the arguments to that method.
     """
     
     
     def __init__(self, remoteMethod, *args, **kwargs):
         """Build a new Command object."""
         
         self.remoteMethod = remoteMethod
         self.args = args
         self.kwargs = kwargs
         self.finished = False
     
     def setDeferred(self, d):
         """Sets the deferred attribute of the Command."""  
           
         self.deferred = d
     
     def __repr__(self):
         if not self.args:
             args = ''
         else:
             args = str(self.args)[1:-2]  #cut off (...,)
         for k,v in self.kwargs.iteritems():
             if args:
                 args += ', '
             args += '%s=%r' %(k,v)
         return "%s(%s)" %(self.remoteMethod, args)
     
     def handleResult(self, result):
         """When the result is ready, relay it to self.deferred."""
         
         self.deferred.callback(result)
     
     def handleError(self, reason):
         """When an error has occured, relay it to self.deferred."""
         
         self.deferred.errback(reason)
 
 class ThreadedEngineService(EngineService):
-    """An EngineService subclass that defers execute commands to a separate thread.
+    """An EngineService subclass that defers execute commands to a separate 
+    thread.
     
-    ThreadedEngineService uses twisted.internet.threads.deferToThread to defer execute
-    requests to a separate thread. GUI frontends may want to use ThreadedEngineService as
-    the engine in an IPython.frontend.frontendbase.FrontEndBase subclass to prevent
+    ThreadedEngineService uses twisted.internet.threads.deferToThread to 
+    defer execute requests to a separate thread. GUI frontends may want to 
+    use ThreadedEngineService as the engine in an 
+    IPython.frontend.frontendbase.FrontEndBase subclass to prevent
     block execution from blocking the GUI thread.
     """
     
     zi.implements(IEngineBase)
 
     def __init__(self, shellClass=Interpreter, mpi=None):
         EngineService.__init__(self, shellClass, mpi)
     
     
     def execute(self, lines):
         # Only import this if we are going to use this class
         from twisted.internet import threads
         
         msg = {'engineid':self.id,
                'method':'execute',
                'args':[lines]}
         
         d = threads.deferToThread(self.shell.execute, lines)
         d.addCallback(self.addIDToResult)
         return d

docs/source/changes.txt

@@ -1,161 +1,162 b''
 .. _changes:
 
 ==========
 What's new
 ==========
 
 .. contents::
 
 Release 0.9
 ===========
 
 New features
 ------------
 
 	* All of the parallel computing capabilities from `ipython1-dev` have been merged into
 	  IPython proper.  This resulted in the following new subpackages:  
 	  :mod:`IPython.kernel`, :mod:`IPython.kernel.core`, :mod:`IPython.config`,
 	  :mod:`IPython.tools` and :mod:`IPython.testing`.
 	* As part of merging in the `ipython1-dev` stuff, the `setup.py` script and friends
 	  have been completely refactored.  Now we are checking for dependencies using
 	  the approach that matplotlib uses.
 	* The documentation has been completely reorganized to accept the documentation
 	  from `ipython1-dev`.
 	* We have switched to using Foolscap for all of our network protocols in
 	  :mod:`IPython.kernel`.  This gives us secure connections that are both encrypted
 	  and authenticated.
 	* We have a brand new `COPYING.txt` files that describes the IPython license
 	  and copyright.  The biggest change is that we are putting "The IPython
 	  Development Team" as the copyright holder.  We give more details about exactly
 	  what this means in this file.  All developer should read this and use the new
 	  banner in all IPython source code files.
+        * sh profile: ./foo runs foo as system command, no need to do !./foo anymore
 
 Bug fixes
 ---------
 
 	* A few subpackages has missing `__init__.py` files.
 	* The documentation is only created is Sphinx is found.  Previously, the `setup.py`
 	  script would fail if it was missing.
 
 Backwards incompatible changes
 ------------------------------
 
 	* IPython has a larger set of dependencies if you want all of its capabilities.
 	  See the `setup.py` script for details.
 	* The constructors for :class:`IPython.kernel.client.MultiEngineClient` and 
 	  :class:`IPython.kernel.client.TaskClient` no longer take the (ip,port) tuple.
 	  Instead they take the filename of a file that contains the FURL for that
 	  client.  If the FURL file is in your IPYTHONDIR, it will be found automatically
 	  and the constructor can be left empty.
 	* The asynchronous clients in :mod:`IPython.kernel.asyncclient` are now created 
 	  using the factory functions :func:`get_multiengine_client` and 
 	  :func:`get_task_client`.  These return a `Deferred` to the actual client.
 	* The command line options to `ipcontroller` and `ipengine` have changed to
 	  reflect the new Foolscap network protocol and the FURL files.  Please see the
 	  help for these scripts for details.
 	* The configuration files for the kernel have changed because of the Foolscap stuff.
 	  If you were using custom config files before, you should delete them and regenerate
 	  new ones.
 
 Changes merged in from IPython1
 -------------------------------
 
 New features
 ............
 
 	* Much improved ``setup.py`` and ``setupegg.py`` scripts.  Because Twisted
 	  and zope.interface are now easy installable, we can declare them as dependencies
 	  in our setupegg.py script.
 	* IPython is now compatible with Twisted 2.5.0 and 8.x.
 	* Added a new example of how to use :mod:`ipython1.kernel.asynclient`.
 	* Initial draft of a process daemon in :mod:`ipython1.daemon`.  This has not
 	  been merged into IPython and is still in `ipython1-dev`.
 	* The ``TaskController`` now has methods for getting the queue status.
 	* The ``TaskResult`` objects not have information about how long the task
 	  took to run.
 	* We are attaching additional attributes to exceptions ``(_ipython_*)`` that 
 	  we use to carry additional info around.
 	* New top-level module :mod:`asyncclient` that has asynchronous versions (that
 	  return deferreds) of the client classes.  This is designed to users who want
 	  to run their own Twisted reactor
 	* All the clients in :mod:`client` are now based on Twisted.  This is done by 
 	  running the Twisted reactor in a separate thread and using the
 	  :func:`blockingCallFromThread` function that is in recent versions of Twisted.
 	* Functions can now be pushed/pulled to/from engines using
 	  :meth:`MultiEngineClient.push_function` and :meth:`MultiEngineClient.pull_function`.
 	* Gather/scatter are now implemented in the client to reduce the work load
 	  of the controller and improve performance.
 	* Complete rewrite of the IPython docuementation.  All of the documentation
 	  from the IPython website has been moved into docs/source as restructured
 	  text documents.  PDF and HTML documentation are being generated using 
 	  Sphinx.
 	* New developer oriented documentation: development guidelines and roadmap. 
 	* Traditional ``ChangeLog`` has been changed to a more useful ``changes.txt`` file
 	  that is organized by release and is meant to provide something more relevant
 	  for users.
 
 Bug fixes
 .........
 
 	* Created a proper ``MANIFEST.in`` file to create source distributions.
 	* Fixed a bug in the ``MultiEngine`` interface.  Previously, multi-engine 
 	  actions were being collected with a :class:`DeferredList` with 
 	  ``fireononeerrback=1``.  This meant that methods were returning 
 	  before all engines had given their results.  This was causing extremely odd 
 	  bugs in certain cases. To fix this problem, we have 1) set 
 	  ``fireononeerrback=0`` to make sure all results (or exceptions) are in 
 	  before returning and 2) introduced a :exc:`CompositeError` exception 
 	  that wraps all of the engine exceptions.  This is a huge change as it means 
 	  that users will have to catch :exc:`CompositeError` rather than the actual
 	  exception.
 
 Backwards incompatible changes
 ..............................
 
 	* All names have been renamed to conform to the lowercase_with_underscore
 	  convention.  This will require users to change references to all names like
 	  ``queueStatus`` to ``queue_status``.
 	* Previously, methods like :meth:`MultiEngineClient.push` and    	
 	  :meth:`MultiEngineClient.push` used ``*args`` and ``**kwargs``.  This was 
 	  becoming a problem as we weren't able to introduce new keyword arguments into
 	  the API.  Now these methods simple take a dict or sequence.  This has also allowed
 	  us to get rid of the ``*All`` methods like :meth:`pushAll` and :meth:`pullAll`.
 	  These things are now handled with the ``targets`` keyword argument that defaults
 	  to ``'all'``.
 	* The :attr:`MultiEngineClient.magicTargets` has been renamed to
 	  :attr:`MultiEngineClient.targets`. 
 	* All methods in the MultiEngine interface now accept the optional keyword argument
 	  ``block``.
 	* Renamed :class:`RemoteController` to :class:`MultiEngineClient` and 
 	  :class:`TaskController` to :class:`TaskClient`.
 	* Renamed the top-level module from :mod:`api` to :mod:`client`.
 	* Most methods in the multiengine interface now raise a :exc:`CompositeError` exception
 	  that wraps the user's exceptions, rather than just raising the raw user's exception.
 	* Changed the ``setupNS`` and ``resultNames`` in the ``Task`` class to ``push`` 
 	  and ``pull``.
 
 Release 0.8.4
 =============
 
 Someone needs to describe what went into 0.8.4.
 
 Release 0.8.2
 =============
 
 * %pushd/%popd behave differently; now "pushd /foo" pushes CURRENT directory 
   and jumps to /foo. The current behaviour is closer to the documented 
   behaviour, and should not trip anyone.
   
 Release 0.8.3
 =============
   
 * pydb is now disabled by default (due to %run -d problems). You can enable
   it by passing -pydb command line argument to IPython. Note that setting
   it in config file won't work.
 
 Older releases
 ==============
 
 Changes in earlier releases of IPython are described in the older file ``ChangeLog``.  
 Please refer to this document for details.
 

docs/source/development/development.txt

@@ -1,315 +1,360 b''
 .. _development:
 
 ==================================
 IPython development guidelines
 ==================================
 
 .. contents::
-..
-    1  Overview
-    2  Project organization
-      2.1  Subpackages
-      2.2  Installation and dependencies
-      2.3  Specific subpackages
-    3  Version control
-    4  Documentation
-      4.1  Standalone documentation
-      4.2  Docstring format
-    5  Coding conventions
-      5.1  General
-      5.2  Naming conventions
-    6  Testing
-    7  Configuration
-..
 
 
 Overview
 ========
 
 IPython is the next generation of IPython.  It is named such for two reasons:
 
 - Eventually, IPython will become IPython version 1.0.
 - This new code base needs to be able to co-exist with the existing IPython until
   it is a full replacement for it.  Thus we needed a different name.  We couldn't
   use ``ipython`` (lowercase) as some files systems are case insensitive.
 
 There are two, no three, main goals of the IPython effort:
 
 1. Clean up the existing codebase and write lots of tests.
 2. Separate the core functionality of IPython from the terminal to enable IPython
    to be used from within a variety of GUI applications.
 3. Implement a system for interactive parallel computing.
 
 While the third goal may seem a bit unrelated to the main focus of IPython, it turns
 out that the technologies required for this goal are nearly identical with those
 required for goal two. This is the main reason the interactive parallel computing
 capabilities are being put into IPython proper. Currently the third of these goals is
 furthest along.
 
 This document describes IPython from the perspective of developers.  
 
 
 Project organization
 ====================
 
 Subpackages
 -----------
 
 IPython is organized into semi self-contained subpackages.  Each of the subpackages will have its own:
 
 - **Dependencies**.  One of the most important things to keep in mind in
   partitioning code amongst subpackages, is that they should be used to cleanly
   encapsulate dependencies.  
 - **Tests**.  Each subpackage shoud have its own ``tests`` subdirectory that
   contains all of the tests for that package.  For information about writing tests
   for IPython, see the `Testing System`_ section of this document.
 - **Configuration**.  Each subpackage should have its own ``config`` subdirectory
   that contains the configuration information for the components of the 
   subpackage.  For information about how the IPython configuration system
   works, see the `Configuration System`_ section of this document.
 - **Scripts**.  Each subpackage should have its own ``scripts`` subdirectory that
   contains all of the command line scripts associated with the subpackage.
 
 Installation and dependencies
 -----------------------------
    
 IPython will not use `setuptools`_ for installation. Instead, we will use standard
 ``setup.py`` scripts that use `distutils`_. While there are a number a extremely nice
 features that `setuptools`_ has (like namespace packages), the current implementation
 of `setuptools`_ has performance problems, particularly on shared file systems. In
 particular, when Python packages are installed on NSF file systems, import times
 become much too long (up towards 10 seconds). 
 
 Because IPython is being used extensively in the context of high performance
 computing, where performance is critical but shared file systems are common, we feel
 these performance hits are not acceptable. Thus, until the performance problems
 associated with `setuptools`_ are addressed, we will stick with plain `distutils`_. We
 are hopeful that these problems will be addressed and that we will eventually begin
 using `setuptools`_. Because of this, we are trying to organize IPython in a way that
 will make the eventual transition to `setuptools`_ as painless as possible.
 
 Because we will be using `distutils`_, there will be no method for automatically installing dependencies.  Instead, we are following the approach of `Matplotlib`_ which can be summarized as follows:
 
 - Distinguish between required and optional dependencies.  However, the required
   dependencies for IPython should be only the Python standard library.
 - Upon installation check to see which optional dependencies are present and tell
   the user which parts of IPython need which optional dependencies.
 
 It is absolutely critical that each subpackage of IPython has a clearly specified set
 of dependencies and that dependencies are not carelessly inherited from other IPython
 subpackages. Furthermore, tests that have certain dependencies should not fail if
 those dependencies are not present. Instead they should be skipped and print a
 message.
 
 .. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools
 .. _distutils: http://docs.python.org/lib/module-distutils.html
 .. _Matplotlib: http://matplotlib.sourceforge.net/
 
 Specific subpackages
 --------------------
 
 ``core``
 	This is the core functionality of IPython that is independent of the
 	terminal, network and GUIs.  Most of the code that is in the current
 	IPython trunk will be refactored, cleaned up and moved here.
 
 ``kernel``
 	The enables the IPython core to be expose to a the network.  This is
 	also where all of the parallel computing capabilities are to be found.
 	
 ``config``
 	The configuration package used by IPython.
 
 ``frontends``
 	The various frontends for IPython.  A frontend is the end-user application
 	that exposes the capabilities of IPython to the user.  The most basic frontend
 	will simply be a terminal based application that looks just like today 's 
 	IPython.  Other frontends will likely be more powerful and based on GUI toolkits.
 
 ``notebook``
 	An application that allows users to work with IPython notebooks.
 
 ``tools``
 	This is where general utilities go.
 
 
 Version control
 ===============
 
-In the past, IPython development has been done using `Subversion`__.  We are currently trying out `Bazaar`__ and `Launchpad`__.
+In the past, IPython development has been done using `Subversion`__.  Recently, we made the transition to using `Bazaar`__ and `Launchpad`__.  This makes it much easier for people
+to contribute code to IPython.  Here is a sketch of how to use Bazaar for IPython
+development.  First, you should install Bazaar.  After you have done that, make
+sure that it is working by getting the latest main branch of IPython::
+
+	$ bzr branch lp:ipython
+
+Now you can create a new branch for you to do your work in::
+
+	$ bzr branch ipython ipython-mybranch
+
+The typical work cycle in this branch will be to make changes in `ipython-mybranch`
+and then commit those changes using the commit command::
+
+	$ ...do work in ipython-mybranch...
+	$ bzr ci -m "the commit message goes here"
+
+Please note that since we now don't use an old-style linear ChangeLog
+(that tends to cause problems with distributed version control
+systems), you should ensure that your log messages are reasonably
+detailed.  Use a docstring-like approach in the commit messages
+(including the second line being left *blank*)::
+
+  Single line summary of  changes being committed.
+
+  - more details when warranted ...
+  - including crediting outside contributors if they sent the
+    code/bug/idea!
+
+If we couple this with a policy of making single commits for each
+reasonably atomic change, the bzr log should give an excellent view of
+the project, and the `--short` log option becomes a nice summary.
+
+While working with this branch, it is a good idea to merge in changes that have been
+made upstream in the parent branch.  This can be done by doing::
+
+	$ bzr pull
+	
+If this command shows that the branches have diverged, then you should do a merge
+instead::
+
+	$ bzr merge lp:ipython
+
+If you want others to be able to see your branch, you can create an account with
+launchpad and push the branch to your own workspace::
+
+	$ bzr push bzr+ssh://<me>@bazaar.launchpad.net/~<me>/+junk/ipython-mybranch
+
+Finally, once the work in your branch is done, you can merge your changes back into
+the `ipython` branch by using merge::
+
+	$ cd ipython
+	$ merge ../ipython-mybranch
+	[resolve any conflicts]
+	$ bzr ci -m "Fixing that bug"
+	$ bzr push
+
+But this will require you to have write permissions to the `ipython` branch.  It you don't
+you can tell one of the IPython devs about your branch and they can do the merge for you.
+
+More information about Bazaar workflows can be found `here`__.
 
 .. __: http://subversion.tigris.org/
 .. __: http://bazaar-vcs.org/
 .. __: http://www.launchpad.net/ipython
+.. __: http://doc.bazaar-vcs.org/bzr.dev/en/user-guide/index.html
 
 Documentation
 =============
 
 Standalone documentation
 ------------------------
 
 All standalone documentation should be written in plain text (``.txt``) files using
 `reStructuredText`_ for markup and formatting. All such documentation should be placed
 in the top level directory ``docs`` of the IPython source tree. Or, when appropriate,
 a suitably named subdirectory should be used. The documentation in this location will
 serve as the main source for IPython documentation and all existing documentation
 should be converted to this format.
 
 In the future, the text files in the ``docs`` directory will be used to generate all
 forms of documentation for IPython. This include documentation on the IPython website
 as well as *pdf* documentation.
 
 .. _reStructuredText: http://docutils.sourceforge.net/rst.html
 
 Docstring format
 ----------------
 
 Good docstrings are very important. All new code will use `Epydoc`_ for generating API
 docs, so we will follow the `Epydoc`_ conventions. More specifically, we will use
 `reStructuredText`_ for markup and formatting, since it is understood by a wide
 variety of tools. This means that if in the future we have any reason to change from
 `Epydoc`_ to something else, we'll have fewer transition pains.
 
 Details about using `reStructuredText`_ for docstrings can be found `here
 <http://epydoc.sourceforge.net/manual-othermarkup.html>`_.
 
 .. _Epydoc: http://epydoc.sourceforge.net/
 
 Additional PEPs of interest regarding documentation of code:
 
 - `Docstring Conventions <http://www.python.org/peps/pep-0257.html>`_
 - `Docstring Processing System Framework <http://www.python.org/peps/pep-0256.html>`_
 - `Docutils Design Specification <http://www.python.org/peps/pep-0258.html>`_
 
 
 Coding conventions
 ==================
 
 General
 -------
 
 In general, we'll try to follow the standard Python style conventions as described here:
 
 - `Style Guide for Python Code  <http://www.python.org/peps/pep-0008.html>`_  
 
 
 Other comments:
 
 - In a large file, top level classes and functions should be 
   separated by 2-3 lines to make it easier to separate them visually.
 - Use 4 spaces for indentation.
 - Keep the ordering of methods the same in classes that have the same 
   methods.  This is particularly true for classes that implement 
   similar interfaces and for interfaces that are similar.
 
 Naming conventions
 ------------------
 
 In terms of naming conventions, we'll follow the guidelines from the `Style Guide for
 Python Code`_.
 
 For all new IPython code (and much existing code is being refactored), we'll use:
 
 - All ``lowercase`` module names.
 
 - ``CamelCase`` for class names.
 
 - ``lowercase_with_underscores`` for methods, functions, variables and attributes.
 
 This may be confusing as most of the existing IPython codebase uses a different convention (``lowerCamelCase`` for methods and attributes).  Slowly, we will move IPython over to the new
 convention, providing shadow names for backward compatibility in public interfaces. 
 
 There are, however, some important exceptions to these rules.  In some cases, IPython
 code will interface with packages (Twisted, Wx, Qt) that use other conventions.  At some level this makes it impossible to adhere to our own standards at all times.  In particular, when subclassing classes that use other naming conventions, you must follow their naming conventions.  To deal with cases like this, we propose the following policy:
 
 - If you are subclassing a class that uses different conventions, use its
   naming conventions throughout your subclass.  Thus, if you are creating a 
   Twisted Protocol class, used Twisted's ``namingSchemeForMethodsAndAttributes.``
 
 - All IPython's official interfaces should use our conventions.  In some cases 
   this will mean that you need to provide shadow names (first implement ``fooBar``
   and then ``foo_bar = fooBar``).  We want to avoid this at all costs, but it 
   will probably be necessary at times.  But, please use this sparingly!
 
 Implementation-specific *private* methods will use ``_single_underscore_prefix``.
 Names with a leading double underscore will *only* be used in special cases, as they
 makes subclassing difficult (such names are not easily seen by child classes).
 
 Occasionally some run-in lowercase names are used, but mostly for very short names or
 where we are implementing methods very similar to existing ones in a base class (like
 ``runlines()`` where ``runsource()`` and ``runcode()`` had established precedent).
 
 The old IPython codebase has a big mix of classes and modules prefixed with an
 explicit ``IP``. In Python this is mostly unnecessary, redundant and frowned upon, as
 namespaces offer cleaner prefixing. The only case where this approach is justified is
 for classes which are expected to be imported into external namespaces and a very
 generic name (like Shell) is too likely to clash with something else. We'll need to
 revisit this issue as we clean up and refactor the code, but in general we should
 remove as many unnecessary ``IP``/``ip`` prefixes as possible. However, if a prefix
 seems absolutely necessary the more specific ``IPY`` or ``ipy`` are preferred.
 
 .. _devel_testing:
 
 Testing system
 ==============
 
 It is extremely important that all code contributed to IPython has tests. Tests should
 be written as unittests, doctests or as entities that the `Nose`_ testing package will
 find. Regardless of how the tests are written, we will use `Nose`_ for discovering and
 running the tests. `Nose`_ will be required to run the IPython test suite, but will
 not be required to simply use IPython.
 
 .. _Nose: http://code.google.com/p/python-nose/
 
 Tests of `Twisted`__ using code should be written by subclassing the ``TestCase`` class
 that comes with ``twisted.trial.unittest``. When this is done, `Nose`_ will be able to
 run the tests and the twisted reactor will be handled correctly.
 
 .. __: http://www.twistedmatrix.com
 
 Each subpackage in IPython should have its own ``tests`` directory that contains all
 of the tests for that subpackage. This allows each subpackage to be self-contained. If
 a subpackage has any dependencies beyond the Python standard library, the tests for
 that subpackage should be skipped if the dependencies are not found. This is very
 important so users don't get tests failing simply because they don't have dependencies.
 
 We also need to look into use Noses ability to tag tests to allow a more modular
 approach of running tests.
 
 .. _devel_config:
 
 Configuration system
 ====================
 
 IPython uses `.ini`_ files for configuration purposes. This represents a huge
 improvement over the configuration system used in IPython. IPython works with these
 files using the `ConfigObj`_ package, which IPython includes as
 ``ipython1/external/configobj.py``.
 
 Currently, we are using raw `ConfigObj`_ objects themselves. Each subpackage of IPython
 should contain a ``config`` subdirectory that contains all of the configuration
 information for the subpackage. To see how configuration information is defined (along
 with defaults) see at the examples in ``ipython1/kernel/config`` and
 ``ipython1/core/config``. Likewise, to see how the configuration information is used,
 see examples in ``ipython1/kernel/scripts/ipengine.py``.
 
 Eventually, we will add a new layer on top of the raw `ConfigObj`_ objects. We are
 calling this new layer, ``tconfig``, as it will use a `Traits`_-like validation model.
 We won't actually use `Traits`_, but will implement something similar in pure Python.
 But, even in this new system, we will still use `ConfigObj`_ and `.ini`_ files
 underneath the hood. Talk to Fernando if you are interested in working on this part of
 IPython. The current prototype of ``tconfig`` is located in the IPython sandbox.
 
 .. _.ini: http://docs.python.org/lib/module-ConfigParser.html
 .. _ConfigObj: http://www.voidspace.org.uk/python/configobj.html
 .. _Traits: http://code.enthought.com/traits/
 
 
 
 
 
 
 
 
 
 

setupbase.py

@@ -1,226 +1,228 b''
 # encoding: utf-8
 
 """
 This module defines the things that are used in setup.py for building IPython
 
 This includes:
 
     * The basic arguments to setup
     * Functions for finding things like packages, package data, etc.
     * A function for checking dependencies.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import os, sys
 
 from glob import glob
 
 from setupext import install_data_ext
 
 #-------------------------------------------------------------------------------
 # Useful globals and utility functions
 #-------------------------------------------------------------------------------
 
 # A few handy globals
 isfile = os.path.isfile
 pjoin = os.path.join
 
 def oscmd(s):
     print ">", s
     os.system(s)
 
 # A little utility we'll need below, since glob() does NOT allow you to do
 # exclusion on multiple endings!
 def file_doesnt_endwith(test,endings):
     """Return true if test is a file and its name does NOT end with any
     of the strings listed in endings."""
     if not isfile(test):
         return False
     for e in endings:
         if test.endswith(e):
             return False
     return True
 
 #---------------------------------------------------------------------------
 # Basic project information
 #---------------------------------------------------------------------------
 
 # Release.py contains version, authors, license, url, keywords, etc.
 execfile(pjoin('IPython','Release.py'))
 
 # Create a dict with the basic information
 # This dict is eventually passed to setup after additional keys are added.
 setup_args = dict(
       name             = name,
       version          = version,
       description      = description,
       long_description = long_description,
       author           = author,
       author_email     = author_email,
       url              = url,
       download_url     = download_url,
       license          = license,
       platforms        = platforms,
       keywords         = keywords,
       cmdclass         = {'install_data': install_data_ext},
       )
 
 
 #---------------------------------------------------------------------------
 # Find packages
 #---------------------------------------------------------------------------
 
 def add_package(packages, pname, config=False, tests=False, scripts=False, others=None):
     """
     Add a package to the list of packages, including certain subpackages.
     """
     packages.append('.'.join(['IPython',pname]))
     if config:
         packages.append('.'.join(['IPython',pname,'config']))
     if tests:
         packages.append('.'.join(['IPython',pname,'tests']))
     if scripts:
         packages.append('.'.join(['IPython',pname,'scripts']))
     if others is not None:
         for o in others:
             packages.append('.'.join(['IPython',pname,o]))
 
 def find_packages():
     """
     Find all of IPython's packages.
     """
     packages = ['IPython']
     add_package(packages, 'config', tests=True)
     add_package(packages , 'Extensions')
     add_package(packages, 'external')
     add_package(packages, 'gui')
     add_package(packages, 'gui.wx')
     add_package(packages, 'kernel', config=True, tests=True, scripts=True)
     add_package(packages, 'kernel.core', config=True, tests=True)
     add_package(packages, 'testing', tests=True)
     add_package(packages, 'tools', tests=True)
     add_package(packages, 'UserConfig')
     return packages
 
 #---------------------------------------------------------------------------
 # Find package data
 #---------------------------------------------------------------------------
 
 def find_package_data():
     """
     Find IPython's package_data.
     """
     # This is not enough for these things to appear in an sdist.
     # We need to muck with the MANIFEST to get this to work
     package_data = {'IPython.UserConfig' : ['*'] }
     return package_data
 
 
 #---------------------------------------------------------------------------
 # Find data files
 #---------------------------------------------------------------------------
 
 def find_data_files():
     """
     Find IPython's data_files.
     """
     
     # I can't find how to make distutils create a nested dir. structure, so
     # in the meantime do it manually. Butt ugly.
     # Note that http://www.redbrick.dcu.ie/~noel/distutils.html, ex. 2/3, contain
     # information on how to do this more cleanly once python 2.4 can be assumed.
     # Thanks to Noel for the tip.
     docdirbase  = 'share/doc/ipython'
     manpagebase = 'share/man/man1'
 
     # We only need to exclude from this things NOT already excluded in the
     # MANIFEST.in file.
     exclude     = ('.sh','.1.gz')
     # We need to figure out how we want to package all of our rst docs?
     # docfiles    = filter(lambda f:file_doesnt_endwith(f,exclude),glob('docs/*'))
     examfiles   = filter(isfile, glob('docs/examples/core/*.py'))
     examfiles.append(filter(isfile, glob('docs/examples/kernel/*.py')))
     manpages    = filter(isfile, glob('docs/man/*.1.gz'))
     igridhelpfiles = filter(isfile, glob('IPython/Extensions/igrid_help.*'))
     
     data_files = [#('data', docdirbase, docfiles),
                  ('data', pjoin(docdirbase, 'examples'),examfiles),
                  ('data', manpagebase, manpages),
                  ('data',pjoin(docdirbase, 'extensions'),igridhelpfiles),
                  ]
-    return data_files
+    # import pprint
+    # pprint.pprint(data_files)
+    return []
 
 #---------------------------------------------------------------------------
 # Find scripts
 #---------------------------------------------------------------------------
 
 def find_scripts():
     """
     Find IPython's scripts.
     """
     scripts = []
     scripts.append('IPython/kernel/scripts/ipengine')
     scripts.append('IPython/kernel/scripts/ipcontroller')
     scripts.append('IPython/kernel/scripts/ipcluster')
     scripts.append('scripts/ipython')
     scripts.append('scripts/pycolor')
     scripts.append('scripts/irunner')
     
     # Script to be run by the windows binary installer after the default setup
     # routine, to add shortcuts and similar windows-only things.  Windows
     # post-install scripts MUST reside in the scripts/ dir, otherwise distutils
     # doesn't find them.
     if 'bdist_wininst' in sys.argv:
         if len(sys.argv) > 2 and ('sdist' in sys.argv or 'bdist_rpm' in sys.argv):
             print >> sys.stderr,"ERROR: bdist_wininst must be run alone. Exiting."
             sys.exit(1)
         scripts.append('scripts/ipython_win_post_install.py')
     
     return scripts
 
 #---------------------------------------------------------------------------
 # Find scripts
 #---------------------------------------------------------------------------
 
 def check_for_dependencies():
     """Check for IPython's dependencies.
     
     This function should NOT be called if running under setuptools!
     """
     from setupext.setupext import (
         print_line, print_raw, print_status, print_message,
         check_for_zopeinterface, check_for_twisted,
         check_for_foolscap, check_for_pyopenssl,
         check_for_sphinx, check_for_pygments,
         check_for_nose, check_for_pexpect
     )
     print_line()
     print_raw("BUILDING IPYTHON")
     print_status('python', sys.version)
     print_status('platform', sys.platform)
     if sys.platform == 'win32':
         print_status('Windows version', sys.getwindowsversion())
     
     print_raw("")
     print_raw("OPTIONAL DEPENDENCIES")
 
     check_for_zopeinterface()
     check_for_twisted()
     check_for_foolscap()
     check_for_pyopenssl()
     check_for_sphinx()
     check_for_pygments()
     check_for_nose()
     check_for_pexpect()
  No newline at end of file