fix handling of aliases/system calls for multiline input

IPython/Release.py

@@ -1,76 +1,76 b''
 # -*- coding: utf-8 -*-
 """Release data for the IPython project.
 
-$Id: Release.py 981 2005-12-30 15:43:43Z fperez $"""
+$Id: Release.py 982 2005-12-30 23:57:07Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2005 Fernando Perez <[e-mail]>
 #
 #       Copyright (c) 2001 Janko Hauser <[e-mail]> and Nathaniel Gray
 #       <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 # Name of the package for release purposes.  This is the name which labels
 # the tarballs and RPMs made by distutils, so it's best to lowercase it.
 name = 'ipython'
 
 # For versions with substrings (like 0.6.16.svn), use an extra . to separate
 # the new substring.  We have to avoid using either dashes or underscores,
 # because bdist_rpm does not accept dashes (an RPM) convention, and
 # bdist_deb does not accept underscores (a Debian convention).
 
-version = '0.7.0.rc2'
+version = '0.7.0.rc3'
 
-revision = '$Revision: 981 $'
+revision = '$Revision: 982 $'
 
 description = "An enhanced interactive Python shell."
 
 long_description = \
 """
 IPython provides a replacement for the interactive Python interpreter with
 extra functionality.
 
 Main features:
 
  * Comprehensive object introspection.
 
  * Input history, persistent across sessions.
 
  * Caching of output results during a session with automatically generated
    references.
 
  * Readline based name completion.
 
  * Extensible system of 'magic' commands for controlling the environment and
    performing many tasks related either to IPython or the operating system.
 
  * Configuration system with easy switching between different setups (simpler
    than changing $PYTHONSTARTUP environment variables every time).
 
  * Session logging and reloading.
 
  * Extensible syntax processing for special purpose situations.
 
  * Access to the system shell with user-extensible alias system.
 
  * Easily embeddable in other Python programs.
 
  * Integrated access to the pdb debugger and the Python profiler. """
 
 license = 'BSD'
 
 authors = {'Fernando' : ('Fernando Perez','[e-mail]'),
            'Janko'    : ('Janko Hauser','[e-mail]'),
            'Nathan'   : ('Nathaniel Gray','[e-mail]')
            }
 
 url = 'http://ipython.scipy.org'
 
 download_url = 'http://ipython.scipy.org/dist'
 
 platforms = ['Linux','Mac OSX','Windows XP/2000/NT','Windows 95/98/ME']
 
 keywords = ['Interactive','Interpreter','Shell']

IPython/iplib.py

@@ -1,2056 +1,2060 b''
 # -*- coding: utf-8 -*-
 """
 IPython -- An enhanced Interactive Python
 
 Requires Python 2.1 or newer.
 
 This file contains all the classes and helper functions specific to IPython.
 
-$Id: iplib.py 978 2005-12-30 02:37:15Z fperez $
+$Id: iplib.py 982 2005-12-30 23:57:07Z fperez $
 """
 
 #*****************************************************************************
 #       Copyright (C) 2001 Janko Hauser <[e-mail]> and
 #       Copyright (C) 2001-2005 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #
 # Note: this code originally subclassed code.InteractiveConsole from the
 # Python standard library.  Over time, all of that class has been copied
 # verbatim here for modifications which could not be accomplished by
 # subclassing.  At this point, there are no dependencies at all on the code
 # module anymore (it is not even imported).  The Python License (sec. 2)
 # allows for this, but it's always nice to acknowledge credit where credit is
 # due.
 #*****************************************************************************
 
 #****************************************************************************
 # Modules and globals
 
 from __future__ import generators  # for 2.2 backwards-compatibility
 
 from IPython import Release
 __author__  = '%s <%s>\n%s <%s>' % \
               ( Release.authors['Janko'] + Release.authors['Fernando'] )
 __license__ = Release.license
 __version__ = Release.version
 
 # Python standard modules
 import __main__
 import __builtin__
 import StringIO
 import bdb
 import cPickle as pickle
 import codeop
 import exceptions
 import glob
 import inspect
 import keyword
 import new
 import os
 import pdb
 import pydoc
 import re
 import shutil
 import string
 import sys
 import traceback
 import types
 
 from pprint import pprint, pformat
 
 # IPython's own modules
 import IPython
 from IPython import OInspect,PyColorize,ultraTB
 from IPython.ColorANSI import ColorScheme,ColorSchemeTable  # too long names
 from IPython.FakeModule import FakeModule
 from IPython.Itpl import Itpl,itpl,printpl,ItplNS,itplns
 from IPython.Logger import Logger
 from IPython.Magic import Magic
 from IPython.Prompts import CachedOutput
 from IPython.Struct import Struct
 from IPython.background_jobs import BackgroundJobManager
 from IPython.usage import cmd_line_usage,interactive_usage
 from IPython.genutils import *
 
 # store the builtin raw_input globally, and use this always, in case user code
 # overwrites it (like wx.py.PyShell does)
 raw_input_original = raw_input
 
 # compiled regexps for autoindent management
 ini_spaces_re = re.compile(r'^(\s+)')
 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
 
 #****************************************************************************
 # Some utility function definitions
 
 def softspace(file, newvalue):
     """Copied from code.py, to remove the dependency"""
     oldvalue = 0
     try:
         oldvalue = file.softspace
     except AttributeError:
         pass
     try:
         file.softspace = newvalue
     except (AttributeError, TypeError):
         # "attribute-less object" or "read-only attributes"
         pass
     return oldvalue
 
 #****************************************************************************
 # These special functions get installed in the builtin namespace, to provide
 # programmatic (pure python) access to magics, aliases and system calls.  This
 # is important for logging, user scripting, and more.
 
 # We are basically exposing, via normal python functions, the three mechanisms
 # in which ipython offers special call modes (magics for internal control,
 # aliases for direct system access via pre-selected names, and !cmd for
 # calling arbitrary system commands).
 
 def ipmagic(arg_s):
     """Call a magic function by name.
 
     Input: a string containing the name of the magic function to call and any
     additional arguments to be passed to the magic.
 
     ipmagic('name -opt foo bar') is equivalent to typing at the ipython
     prompt:
 
     In[1]: %name -opt foo bar
 
     To call a magic without arguments, simply use ipmagic('name').
 
     This provides a proper Python function to call IPython's magics in any
     valid Python code you can type at the interpreter, including loops and
     compound statements.  It is added by IPython to the Python builtin
     namespace upon initialization."""
 
     args = arg_s.split(' ',1)
     magic_name = args[0]
     if magic_name.startswith(__IPYTHON__.ESC_MAGIC):
         magic_name = magic_name[1:]
     try:
         magic_args = args[1]
     except IndexError:
         magic_args = ''
     fn = getattr(__IPYTHON__,'magic_'+magic_name,None)
     if fn is None:
         error("Magic function `%s` not found." % magic_name)
     else:
         magic_args = __IPYTHON__.var_expand(magic_args)
         return fn(magic_args)
 
 def ipalias(arg_s):
     """Call an alias by name.
 
     Input: a string containing the name of the alias to call and any
     additional arguments to be passed to the magic.
 
     ipalias('name -opt foo bar') is equivalent to typing at the ipython
     prompt:
 
     In[1]: name -opt foo bar
 
     To call an alias without arguments, simply use ipalias('name').
 
     This provides a proper Python function to call IPython's aliases in any
     valid Python code you can type at the interpreter, including loops and
     compound statements.  It is added by IPython to the Python builtin
     namespace upon initialization."""
 
     args = arg_s.split(' ',1)
     alias_name = args[0]
     try:
         alias_args = args[1]
     except IndexError:
         alias_args = ''
     if alias_name in __IPYTHON__.alias_table:
         __IPYTHON__.call_alias(alias_name,alias_args)
     else:
         error("Alias `%s` not found." % alias_name)
 
 def ipsystem(arg_s):
     """Make a system call, using IPython."""
     __IPYTHON__.system(arg_s)
 
 
 #****************************************************************************
 # Local use exceptions
 class SpaceInInput(exceptions.Exception): pass
 
 #****************************************************************************
 # Local use classes
 class Bunch: pass
 
 class InputList(list):
     """Class to store user input.
 
     It's basically a list, but slices return a string instead of a list, thus
     allowing things like (assuming 'In' is an instance):
 
     exec In[4:7]
 
     or
 
     exec In[5:9] + In[14] + In[21:25]"""
 
     def __getslice__(self,i,j):
         return ''.join(list.__getslice__(self,i,j))
 
 class SyntaxTB(ultraTB.ListTB):
     """Extension which holds some state: the last exception value"""
 
     def __init__(self,color_scheme = 'NoColor'):
         ultraTB.ListTB.__init__(self,color_scheme)
         self.last_syntax_error = None
 
     def __call__(self, etype, value, elist):
         self.last_syntax_error = value
         ultraTB.ListTB.__call__(self,etype,value,elist)
 
     def clear_err_state(self):
         """Return the current error state and clear it"""
         e = self.last_syntax_error
         self.last_syntax_error = None
         return e
 
 #****************************************************************************
 # Main IPython class
 
 # FIXME: the Magic class is a mixin for now, and will unfortunately remain so
 # until a full rewrite is made.  I've cleaned all cross-class uses of
 # attributes and methods, but too much user code out there relies on the
 # equlity %foo == __IP.magic_foo, so I can't actually remove the mixin usage.
 #
 # But at least now, all the pieces have been separated and we could, in
 # principle, stop using the mixin.  This will ease the transition to the
 # chainsaw branch.
 
 # For reference, the following is the list of 'self.foo' uses in the Magic
 # class as of 2005-12-28.  These are names we CAN'T use in the main ipython
 # class, to prevent clashes.
 
 # ['self.__class__', 'self.__dict__', 'self._inspect', 'self._ofind',
 #  'self.arg_err', 'self.extract_input', 'self.format_', 'self.lsmagic',
 #  'self.magic_', 'self.options_table', 'self.parse', 'self.shell',
 #  'self.value']
 
 class InteractiveShell(Magic):
     """An enhanced console for Python."""
 
     # class attribute to indicate whether the class supports threads or not.
     # Subclasses with thread support should override this as needed.
     isthreaded = False
 
     def __init__(self,name,usage=None,rc=Struct(opts=None,args=None),
                  user_ns = None,user_global_ns=None,banner2='',
                  custom_exceptions=((),None),embedded=False):
 
         # some minimal strict typechecks.  For some core data structures, I
         # want actual basic python types, not just anything that looks like
         # one.  This is especially true for namespaces.
         for ns in (user_ns,user_global_ns):
             if ns is not None and type(ns) != types.DictType:
                 raise TypeError,'namespace must be a dictionary'
 
         # Put a reference to self in builtins so that any form of embedded or
         # imported code can test for being inside IPython.
         __builtin__.__IPYTHON__ = self
 
         # And load into builtins ipmagic/ipalias/ipsystem as well
         __builtin__.ipmagic  = ipmagic
         __builtin__.ipalias  = ipalias
         __builtin__.ipsystem = ipsystem
 
         # Add to __builtin__ other parts of IPython's public API
         __builtin__.ip_set_hook = self.set_hook
 
         # Keep in the builtins a flag for when IPython is active.  We set it
         # with setdefault so that multiple nested IPythons don't clobber one
         # another.  Each will increase its value by one upon being activated,
         # which also gives us a way to determine the nesting level.
         __builtin__.__dict__.setdefault('__IPYTHON__active',0)
 
         # Do the intuitively correct thing for quit/exit: we remove the
         # builtins if they exist, and our own prefilter routine will handle
         # these special cases
         try:
             del __builtin__.exit, __builtin__.quit
         except AttributeError:
             pass
 
         # Store the actual shell's name
         self.name = name
 
         # We need to know whether the instance is meant for embedding, since
         # global/local namespaces need to be handled differently in that case
         self.embedded = embedded
 
         # command compiler
         self.compile = codeop.CommandCompiler()
 
         # User input buffer
         self.buffer = []
 
         # Default name given in compilation of code
         self.filename = '<ipython console>'
 
         # Create the namespace where the user will operate.  user_ns is
         # normally the only one used, and it is passed to the exec calls as
         # the locals argument.  But we do carry a user_global_ns namespace
         # given as the exec 'globals' argument,  This is useful in embedding
         # situations where the ipython shell opens in a context where the
         # distinction between locals and globals is meaningful.
 
         # FIXME. For some strange reason, __builtins__ is showing up at user
         # level as a dict instead of a module. This is a manual fix, but I
         # should really track down where the problem is coming from. Alex
         # Schmolck reported this problem first.
 
         # A useful post by Alex Martelli on this topic:
         # Re: inconsistent value from __builtins__
         # Von: Alex Martelli <[e-mail]>
         # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
         # Gruppen: comp.lang.python
 
         # Michael Hohn <[e-mail]> wrote:
         # > >>> print type(builtin_check.get_global_binding('__builtins__'))
         # > <type 'dict'>
         # > >>> print type(__builtins__)
         # > <type 'module'>
         # > Is this difference in return value intentional?
 
         # Well, it's documented that '__builtins__' can be either a dictionary
         # or a module, and it's been that way for a long time. Whether it's
         # intentional (or sensible), I don't know. In any case, the idea is
         # that if you need to access the built-in namespace directly, you
         # should start with "import __builtin__" (note, no 's') which will
         # definitely give you a module. Yeah, it's somewhat confusing:-(.
         
         if user_ns is None:
             # Set __name__ to __main__ to better match the behavior of the
             # normal interpreter.
             user_ns = {'__name__'     :'__main__',
                        '__builtins__' : __builtin__,
                        }
             
         if user_global_ns is None:
             user_global_ns = {}
 
         # Assign namespaces
         # This is the namespace where all normal user variables live
         self.user_ns = user_ns
         # Embedded instances require a separate namespace for globals.
         # Normally this one is unused by non-embedded instances.
         self.user_global_ns = user_global_ns
         # A namespace to keep track of internal data structures to prevent
         # them from cluttering user-visible stuff.  Will be updated later
         self.internal_ns = {}
 
         # Namespace of system aliases.  Each entry in the alias
         # table must be a 2-tuple of the form (N,name), where N is the number
         # of positional arguments of the alias.
         self.alias_table = {}
 
         # A table holding all the namespaces IPython deals with, so that
         # introspection facilities can search easily.
         self.ns_table = {'user':user_ns,
                          'user_global':user_global_ns,
                          'alias':self.alias_table,
                          'internal':self.internal_ns,
                          'builtin':__builtin__.__dict__
                          }
 
         # The user namespace MUST have a pointer to the shell itself.
         self.user_ns[name] = self
 
         # We need to insert into sys.modules something that looks like a
         # module but which accesses the IPython namespace, for shelve and
         # pickle to work interactively. Normally they rely on getting
         # everything out of __main__, but for embedding purposes each IPython
         # instance has its own private namespace, so we can't go shoving
         # everything into __main__.
 
         # note, however, that we should only do this for non-embedded
         # ipythons, which really mimic the __main__.__dict__ with their own
         # namespace.  Embedded instances, on the other hand, should not do
         # this because they need to manage the user local/global namespaces
         # only, but they live within a 'normal' __main__ (meaning, they
         # shouldn't overtake the execution environment of the script they're
         # embedded in).
 
         if not embedded:
             try:
                 main_name = self.user_ns['__name__']
             except KeyError:
                 raise KeyError,'user_ns dictionary MUST have a "__name__" key'
             else:
                 #print "pickle hack in place"  # dbg
                 sys.modules[main_name] = FakeModule(self.user_ns)
 
         # List of input with multi-line handling.
         # Fill its zero entry, user counter starts at 1
         self.input_hist = InputList(['\n'])
 
         # list of visited directories
         try:
             self.dir_hist = [os.getcwd()]
         except IOError, e:
             self.dir_hist = []
 
         # dict of output history
         self.output_hist = {}
 
         # dict of things NOT to alias (keywords, builtins and some magics)
         no_alias = {}
         no_alias_magics = ['cd','popd','pushd','dhist','alias','unalias']
         for key in keyword.kwlist + no_alias_magics:
             no_alias[key] = 1
         no_alias.update(__builtin__.__dict__)
         self.no_alias = no_alias
                 
         # make global variables for user access to these
         self.user_ns['_ih'] = self.input_hist
         self.user_ns['_oh'] = self.output_hist
         self.user_ns['_dh'] = self.dir_hist
 
         # user aliases to input and output histories
         self.user_ns['In']  = self.input_hist
         self.user_ns['Out'] = self.output_hist
 
         # Object variable to store code object waiting execution.  This is
         # used mainly by the multithreaded shells, but it can come in handy in
         # other situations.  No need to use a Queue here, since it's a single
         # item which gets cleared once run.
         self.code_to_run = None
         
         # Job manager (for jobs run as background threads)
         self.jobs = BackgroundJobManager()
         # Put the job manager into builtins so it's always there.
         __builtin__.jobs = self.jobs
 
         # escapes for automatic behavior on the command line
         self.ESC_SHELL  = '!'
         self.ESC_HELP   = '?'
         self.ESC_MAGIC  = '%'
         self.ESC_QUOTE  = ','
         self.ESC_QUOTE2 = ';'
         self.ESC_PAREN  = '/'
 
         # And their associated handlers
         self.esc_handlers = {self.ESC_PAREN  : self.handle_auto,
                              self.ESC_QUOTE  : self.handle_auto,
                              self.ESC_QUOTE2 : self.handle_auto,
                              self.ESC_MAGIC  : self.handle_magic,
                              self.ESC_HELP   : self.handle_help,
                              self.ESC_SHELL  : self.handle_shell_escape,
                              }
 
         # class initializations
         Magic.__init__(self,self)
 
         # Python source parser/formatter for syntax highlighting
         pyformat = PyColorize.Parser().format
         self.pycolorize = lambda src: pyformat(src,'str',self.rc['colors'])
 
         # hooks holds pointers used for user-side customizations
         self.hooks = Struct()
         
         # Set all default hooks, defined in the IPython.hooks module.
         hooks = IPython.hooks
         for hook_name in hooks.__all__:
             self.set_hook(hook_name,getattr(hooks,hook_name))
 
         # Flag to mark unconditional exit
         self.exit_now = False
 
         self.usage_min =  """\
         An enhanced console for Python.
         Some of its features are:
         - Readline support if the readline library is present.
         - Tab completion in the local namespace.
         - Logging of input, see command-line options.
         - System shell escape via ! , eg !ls.
         - Magic commands, starting with a % (like %ls, %pwd, %cd, etc.)
         - Keeps track of locally defined variables via %who, %whos.
         - Show object information with a ? eg ?x or x? (use ?? for more info).
         """
         if usage: self.usage = usage
         else: self.usage = self.usage_min
 
         # Storage
         self.rc = rc   # This will hold all configuration information
         self.pager = 'less'
         # temporary files used for various purposes.  Deleted at exit.
         self.tempfiles = []
 
         # Keep track of readline usage (later set by init_readline)
         self.has_readline = False
 
         # template for logfile headers.  It gets resolved at runtime by the
         # logstart method.
         self.loghead_tpl = \
 """#log# Automatic Logger file. *** THIS MUST BE THE FIRST LINE ***
 #log# DO NOT CHANGE THIS LINE OR THE TWO BELOW
 #log# opts = %s
 #log# args = %s
 #log# It is safe to make manual edits below here.
 #log#-----------------------------------------------------------------------
 """
         # for pushd/popd management
         try:
             self.home_dir = get_home_dir()
         except HomeDirError,msg:
             fatal(msg)
 
         self.dir_stack = [os.getcwd().replace(self.home_dir,'~')]
 
         # Functions to call the underlying shell.
 
         # utility to expand user variables via Itpl
         self.var_expand = lambda cmd: str(ItplNS(cmd.replace('#','\#'),
                                                  self.user_ns))
         # The first is similar to os.system, but it doesn't return a value,
         # and it allows interpolation of variables in the user's namespace.
         self.system = lambda cmd: shell(self.var_expand(cmd),
                                         header='IPython system call: ',
                                         verbose=self.rc.system_verbose)
         # These are for getoutput and getoutputerror:
         self.getoutput = lambda cmd: \
                          getoutput(self.var_expand(cmd),
                                    header='IPython system call: ',
                                    verbose=self.rc.system_verbose)
         self.getoutputerror = lambda cmd: \
                               getoutputerror(str(ItplNS(cmd.replace('#','\#'),
                                                         self.user_ns)),
                                              header='IPython system call: ',
                                              verbose=self.rc.system_verbose)
  
         # RegExp for splitting line contents into pre-char//first
         # word-method//rest.  For clarity, each group in on one line.
 
         # WARNING: update the regexp if the above escapes are changed, as they
         # are hardwired in.
 
         # Don't get carried away with trying to make the autocalling catch too
         # much:  it's better to be conservative rather than to trigger hidden
         # evals() somewhere and end up causing side effects.
 
         self.line_split = re.compile(r'^([\s*,;/])'
                                      r'([\?\w\.]+\w*\s*)'
                                      r'(\(?.*$)')
 
         # Original re, keep around for a while in case changes break something
         #self.line_split = re.compile(r'(^[\s*!\?%,/]?)'
         #                             r'(\s*[\?\w\.]+\w*\s*)'
         #                             r'(\(?.*$)')
 
         # RegExp to identify potential function names
         self.re_fun_name = re.compile(r'[a-zA-Z_]([a-zA-Z0-9_.]*) *$')
         # RegExp to exclude strings with this start from autocalling
         self.re_exclude_auto = re.compile('^[!=()<>,\*/\+-]|^is ')
 
         # try to catch also methods for stuff in lists/tuples/dicts: off
         # (experimental). For this to work, the line_split regexp would need
         # to be modified so it wouldn't break things at '['. That line is
         # nasty enough that I shouldn't change it until I can test it _well_.
         #self.re_fun_name = re.compile (r'[a-zA-Z_]([a-zA-Z0-9_.\[\]]*) ?$')
 
         # keep track of where we started running (mainly for crash post-mortem)
         self.starting_dir = os.getcwd()
 
         # Various switches which can be set
         self.CACHELENGTH = 5000  # this is cheap, it's just text
         self.BANNER = "Python %(version)s on %(platform)s\n" % sys.__dict__
         self.banner2 = banner2
 
         # TraceBack handlers:
 
         # Syntax error handler.
         self.SyntaxTB = SyntaxTB(color_scheme='NoColor')
         
         # The interactive one is initialized with an offset, meaning we always
         # want to remove the topmost item in the traceback, which is our own
         # internal code. Valid modes: ['Plain','Context','Verbose']
         self.InteractiveTB = ultraTB.AutoFormattedTB(mode = 'Plain',
                                                      color_scheme='NoColor',
                                                      tb_offset = 1)
 
         # IPython itself shouldn't crash. This will produce a detailed
         # post-mortem if it does.  But we only install the crash handler for
         # non-threaded shells, the threaded ones use a normal verbose reporter
         # and lose the crash handler.  This is because exceptions in the main
         # thread (such as in GUI code) propagate directly to sys.excepthook,
         # and there's no point in printing crash dumps for every user exception.
         if self.isthreaded:
             sys.excepthook = ultraTB.FormattedTB()
         else:
             from IPython import CrashHandler
             sys.excepthook = CrashHandler.CrashHandler(self)
 
         # The instance will store a pointer to this, so that runtime code
         # (such as magics) can access it.  This is because during the
         # read-eval loop, it gets temporarily overwritten (to deal with GUI
         # frameworks).
         self.sys_excepthook = sys.excepthook
 
         # and add any custom exception handlers the user may have specified
         self.set_custom_exc(*custom_exceptions)
 
         # Object inspector
         self.inspector = OInspect.Inspector(OInspect.InspectColors,
                                             PyColorize.ANSICodeColors,
                                             'NoColor')
         # indentation management
         self.autoindent = False
         self.indent_current_nsp = 0
         self.indent_current = '' # actual indent string
 
         # Make some aliases automatically
         # Prepare list of shell aliases to auto-define
         if os.name == 'posix':            
             auto_alias = ('mkdir mkdir', 'rmdir rmdir',
                           'mv mv -i','rm rm -i','cp cp -i',
                           'cat cat','less less','clear clear',
                           # a better ls
                           'ls ls -F',
                           # long ls
                           'll ls -lF',
                           # color ls
                           'lc ls -F -o --color',
                           # ls normal files only
                           'lf ls -F -o --color %l | grep ^-',
                           # ls symbolic links 
                           'lk ls -F -o --color %l | grep ^l',
                           # directories or links to directories,
                           'ldir ls -F -o --color %l | grep /$',
                           # things which are executable
                           'lx ls -F -o --color %l | grep ^-..x',
                           )
         elif os.name in ['nt','dos']:
             auto_alias = ('dir dir /on', 'ls dir /on',
                           'ddir dir /ad /on', 'ldir dir /ad /on',
                           'mkdir mkdir','rmdir rmdir','echo echo',
                           'ren ren','cls cls','copy copy')
         else:
             auto_alias = ()
         self.auto_alias = map(lambda s:s.split(None,1),auto_alias)
         # Call the actual (public) initializer
         self.init_auto_alias()
     # end __init__
 
     def post_config_initialization(self):
         """Post configuration init method
 
         This is called after the configuration files have been processed to
         'finalize' the initialization."""
 
         rc = self.rc
         
         # Load readline proper
         if rc.readline:
             self.init_readline()
 
         # log system
         self.logger = Logger(self,logfname='ipython_log.py',logmode='rotate')
         # local shortcut, this is used a LOT
         self.log = self.logger.log
 
         # Initialize cache, set in/out prompts and printing system
         self.outputcache = CachedOutput(self,
                                         rc.cache_size,
                                         rc.pprint,
                                         input_sep = rc.separate_in,
                                         output_sep = rc.separate_out,
                                         output_sep2 = rc.separate_out2,
                                         ps1 = rc.prompt_in1,
                                         ps2 = rc.prompt_in2,
                                         ps_out = rc.prompt_out,
                                         pad_left = rc.prompts_pad_left)
 
         # user may have over-ridden the default print hook:
         try:
             self.outputcache.__class__.display = self.hooks.display
         except AttributeError:
             pass
 
         # I don't like assigning globally to sys, because it means when embedding
         # instances, each embedded instance overrides the previous choice. But
         # sys.displayhook seems to be called internally by exec, so I don't see a
         # way around it.
         sys.displayhook = self.outputcache
 
         # Set user colors (don't do it in the constructor above so that it
         # doesn't crash if colors option is invalid)
         self.magic_colors(rc.colors)
 
         # Set calling of pdb on exceptions
         self.call_pdb = rc.pdb
 
         # Load user aliases
         for alias in rc.alias:
             self.magic_alias(alias)
 
         # dynamic data that survives through sessions
         # XXX make the filename a config option?
         persist_base = 'persist'
         if rc.profile:
             persist_base += '_%s' % rc.profile
         self.persist_fname =  os.path.join(rc.ipythondir,persist_base)
 
         try:
             self.persist = pickle.load(file(self.persist_fname))
         except:
             self.persist = {}
         
         
         for (key, value) in [(k[2:],v) for (k,v) in self.persist.items() if k.startswith('S:')]:
             try:
                 obj = pickle.loads(value)
             except:
                 
                 print "Unable to restore variable '%s', ignoring (use %%store -d to forget!)" % key
                 print "The error was:",sys.exc_info()[0]
                 continue
                 
             
             self.user_ns[key] = obj
             
           
         
             
     def set_hook(self,name,hook):
         """set_hook(name,hook) -> sets an internal IPython hook.
 
         IPython exposes some of its internal API as user-modifiable hooks.  By
         resetting one of these hooks, you can modify IPython's behavior to
         call at runtime your own routines."""
 
         # At some point in the future, this should validate the hook before it
         # accepts it.  Probably at least check that the hook takes the number
         # of args it's supposed to.
         setattr(self.hooks,name,new.instancemethod(hook,self,self.__class__))
 
     def set_custom_exc(self,exc_tuple,handler):
         """set_custom_exc(exc_tuple,handler)
 
         Set a custom exception handler, which will be called if any of the
         exceptions in exc_tuple occur in the mainloop (specifically, in the
         runcode() method.
 
         Inputs:
 
           - exc_tuple: a *tuple* of valid exceptions to call the defined
           handler for.  It is very important that you use a tuple, and NOT A
           LIST here, because of the way Python's except statement works.  If
           you only want to trap a single exception, use a singleton tuple:
 
             exc_tuple == (MyCustomException,)
 
           - handler: this must be defined as a function with the following
           basic interface: def my_handler(self,etype,value,tb).
 
           This will be made into an instance method (via new.instancemethod)
           of IPython itself, and it will be called if any of the exceptions
           listed in the exc_tuple are caught.  If the handler is None, an
           internal basic one is used, which just prints basic info.
 
         WARNING: by putting in your own exception handler into IPython's main
         execution loop, you run a very good chance of nasty crashes.  This
         facility should only be used if you really know what you are doing."""
 
         assert type(exc_tuple)==type(()) , \
                "The custom exceptions must be given AS A TUPLE."
 
         def dummy_handler(self,etype,value,tb):
             print '*** Simple custom exception handler ***'
             print 'Exception type :',etype
             print 'Exception value:',value
             print 'Traceback      :',tb
             print 'Source code    :','\n'.join(self.buffer)
 
         if handler is None: handler = dummy_handler
 
         self.CustomTB = new.instancemethod(handler,self,self.__class__)
         self.custom_exceptions = exc_tuple
 
     def set_custom_completer(self,completer,pos=0):
         """set_custom_completer(completer,pos=0)
 
         Adds a new custom completer function.
 
         The position argument (defaults to 0) is the index in the completers
         list where you want the completer to be inserted."""
 
         newcomp = new.instancemethod(completer,self.Completer,
                                      self.Completer.__class__)
         self.Completer.matchers.insert(pos,newcomp)
 
     def _get_call_pdb(self):
         return self._call_pdb
 
     def _set_call_pdb(self,val):
 
         if val not in (0,1,False,True):
             raise ValueError,'new call_pdb value must be boolean'
 
         # store value in instance
         self._call_pdb = val
 
         # notify the actual exception handlers
         self.InteractiveTB.call_pdb = val
         if self.isthreaded:
             try:
                 self.sys_excepthook.call_pdb = val
             except:
                 warn('Failed to activate pdb for threaded exception handler')
 
     call_pdb = property(_get_call_pdb,_set_call_pdb,None,
                         'Control auto-activation of pdb at exceptions')
  
     def complete(self,text):
         """Return a sorted list of all possible completions on text.
 
         Inputs:
 
           - text: a string of text to be completed on.
 
         This is a wrapper around the completion mechanism, similar to what
         readline does at the command line when the TAB key is hit.  By
         exposing it as a method, it can be used by other non-readline
         environments (such as GUIs) for text completion.
 
         Simple usage example:
 
         In [1]: x = 'hello'
 
         In [2]: __IP.complete('x.l')
         Out[2]: ['x.ljust', 'x.lower', 'x.lstrip']"""
         
         complete = self.Completer.complete
         state = 0
         # use a dict so we get unique keys, since ipyhton's multiple
         # completers can return duplicates.
         comps = {}
         while True:
             newcomp = complete(text,state)
             if newcomp is None:
                 break
             comps[newcomp] = 1
             state += 1
         outcomps = comps.keys()
         outcomps.sort()
         return outcomps
         
     def set_completer_frame(self, frame):
         if frame:
             self.Completer.namespace = frame.f_locals
             self.Completer.global_namespace = frame.f_globals
         else:
             self.Completer.namespace = self.user_ns
             self.Completer.global_namespace = self.user_global_ns
 
     def init_auto_alias(self):
         """Define some aliases automatically.
 
         These are ALL parameter-less aliases"""
         for alias,cmd in self.auto_alias:
             self.alias_table[alias] = (0,cmd)
 
     def alias_table_validate(self,verbose=0):
         """Update information about the alias table.
 
         In particular, make sure no Python keywords/builtins are in it."""
 
         no_alias = self.no_alias
         for k in self.alias_table.keys():
             if k in no_alias:
                 del self.alias_table[k]
                 if verbose:
                     print ("Deleting alias <%s>, it's a Python "
                            "keyword or builtin." % k)
     
     def set_autoindent(self,value=None):
         """Set the autoindent flag, checking for readline support.
 
         If called with no arguments, it acts as a toggle."""
 
         if not self.has_readline:
             if os.name == 'posix':
                 warn("The auto-indent feature requires the readline library")
             self.autoindent = 0
             return
         if value is None:
             self.autoindent = not self.autoindent
         else:
             self.autoindent = value
 
     def rc_set_toggle(self,rc_field,value=None):
         """Set or toggle a field in IPython's rc config. structure.
 
         If called with no arguments, it acts as a toggle.
 
         If called with a non-existent field, the resulting AttributeError
         exception will propagate out."""
 
         rc_val = getattr(self.rc,rc_field)
         if value is None:
             value = not rc_val
         setattr(self.rc,rc_field,value)
 
     def user_setup(self,ipythondir,rc_suffix,mode='install'):
         """Install the user configuration directory.
 
         Can be called when running for the first time or to upgrade the user's
         .ipython/ directory with the mode parameter. Valid modes are 'install'
         and 'upgrade'."""
 
         def wait():
             try:
                 raw_input("Please press <RETURN> to start IPython.")
             except EOFError:
                 print >> Term.cout
             print '*'*70
 
         cwd = os.getcwd()  # remember where we started
         glb = glob.glob
         print '*'*70
         if mode == 'install':
             print \
 """Welcome to IPython. I will try to create a personal configuration directory
 where you can customize many aspects of IPython's functionality in:\n"""
         else:
             print 'I am going to upgrade your configuration in:'
 
         print ipythondir
 
         rcdirend = os.path.join('IPython','UserConfig')
         cfg = lambda d: os.path.join(d,rcdirend)
         try:
             rcdir = filter(os.path.isdir,map(cfg,sys.path))[0]
         except IOError:
             warning = """
 Installation error. IPython's directory was not found.
 
 Check the following:
 
 The ipython/IPython directory should be in a directory belonging to your
 PYTHONPATH environment variable (that is, it should be in a directory
 belonging to sys.path). You can copy it explicitly there or just link to it.
 
 IPython will proceed with builtin defaults.
 """
             warn(warning)
             wait()
             return
 
         if mode == 'install':
             try:
                 shutil.copytree(rcdir,ipythondir)
                 os.chdir(ipythondir)
                 rc_files = glb("ipythonrc*")
                 for rc_file in rc_files:
                     os.rename(rc_file,rc_file+rc_suffix)
             except:
                 warning = """
 
 There was a problem with the installation:
 %s
 Try to correct it or contact the developers if you think it's a bug.
 IPython will proceed with builtin defaults.""" % sys.exc_info()[1]
                 warn(warning)
                 wait()
                 return
 
         elif mode == 'upgrade':
             try:
                 os.chdir(ipythondir)
             except:
                 print """
 Can not upgrade: changing to directory %s failed. Details:
 %s
 """ % (ipythondir,sys.exc_info()[1])
                 wait()
                 return
             else:
                 sources = glb(os.path.join(rcdir,'[A-Za-z]*'))
                 for new_full_path in sources:
                     new_filename = os.path.basename(new_full_path)
                     if new_filename.startswith('ipythonrc'):
                         new_filename = new_filename + rc_suffix
                     # The config directory should only contain files, skip any
                     # directories which may be there (like CVS)
                     if os.path.isdir(new_full_path):
                         continue
                     if os.path.exists(new_filename):
                         old_file = new_filename+'.old'
                         if os.path.exists(old_file):
                             os.remove(old_file)
                         os.rename(new_filename,old_file)
                     shutil.copy(new_full_path,new_filename)
         else:
             raise ValueError,'unrecognized mode for install:',`mode`
 
         # Fix line-endings to those native to each platform in the config
         # directory.
         try:
             os.chdir(ipythondir)
         except:
             print """
 Problem: changing to directory %s failed.
 Details:
 %s
 
 Some configuration files may have incorrect line endings.  This should not
 cause any problems during execution.  """ % (ipythondir,sys.exc_info()[1])
             wait()
         else:
             for fname in glb('ipythonrc*'):
                 try:
                     native_line_ends(fname,backup=0)
                 except IOError:
                     pass
 
         if mode == 'install':
             print """
 Successful installation!
 
 Please read the sections 'Initial Configuration' and 'Quick Tips' in the
 IPython manual (there are both HTML and PDF versions supplied with the
 distribution) to make sure that your system environment is properly configured
 to take advantage of IPython's features."""
         else:
             print """
 Successful upgrade!
 
 All files in your directory:
 %(ipythondir)s
 which would have been overwritten by the upgrade were backed up with a .old
 extension.  If you had made particular customizations in those files you may
 want to merge them back into the new files.""" % locals()
         wait()
         os.chdir(cwd)
         # end user_setup()
 
     def atexit_operations(self):
         """This will be executed at the time of exit.
 
         Saving of persistent data should be performed here. """
 
         # input history
         self.savehist()
 
         # Cleanup all tempfiles left around
         for tfile in self.tempfiles:
             try:
                 os.unlink(tfile)
             except OSError:
                 pass
 
         # save the "persistent data" catch-all dictionary
         try:
             pickle.dump(self.persist, open(self.persist_fname,"w"))
         except:
             print "*** ERROR *** persistent data saving failed."
         
     def savehist(self):
         """Save input history to a file (via readline library)."""
         try:
             self.readline.write_history_file(self.histfile)
         except:
             print 'Unable to save IPython command history to file: ' + \
                   `self.histfile`
 
     def pre_readline(self):
         """readline hook to be used at the start of each line.
 
         Currently it handles auto-indent only."""
 
         self.readline.insert_text(self.indent_current)
 
     def init_readline(self):
         """Command history completion/saving/reloading."""
         try:
             import readline
         except ImportError:
             self.has_readline = 0
             self.readline = None
             # no point in bugging windows users with this every time:
             if os.name == 'posix':
                 warn('Readline services not available on this platform.')
         else:
             import atexit
             from IPython.completer import IPCompleter
             self.Completer = IPCompleter(self,
                                             self.user_ns,
                                             self.user_global_ns,
                                             self.rc.readline_omit__names,
                                             self.alias_table)
 
             # Platform-specific configuration
             if os.name == 'nt':
                 self.readline_startup_hook = readline.set_pre_input_hook
             else:
                 self.readline_startup_hook = readline.set_startup_hook
 
             # Load user's initrc file (readline config)
             inputrc_name = os.environ.get('INPUTRC')
             if inputrc_name is None:
                 home_dir = get_home_dir()
                 if home_dir is not None:
                     inputrc_name = os.path.join(home_dir,'.inputrc')
             if os.path.isfile(inputrc_name):
                 try:
                     readline.read_init_file(inputrc_name)
                 except:
                     warn('Problems reading readline initialization file <%s>'
                          % inputrc_name)
             
             self.has_readline = 1
             self.readline = readline
             # save this in sys so embedded copies can restore it properly
             sys.ipcompleter = self.Completer.complete
             readline.set_completer(self.Completer.complete)
 
             # Configure readline according to user's prefs
             for rlcommand in self.rc.readline_parse_and_bind:
                 readline.parse_and_bind(rlcommand)
 
             # remove some chars from the delimiters list
             delims = readline.get_completer_delims()
             delims = delims.translate(string._idmap,
                                       self.rc.readline_remove_delims)
             readline.set_completer_delims(delims)
             # otherwise we end up with a monster history after a while:
             readline.set_history_length(1000)
             try:
                 #print '*** Reading readline history'  # dbg
                 readline.read_history_file(self.histfile)
             except IOError:
                 pass  # It doesn't exist yet.
 
             atexit.register(self.atexit_operations)
             del atexit
 
         # Configure auto-indent for all platforms
         self.set_autoindent(self.rc.autoindent)
 
     def _should_recompile(self,e):
         """Utility routine for edit_syntax_error"""
         
         if e.filename in ('<ipython console>','<input>','<string>',
                           '<console>'):
             return False
         try:
             if not ask_yes_no('Return to editor to correct syntax error? '
                               '[Y/n] ','y'):
                 return False
         except EOFError:
             return False
         self.hooks.fix_error_editor(e.filename,e.lineno,e.offset,e.msg)
         return True
         
     def edit_syntax_error(self):
         """The bottom half of the syntax error handler called in the main loop.
 
         Loop until syntax error is fixed or user cancels.
         """
 
         while self.SyntaxTB.last_syntax_error:
             # copy and clear last_syntax_error
             err = self.SyntaxTB.clear_err_state()
             if not self._should_recompile(err):
                 return
             try:
                 # may set last_syntax_error again if a SyntaxError is raised
                 self.safe_execfile(err.filename,self.shell.user_ns)
             except:
                 self.showtraceback()
             else:
                 f = file(err.filename)
                 try:
                     sys.displayhook(f.read())
                 finally:
                     f.close()
 
     def showsyntaxerror(self, filename=None):
         """Display the syntax error that just occurred.
 
         This doesn't display a stack trace because there isn't one.
 
         If a filename is given, it is stuffed in the exception instead
         of what was there before (because Python's parser always uses
         "<string>" when reading from a string).
         """
         etype, value, last_traceback = sys.exc_info()
         if filename and etype is SyntaxError:
             # Work hard to stuff the correct filename in the exception
             try:
                 msg, (dummy_filename, lineno, offset, line) = value
             except:
                 # Not the format we expect; leave it alone
                 pass
             else:
                 # Stuff in the right filename
                 try:
                     # Assume SyntaxError is a class exception
                     value = SyntaxError(msg, (filename, lineno, offset, line))
                 except:
                     # If that failed, assume SyntaxError is a string
                     value = msg, (filename, lineno, offset, line)
         self.SyntaxTB(etype,value,[])
 
     def debugger(self):
         """Call the pdb debugger."""
 
         if not self.rc.pdb:
             return
         pdb.pm()
 
     def showtraceback(self,exc_tuple = None,filename=None):
         """Display the exception that just occurred."""
 
         # Though this won't be called by syntax errors in the input line,
         # there may be SyntaxError cases whith imported code.
         if exc_tuple is None:
             type, value, tb = sys.exc_info()
         else:
             type, value, tb = exc_tuple
         if type is SyntaxError:
             self.showsyntaxerror(filename)
         else:
             self.InteractiveTB()
             if self.InteractiveTB.call_pdb and self.has_readline:
                 # pdb mucks up readline, fix it back
                 self.readline.set_completer(self.Completer.complete)
 
     def mainloop(self,banner=None):
         """Creates the local namespace and starts the mainloop.
 
         If an optional banner argument is given, it will override the
         internally created default banner."""
         
         if self.rc.c:  # Emulate Python's -c option
             self.exec_init_cmd()
         if banner is None:
             if self.rc.banner:
                 banner = self.BANNER+self.banner2
             else:
                 banner = ''
         self.interact(banner)
 
     def exec_init_cmd(self):
         """Execute a command given at the command line.
 
         This emulates Python's -c option."""
 
         sys.argv = ['-c']
         self.push(self.rc.c)
 
     def embed_mainloop(self,header='',local_ns=None,global_ns=None,stack_depth=0):
         """Embeds IPython into a running python program.
 
         Input:
 
           - header: An optional header message can be specified.
 
           - local_ns, global_ns: working namespaces. If given as None, the
           IPython-initialized one is updated with __main__.__dict__, so that
           program variables become visible but user-specific configuration
           remains possible.
 
           - stack_depth: specifies how many levels in the stack to go to
           looking for namespaces (when local_ns and global_ns are None).  This
           allows an intermediate caller to make sure that this function gets
           the namespace from the intended level in the stack.  By default (0)
           it will get its locals and globals from the immediate caller.
 
         Warning: it's possible to use this in a program which is being run by
         IPython itself (via %run), but some funny things will happen (a few
         globals get overwritten). In the future this will be cleaned up, as
         there is no fundamental reason why it can't work perfectly."""
 
         # Get locals and globals from caller
         if local_ns is None or global_ns is None:
             call_frame = sys._getframe(stack_depth).f_back
 
             if local_ns is None:
                 local_ns = call_frame.f_locals
             if global_ns is None:
                 global_ns = call_frame.f_globals
 
         # Update namespaces and fire up interpreter
         self.user_ns = local_ns
         self.user_global_ns = global_ns
 
         # Patch for global embedding to make sure that things don't overwrite
         # user globals accidentally. Thanks to Richard <[e-mail]>
         # FIXME. Test this a bit more carefully (the if.. is new)
         if local_ns is None and global_ns is None:
             self.user_global_ns.update(__main__.__dict__)
 
         # make sure the tab-completer has the correct frame information, so it
         # actually completes using the frame's locals/globals
         self.set_completer_frame(call_frame)
         
         self.interact(header)
 
     def interact(self, banner=None):
         """Closely emulate the interactive Python console.
 
         The optional banner argument specify the banner to print
         before the first interaction; by default it prints a banner
         similar to the one printed by the real Python interpreter,
         followed by the current class name in parentheses (so as not
         to confuse this with the real interpreter -- since it's so
         close!).
 
         """
         cprt = 'Type "copyright", "credits" or "license" for more information.'
         if banner is None:
             self.write("Python %s on %s\n%s\n(%s)\n" %
                        (sys.version, sys.platform, cprt,
                         self.__class__.__name__))
         else:
             self.write(banner)
 
         more = 0
         
         # Mark activity in the builtins
         __builtin__.__dict__['__IPYTHON__active'] += 1
 
         # exit_now is set by a call to %Exit or %Quit
         while not self.exit_now:
             try:
                 if more:
                     prompt = self.outputcache.prompt2
                     if self.autoindent:
                         self.readline_startup_hook(self.pre_readline)
                 else:
                     prompt = self.outputcache.prompt1
                 try:
                     line = self.raw_input(prompt,more)
                     if self.autoindent:
                         self.readline_startup_hook(None)
                 except EOFError:
                     if self.autoindent:
                         self.readline_startup_hook(None)
                     self.write("\n")
                     self.exit()
                 else:
                     more = self.push(line)
 
                     if (self.SyntaxTB.last_syntax_error and
                         self.rc.autoedit_syntax):
                         self.edit_syntax_error()
 
             except KeyboardInterrupt:
                 self.write("\nKeyboardInterrupt\n")
                 self.resetbuffer()
                 more = 0
                 # keep cache in sync with the prompt counter:
                 self.outputcache.prompt_count -= 1
 
                 if self.autoindent:
                     self.indent_current_nsp = 0
                     self.indent_current = ' '* self.indent_current_nsp
 
             except bdb.BdbQuit:
                 warn("The Python debugger has exited with a BdbQuit exception.\n"
                      "Because of how pdb handles the stack, it is impossible\n"
                      "for IPython to properly format this particular exception.\n"
                      "IPython will resume normal operation.")
             
         # We are off again...
         __builtin__.__dict__['__IPYTHON__active'] -= 1
 
     def excepthook(self, type, value, tb):
       """One more defense for GUI apps that call sys.excepthook.
 
       GUI frameworks like wxPython trap exceptions and call
       sys.excepthook themselves.  I guess this is a feature that
       enables them to keep running after exceptions that would
       otherwise kill their mainloop. This is a bother for IPython
       which excepts to catch all of the program exceptions with a try:
       except: statement.
 
       Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
       any app directly invokes sys.excepthook, it will look to the user like
       IPython crashed.  In order to work around this, we can disable the
       CrashHandler and replace it with this excepthook instead, which prints a
       regular traceback using our InteractiveTB.  In this fashion, apps which
       call sys.excepthook will generate a regular-looking exception from
       IPython, and the CrashHandler will only be triggered by real IPython
       crashes.
 
       This hook should be used sparingly, only in places which are not likely
       to be true IPython errors.
       """
       
       self.InteractiveTB(type, value, tb, tb_offset=0)
       if self.InteractiveTB.call_pdb and self.has_readline:
           self.readline.set_completer(self.Completer.complete)
 
     def call_alias(self,alias,rest=''):
         """Call an alias given its name and the rest of the line.
 
         This function MUST be given a proper alias, because it doesn't make
         any checks when looking up into the alias table.  The caller is
         responsible for invoking it only with a valid alias."""
 
         #print 'ALIAS: <%s>+<%s>' % (alias,rest) # dbg
         nargs,cmd = self.alias_table[alias]
         # Expand the %l special to be the user's input line
         if cmd.find('%l') >= 0:
             cmd = cmd.replace('%l',rest)
             rest = ''
         if nargs==0:
             # Simple, argument-less aliases
             cmd = '%s %s' % (cmd,rest)
         else:
             # Handle aliases with positional arguments
             args = rest.split(None,nargs)
             if len(args)< nargs:
                 error('Alias <%s> requires %s arguments, %s given.' %
                       (alias,nargs,len(args)))
                 return
             cmd = '%s %s' % (cmd % tuple(args[:nargs]),' '.join(args[nargs:]))
         # Now call the macro, evaluating in the user's namespace
         try:
             self.system(cmd)
         except:
             self.showtraceback()
 
     def autoindent_update(self,line):
         """Keep track of the indent level."""
         if self.autoindent:
             if line:
                 ini_spaces = ini_spaces_re.match(line)
                 if ini_spaces:
                     nspaces = ini_spaces.end()
                 else:
                     nspaces = 0
                 self.indent_current_nsp = nspaces
 
                 if line[-1] == ':':
                     self.indent_current_nsp += 4
                 elif dedent_re.match(line):
                     self.indent_current_nsp -= 4
             else:
                 self.indent_current_nsp = 0
 
             # indent_current is the actual string to be inserted
             # by the readline hooks for indentation
             self.indent_current = ' '* self.indent_current_nsp
 
     def runlines(self,lines):
         """Run a string of one or more lines of source.
 
         This method is capable of running a string containing multiple source
         lines, as if they had been entered at the IPython prompt.  Since it
         exposes IPython's processing machinery, the given strings can contain
         magic calls (%magic), special shell access (!cmd), etc."""
 
         # We must start with a clean buffer, in case this is run from an
         # interactive IPython session (via a magic, for example).
         self.resetbuffer()
         lines = lines.split('\n')
         more = 0
         for line in lines:
             # skip blank lines so we don't mess up the prompt counter, but do
             # NOT skip even a blank line if we are in a code block (more is
             # true)
             if line or more:
                 more = self.push(self.prefilter(line,more))
                 # IPython's runsource returns None if there was an error
                 # compiling the code.  This allows us to stop processing right
                 # away, so the user gets the error message at the right place.
                 if more is None:
                     break
         # final newline in case the input didn't have it, so that the code
         # actually does get executed
         if more:
             self.push('\n')
 
     def runsource(self, source, filename='<input>', symbol='single'):
         """Compile and run some source in the interpreter.
 
         Arguments are as for compile_command().
 
         One several things can happen:
 
         1) The input is incorrect; compile_command() raised an
         exception (SyntaxError or OverflowError).  A syntax traceback
         will be printed by calling the showsyntaxerror() method.
 
         2) The input is incomplete, and more input is required;
         compile_command() returned None.  Nothing happens.
 
         3) The input is complete; compile_command() returned a code
         object.  The code is executed by calling self.runcode() (which
         also handles run-time exceptions, except for SystemExit).
 
         The return value is:
 
           - True in case 2
 
           - False in the other cases, unless an exception is raised, where
           None is returned instead.  This can be used by external callers to
           know whether to continue feeding input or not.
 
         The return value can be used to decide whether to use sys.ps1 or
         sys.ps2 to prompt the next line."""
 
         try:
             code = self.compile(source,filename,symbol)
         except (OverflowError, SyntaxError, ValueError):
             # Case 1
             self.showsyntaxerror(filename)
             return None
 
         if code is None:
             # Case 2
             return True
 
         # Case 3
         # We store the code object so that threaded shells and
         # custom exception handlers can access all this info if needed.
         # The source corresponding to this can be obtained from the
         # buffer attribute as '\n'.join(self.buffer).
         self.code_to_run = code
         # now actually execute the code object
         if self.runcode(code) == 0:
             return False
         else:
             return None
 
     def runcode(self,code_obj):
         """Execute a code object.
 
         When an exception occurs, self.showtraceback() is called to display a
         traceback.
 
         Return value: a flag indicating whether the code to be run completed
         successfully:
 
           - 0: successful execution.
           - 1: an error occurred.
         """
 
         # Set our own excepthook in case the user code tries to call it
         # directly, so that the IPython crash handler doesn't get triggered
         old_excepthook,sys.excepthook = sys.excepthook, self.excepthook
 
         # we save the original sys.excepthook in the instance, in case config
         # code (such as magics) needs access to it.
         self.sys_excepthook = old_excepthook
         outflag = 1  # happens in more places, so it's easier as default
         try:
             try:
                 # Embedded instances require separate global/local namespaces
                 # so they can see both the surrounding (local) namespace and
                 # the module-level globals when called inside another function.
                 if self.embedded:
                     exec code_obj in self.user_global_ns, self.user_ns
                 # Normal (non-embedded) instances should only have a single
                 # namespace for user code execution, otherwise functions won't
                 # see interactive top-level globals.
                 else:
                     exec code_obj in self.user_ns
             finally:
                 # Reset our crash handler in place
                 sys.excepthook = old_excepthook
         except SystemExit:
             self.resetbuffer()
             self.showtraceback()
             warn("Type exit or quit to exit IPython "
                  "(%Exit or %Quit do so unconditionally).",level=1)
         except self.custom_exceptions:
             etype,value,tb = sys.exc_info()
             self.CustomTB(etype,value,tb)
         except:
             self.showtraceback()
         else:
             outflag = 0
             if softspace(sys.stdout, 0):
                 print
         # Flush out code object which has been run (and source)
         self.code_to_run = None
         return outflag
         
     def push(self, line):
         """Push a line to the interpreter.
 
         The line should not have a trailing newline; it may have
         internal newlines.  The line is appended to a buffer and the
         interpreter's runsource() method is called with the
         concatenated contents of the buffer as source.  If this
         indicates that the command was executed or invalid, the buffer
         is reset; otherwise, the command is incomplete, and the buffer
         is left as it was after the line was appended.  The return
         value is 1 if more input is required, 0 if the line was dealt
         with in some way (this is the same as runsource()).
         """
 
         # autoindent management should be done here, and not in the
         # interactive loop, since that one is only seen by keyboard input.  We
         # need this done correctly even for code run via runlines (which uses
         # push).
+
+        print 'push line: <%s>' % line  # dbg
         self.autoindent_update(line)
 
         self.buffer.append(line)
         more = self.runsource('\n'.join(self.buffer), self.filename)
         if not more:
             self.resetbuffer()
         return more
 
     def resetbuffer(self):
         """Reset the input buffer."""
         self.buffer[:] = []
 
     def raw_input(self,prompt='',continue_prompt=False):
         """Write a prompt and read a line.
 
         The returned line does not include the trailing newline.
         When the user enters the EOF key sequence, EOFError is raised.
 
         Optional inputs:
 
           - prompt(''): a string to be printed to prompt the user.
 
           - continue_prompt(False): whether this line is the first one or a
           continuation in a sequence of inputs.
         """
 
         line = raw_input_original(prompt)
         # Try to be reasonably smart about not re-indenting pasted input more
         # than necessary.  We do this by trimming out the auto-indent initial
         # spaces, if the user's actual input started itself with whitespace.
         if self.autoindent:
             line2 = line[self.indent_current_nsp:]
             if line2[0:1] in (' ','\t'):
                 line = line2
         return self.prefilter(line,continue_prompt)
         
     def split_user_input(self,line):
         """Split user input into pre-char, function part and rest."""
 
         lsplit = self.line_split.match(line)
         if lsplit is None:  # no regexp match returns None
             try:
                 iFun,theRest = line.split(None,1)
             except ValueError:
                 iFun,theRest = line,''
             pre = re.match('^(\s*)(.*)',line).groups()[0]
         else:
             pre,iFun,theRest = lsplit.groups()
 
         #print 'line:<%s>' % line # dbg
         #print 'pre <%s> iFun <%s> rest <%s>' % (pre,iFun.strip(),theRest) # dbg
         return pre,iFun.strip(),theRest
 
     def _prefilter(self, line, continue_prompt):
         """Calls different preprocessors, depending on the form of line."""
 
         # All handlers *must* return a value, even if it's blank ('').
 
         # Lines are NOT logged here. Handlers should process the line as
         # needed, update the cache AND log it (so that the input cache array
         # stays synced).
 
         # This function is _very_ delicate, and since it's also the one which
         # determines IPython's response to user input, it must be as efficient
         # as possible.  For this reason it has _many_ returns in it, trying
         # always to exit as quickly as it can figure out what it needs to do.
 
         # This function is the main responsible for maintaining IPython's
         # behavior respectful of Python's semantics.  So be _very_ careful if
         # making changes to anything here.
 
         #.....................................................................
         # Code begins
 
         #if line.startswith('%crash'): raise RuntimeError,'Crash now!'  # dbg
 
         # save the line away in case we crash, so the post-mortem handler can
         # record it
         self._last_input_line = line
 
         #print '***line: <%s>' % line # dbg
         
         # the input history needs to track even empty lines
         if not line.strip():
             if not continue_prompt:
                 self.outputcache.prompt_count -= 1
             return self.handle_normal(line,continue_prompt)
             #return self.handle_normal('',continue_prompt)
 
         # print '***cont',continue_prompt  # dbg
         # special handlers are only allowed for single line statements
         if continue_prompt and not self.rc.multi_line_specials:
             return self.handle_normal(line,continue_prompt)
 
         # For the rest, we need the structure of the input
         pre,iFun,theRest = self.split_user_input(line)
         #print 'pre <%s> iFun <%s> rest <%s>' % (pre,iFun,theRest)  # dbg
 
         # First check for explicit escapes in the last/first character
         handler = None
         if line[-1] == self.ESC_HELP:
             handler = self.esc_handlers.get(line[-1])  # the ? can be at the end
         if handler is None:
             # look at the first character of iFun, NOT of line, so we skip
             # leading whitespace in multiline input
             handler = self.esc_handlers.get(iFun[0:1])
         if handler is not None:
             return handler(line,continue_prompt,pre,iFun,theRest)
         # Emacs ipython-mode tags certain input lines
         if line.endswith('# PYTHON-MODE'):
             return self.handle_emacs(line,continue_prompt)
 
         # Next, check if we can automatically execute this thing
 
         # Allow ! in multi-line statements if multi_line_specials is on:
         if continue_prompt and self.rc.multi_line_specials and \
                iFun.startswith(self.ESC_SHELL):
             return self.handle_shell_escape(line,continue_prompt,
                                             pre=pre,iFun=iFun,
                                             theRest=theRest)
 
         # Let's try to find if the input line is a magic fn
         oinfo = None
         if hasattr(self,'magic_'+iFun):
             # WARNING: _ofind uses getattr(), so it can consume generators and
             # cause other side effects.
             oinfo = self._ofind(iFun) # FIXME - _ofind is part of Magic
             if oinfo['ismagic']:
                 # Be careful not to call magics when a variable assignment is
                 # being made (ls='hi', for example)
                 if self.rc.automagic and \
                        (len(theRest)==0 or theRest[0] not in '!=()<>,') and \
                        (self.rc.multi_line_specials or not continue_prompt):
                     return self.handle_magic(line,continue_prompt,
                                              pre,iFun,theRest)
                 else:
                     return self.handle_normal(line,continue_prompt)
 
         # If the rest of the line begins with an (in)equality, assginment or
         # function call, we should not call _ofind but simply execute it.
         # This avoids spurious geattr() accesses on objects upon assignment.
         #
         # It also allows users to assign to either alias or magic names true
         # python variables (the magic/alias systems always take second seat to
         # true python code).
         if theRest and theRest[0] in '!=()':
             return self.handle_normal(line,continue_prompt)
 
         if oinfo is None:
             # let's try to ensure that _oinfo is ONLY called when autocall is
             # on.  Since it has inevitable potential side effects, at least
             # having autocall off should be a guarantee to the user that no
             # weird things will happen.
 
             if self.rc.autocall:
                 oinfo = self._ofind(iFun) # FIXME - _ofind is part of Magic
             else:
                 # in this case, all that's left is either an alias or
                 # processing the line normally.
                 if iFun in self.alias_table:
                     return self.handle_alias(line,continue_prompt,
                                              pre,iFun,theRest)
                 else:
                     return self.handle_normal(line,continue_prompt)
         
         if not oinfo['found']:
             return self.handle_normal(line,continue_prompt)
         else:
             #print 'iFun <%s> rest <%s>' % (iFun,theRest) # dbg
             if oinfo['isalias']:
                 return self.handle_alias(line,continue_prompt,
                                              pre,iFun,theRest)
 
             if self.rc.autocall and \
                    not self.re_exclude_auto.match(theRest) and \
                    self.re_fun_name.match(iFun) and \
                    callable(oinfo['obj']) :
                 #print 'going auto'  # dbg
                 return self.handle_auto(line,continue_prompt,pre,iFun,theRest)
             else:
                 #print 'was callable?', callable(oinfo['obj'])  # dbg
                 return self.handle_normal(line,continue_prompt)
 
         # If we get here, we have a normal Python line. Log and return.
         return self.handle_normal(line,continue_prompt)
 
     def _prefilter_dumb(self, line, continue_prompt):
         """simple prefilter function, for debugging"""
         return self.handle_normal(line,continue_prompt)
 
     # Set the default prefilter() function (this can be user-overridden)
     prefilter = _prefilter
 
     def handle_normal(self,line,continue_prompt=None,
                       pre=None,iFun=None,theRest=None):
         """Handle normal input lines. Use as a template for handlers."""
 
         # With autoindent on, we need some way to exit the input loop, and I
         # don't want to force the user to have to backspace all the way to
         # clear the line.  The rule will be in this case, that either two
         # lines of pure whitespace in a row, or a line of pure whitespace but
         # of a size different to the indent level, will exit the input loop.
         
         if (continue_prompt and self.autoindent and isspace(line) and
             (line != self.indent_current or isspace(self.buffer[-1]))):
             line = ''
 
         self.log(line,continue_prompt)
         return line
 
     def handle_alias(self,line,continue_prompt=None,
                      pre=None,iFun=None,theRest=None):
         """Handle alias input lines. """
 
-        line_out = 'ipalias("%s %s")' % (iFun,esc_quotes(theRest))
+        # pre is needed, because it carries the leading whitespace.  Otherwise
+        # aliases won't work in indented sections.
+        line_out = '%sipalias("%s %s")' % (pre,iFun,esc_quotes(theRest))
         self.log(line_out,continue_prompt)
         return line_out
 
     def handle_shell_escape(self, line, continue_prompt=None,
                             pre=None,iFun=None,theRest=None):
         """Execute the line in a shell, empty return value"""
 
         #print 'line in :', `line` # dbg
         # Example of a special handler. Others follow a similar pattern.
         if continue_prompt:  # multi-line statements
             if iFun.startswith('!!'):
                 print 'SyntaxError: !! is not allowed in multiline statements'
                 return pre
             else:
                 cmd = ("%s %s" % (iFun[1:],theRest))
-                line_out = 'ipsystem(r"""%s"""[:-1])' % (cmd + "_")
+                line_out = '%sipsystem(r"""%s"""[:-1])' % (pre,cmd + "_")
         else: # single-line input
             if line.startswith('!!'):
                 # rewrite iFun/theRest to properly hold the call to %sx and
                 # the actual command to be executed, so handle_magic can work
                 # correctly
                 theRest = '%s %s' % (iFun[2:],theRest)
                 iFun = 'sx'
                 return self.handle_magic('%ssx %s' % (self.ESC_MAGIC,line[2:]),
                                          continue_prompt,pre,iFun,theRest)
             else:
                 cmd=line[1:]
-                line_out = 'ipsystem(r"""%s"""[:-1])' % (cmd +"_")
+                line_out = '%sipsystem(r"""%s"""[:-1])' % (pre,cmd +"_")
         # update cache/log and return
         self.log(line_out,continue_prompt)
         return line_out
 
     def handle_magic(self, line, continue_prompt=None,
                      pre=None,iFun=None,theRest=None):
         """Execute magic functions.
 
         Also log them with a prepended # so the log is clean Python."""
 
         cmd = '%sipmagic("%s")' % (pre,esc_quotes('%s %s' % (iFun,theRest)))
         self.log(cmd,continue_prompt)
         #print 'in handle_magic, cmd=<%s>' % cmd  # dbg
         return cmd
 
     def handle_auto(self, line, continue_prompt=None,
                     pre=None,iFun=None,theRest=None):
         """Hande lines which can be auto-executed, quoting if requested."""
 
         #print 'pre <%s> iFun <%s> rest <%s>' % (pre,iFun,theRest)  # dbg
         
         # This should only be active for single-line input!
         if continue_prompt:
             return line
 
         if pre == self.ESC_QUOTE:
             # Auto-quote splitting on whitespace
             newcmd = '%s("%s")' % (iFun,'", "'.join(theRest.split()) )
         elif pre == self.ESC_QUOTE2:
             # Auto-quote whole string
             newcmd = '%s("%s")' % (iFun,theRest)
         else:
             # Auto-paren
             if theRest[0:1] in ('=','['):
                 # Don't autocall in these cases.  They can be either
                 # rebindings of an existing callable's name, or item access
                 # for an object which is BOTH callable and implements
                 # __getitem__.
                 return '%s %s' % (iFun,theRest)
             if theRest.endswith(';'):
                 newcmd = '%s(%s);' % (iFun.rstrip(),theRest[:-1])
             else:
                 newcmd = '%s(%s)' % (iFun.rstrip(),theRest)
 
         print >>Term.cout, self.outputcache.prompt1.auto_rewrite() + newcmd
         # log what is now valid Python, not the actual user input (without the
         # final newline)
         self.log(newcmd,continue_prompt)
         return newcmd
 
     def handle_help(self, line, continue_prompt=None,
                     pre=None,iFun=None,theRest=None):
         """Try to get some help for the object.
 
         obj? or ?obj   -> basic information.
         obj?? or ??obj -> more details.
         """
 
         # We need to make sure that we don't process lines which would be
         # otherwise valid python, such as "x=1 # what?"
         try:
             codeop.compile_command(line)
         except SyntaxError:
             # We should only handle as help stuff which is NOT valid syntax
             if line[0]==self.ESC_HELP:
                 line = line[1:]
             elif line[-1]==self.ESC_HELP:
                 line = line[:-1]
             self.log('#?'+line)
             if line:
                 self.magic_pinfo(line)
             else:
                 page(self.usage,screen_lines=self.rc.screen_length)
             return '' # Empty string is needed here!
         except:
             # Pass any other exceptions through to the normal handler
             return self.handle_normal(line,continue_prompt)
         else:
             # If the code compiles ok, we should handle it normally
             return self.handle_normal(line,continue_prompt)
 
     def handle_emacs(self,line,continue_prompt=None,
                     pre=None,iFun=None,theRest=None):
         """Handle input lines marked by python-mode."""
 
         # Currently, nothing is done.  Later more functionality can be added
         # here if needed.
 
         # The input cache shouldn't be updated
 
         return line
 
     def write(self,data):
         """Write a string to the default output"""
         Term.cout.write(data)
 
     def write_err(self,data):
         """Write a string to the default error output"""
         Term.cerr.write(data)
 
     def exit(self):
         """Handle interactive exit.
 
         This method sets the exit_now attribute."""
 
         if self.rc.confirm_exit:
             if ask_yes_no('Do you really want to exit ([y]/n)?','y'):
                 self.exit_now = True
         else:
             self.exit_now = True
         return self.exit_now
 
     def safe_execfile(self,fname,*where,**kw):
         fname = os.path.expanduser(fname)
 
         # find things also in current directory
         dname = os.path.dirname(fname)
         if not sys.path.count(dname):
             sys.path.append(dname)
 
         try:
             xfile = open(fname)
         except:
             print >> Term.cerr, \
                   'Could not open file <%s> for safe execution.' % fname
             return None
 
         kw.setdefault('islog',0)
         kw.setdefault('quiet',1)
         kw.setdefault('exit_ignore',0)
         first = xfile.readline()
         loghead = str(self.loghead_tpl).split('\n',1)[0].strip()
         xfile.close()
         # line by line execution
         if first.startswith(loghead) or kw['islog']:
             print 'Loading log file <%s> one line at a time...' % fname
             if kw['quiet']:
                 stdout_save = sys.stdout
                 sys.stdout = StringIO.StringIO()
             try:
                 globs,locs = where[0:2]
             except:
                 try:
                     globs = locs = where[0]
                 except:
                     globs = locs = globals()
             badblocks = []
 
             # we also need to identify indented blocks of code when replaying
             # logs and put them together before passing them to an exec
             # statement. This takes a bit of regexp and look-ahead work in the
             # file. It's easiest if we swallow the whole thing in memory
             # first, and manually walk through the lines list moving the
             # counter ourselves.
             indent_re = re.compile('\s+\S')
             xfile = open(fname)
             filelines = xfile.readlines()
             xfile.close()
             nlines = len(filelines)
             lnum = 0
             while lnum < nlines:
                 line = filelines[lnum]
                 lnum += 1
                 # don't re-insert logger status info into cache
                 if line.startswith('#log#'):
                     continue
                 else:
                     # build a block of code (maybe a single line) for execution
                     block = line
                     try:
                         next = filelines[lnum] # lnum has already incremented
                     except:
                         next = None
                     while next and indent_re.match(next):
                         block += next
                         lnum += 1
                         try:
                             next = filelines[lnum]
                         except:
                             next = None
                     # now execute the block of one or more lines
                     try:
                         exec block in globs,locs
                     except SystemExit:
                         pass
                     except:
                         badblocks.append(block.rstrip())
             if kw['quiet']:  # restore stdout
                 sys.stdout.close()
                 sys.stdout = stdout_save
             print 'Finished replaying log file <%s>' % fname
             if badblocks:
                 print >> sys.stderr, ('\nThe following lines/blocks in file '
                                       '<%s> reported errors:' % fname)
                     
                 for badline in badblocks:
                     print >> sys.stderr, badline
         else:  # regular file execution
             try:
                 execfile(fname,*where)
             except SyntaxError:
                 etype,evalue = sys.exc_info()[:2]
                 self.SyntaxTB(etype,evalue,[])
                 warn('Failure executing file: <%s>' % fname)
             except SystemExit,status:
                 if not kw['exit_ignore']:
                     self.InteractiveTB()
                     warn('Failure executing file: <%s>' % fname)
             except:
                 self.InteractiveTB()
                 warn('Failure executing file: <%s>' % fname)
 
 #************************* end of file <iplib.py> *****************************

doc/new_design.lyx

@@ -1,1345 +1,1358 b''
 #LyX 1.3 created this file. For more info see http://www.lyx.org/
 \lyxformat 221
 \textclass article
 \begin_preamble
 \usepackage{ae,aecompl}
 \usepackage{hyperref}
 \usepackage{html}
 \end_preamble
 \language english
 \inputencoding auto
 \fontscheme default
 \graphics default
 \paperfontsize default
 \spacing single 
 \papersize Default
 \paperpackage a4
 \use_geometry 1
 \use_amsmath 0
 \use_natbib 0
 \use_numerical_citations 0
 \paperorientation portrait
 \leftmargin 1.25in
 \topmargin 1in
 \rightmargin 1.25in
 \bottommargin 1in
 \secnumdepth 3
 \tocdepth 3
 \paragraph_separation skip
 \defskip medskip
 \quotes_language english
 \quotes_times 2
 \papercolumns 1
 \papersides 1
 \paperpagestyle default
 
 \layout Title
 
 IPython
 \newline 
 
 \size larger 
 New design notes
 \layout Author
 
 Fernando P�rez
 \layout Section
 
 Introduction
 \layout Standard
 
 This is a draft document with notes and ideas for the IPython rewrite.
  The section order and structure of this document roughly reflects in which
  order things should be done and what the dependencies are.
  This document is mainly a draft for developers, a pdf version is provided
  with the standard distribution in case regular users are interested and
  wish to contribute ideas.
 \layout Standard
 
 A tentative plan for the future:
 \layout Itemize
 
 0.6.x series: in practice, enough people are using IPython for real work that
  I think it warrants a higher number.
  This series will continue to evolve with bugfixes and incremental improvements.
 \layout Itemize
 
 0.7.x series: (maybe) If resources allow, there may be a branch for 'unstable'
  development, where the architectural rewrite may take place.
 \layout Standard
 
 However, I am starting to doubt it is feasible to keep two separate branches.
  I am leaning more towards a 
 \begin_inset ERT
 status Collapsed
 
 \layout Standard
 
 \backslash 
 LyX
 \end_inset 
 
 -like approach, where the main branch slowly transforms and evolves.
  Having CVS support now makes this a reasonable alternative, as I don't
  have to make pre-releases as often.
  The active branch can remain the mainline of development, and users interested
  in the bleeding-edge stuff can always grab the CVS code.
 \layout Standard
 
 Ideally, IPython should have a clean class setup that would allow further
  extensions for special-purpose systems.
  I view IPython as a base system that provides a great interactive environment
  with full access to the Python language, and which could be used in many
  different contexts.
  The basic hooks are there: the magic extension syntax and the flexible
  system of recursive configuration files and profiles.
  But with a code as messy as the current one, nobody is going to touch it.
 \layout Section
 
 Immediate TODO and bug list
 \layout Standard
 
 Things that should be done for the current series, before starting major
  changes.
 \layout Itemize
 
 Fix any bugs reported at the online bug tracker.
 \layout Itemize
 
+History bug: I often see that, under certain circumstances, the input history
+ is incorrect.
+ The problem is that so far, I've failed to find a simple way to reproduce
+ it consistently, so I can't easily track it down.
+ It seems to me that it happens when output is generated multiple times
+ for the same input (for i in range(10): i will do it).
+ But even this isn't reliable...
+ Ultimately the right solution for this is to cleanly separate the dataflow
+ for input/output history management; right now that happens all over the
+ place, which makes the code impossible to debug, and almost guaranteed
+ to be buggy in the first place.
+\layout Itemize
+
 
 \series bold 
 Redesign the output traps.
 
 \series default 
  They cause problems when users try to execute code which relies on sys.stdout
  being the 'true' sys.stdout.
  They also prevent scripts which use raw_input() to work as command-line
  arguments.
 \newline 
 The best solution is probably to print the banner first, and then just execute
  all the user code straight with no output traps at all.
  Whatever comes out comes out.
  This makes the ipython code actually simpler, and eliminates the problem
  altogether.
 \newline 
 These things need to be ripped out, they cause no end of problems.
  For example, if user code requires acces to stdin during startup, the process
  just hangs indefinitely.
  For now I've just disabled them, and I'll live with the ugly error messages.
 \layout Itemize
 
 The prompt specials dictionary should be turned into a class which does
  proper namespace management, since the prompt specials need to be evaluated
  in a certain namespace.
  Currently it's just globals, which need to be managed manually by code
  below.
  
 \layout Itemize
 
 Fix coloring of prompts: the pysh color strings don't have any effect on
  prompt numbers, b/c these are controlled by the global scheme.
  Make the prompts fully user-definable, colors and all.
  This is what I said to a user:
 \newline 
 As far as the green 
 \backslash 
 #, this is a minor bug of the coloring code due to the vagaries of history.
  While the color strings allow you to control the coloring of most elements,
  there are a few which are still controlled by the old ipython internal
  coloring code, which only accepts a global 'color scheme' choice.
  So basically the input/output numbers are hardwired to the choice in the
  color scheme, and there are only 'Linux', 'LightBG' and 'NoColor' schemes
  to choose from.
  
 \layout Itemize
 
 Clean up FakeModule issues.
  Currently, unittesting with embedded ipython breaks because a FakeModule
  instance overwrites __main__.
  Maybe ipython should revert back to using __main__ directly as the user
  namespace? Handling a separate namespace is proving 
 \emph on 
 very
 \emph default 
  tricky in all corner cases.
 \layout Itemize
 
 Make the output cache depth independent of the input one.
  This way one can have say only the last 10 results stored and still have
  a long input history/cache.
 \layout Itemize
 
 Fix the fact that importing a shell for embedding screws up the command-line
  history.
  This can be done by not importing the history file when the shell is already
  inside ipython.
 \layout Itemize
 
 Lay out the class structure so that embedding into a gtk/wx/qt app is trivial,
  much like the multithreaded gui shells now provide command-line coexistence
  with the gui toolkits.
  See 
 \begin_inset LatexCommand \url{http://www.livejournal.com/users/glyf/32396.html}
 
 \end_inset 
 
 
 \layout Itemize
 
 Get Holger's completer in, once he adds filename completion.
 \layout Standard
 
 Lower priority stuff:
 \layout Itemize
 
 Add @showopt/@setopt (decide name) for viewing/setting all options.
  The existing option-setting magics should become aliases for setopt calls.
 \layout Itemize
 
 It would be nice to be able to continue with python stuff after an @ command.
  For instance "@run something; test_stuff()" in order to test stuff even
  faster.
  Suggestion by Kasper Souren <[e-mail]>
 \layout Itemize
 
 Run a 'first time wizard' which configures a few things for the user, such
  as color_info, editor and the like.
 \layout Itemize
 
 Logging: @logstart and -log should start logfiles in ~.ipython, but with
  unique names in case of collisions.
  This would prevent ipython.log files all over while also allowing multiple
  sessions.
  Also the -log option should take an optional filename, instead of having
  a separate -logfile option.
 \newline 
 In general the logging system needs a serious cleanup.
  Many functions now in Magic should be moved to Logger, and the magic @s
  should be very simple wrappers to the Logger methods.
 \layout Section
 
 Lighten the code
 \layout Standard
 
 If we decide to base future versions of IPython on Python 2.3, which has
  the new Optik module (called optparse), it should be possible to drop DPyGetOpt.
  We should also remove the need for Itpl.
  Another area for trimming is the Gnuplot stuff: much of that could be merged
  into the mainline project.
 \layout Standard
 
 Double check whether we really need FlexCompleter.
  This was written as an enhanced rlcompleter, but my patches went in for
  python 2.2 (or 2.3, can't remember).
 \layout Standard
 
 With these changes we could shed a fair bit of code from the main trunk.
 \layout Section
 
 Unit testing
 \layout Standard
 
 All new code should use a testing framework.
  Python seems to have very good testing facilities, I just need to learn
  how to use them.
  I should also check out QMTest at 
 \begin_inset LatexCommand \htmlurl{http://www.codesourcery.com/qm/qmtest}
 
 \end_inset 
 
 , it sounds interesting (it's Python-based too).
 \layout Section
 
 Configuration system
 \layout Standard
 
 Move away from the current ipythonrc format to using standard python files
  for configuration.
  This will require users to be slightly more careful in their syntax, but
  reduces code in IPython, is more in line with Python's normal form (using
  the $PYTHONSTARTUP file) and allows much more flexibility.
  I also think it's more 'pythonic', in using a single language for everything.
 \layout Standard
 
 Options can be set up with a function call which takes keywords and updates
  the options Struct.
 \layout Standard
 
 In order to maintain the recursive inclusion system, write an 'include'
  function which is basically a wrapper around safe_execfile().
  Also for alias definitions an alias() function will do.
  All functionality which we want to have at startup time for the users can
  be wrapped in a small module so that config files look like:
 \layout Standard
 
 
 \family typewriter 
 from IPython.Startup import *
 \newline 
 ...
 \newline 
 set_options(automagic=1,colors='NoColor',...)
 \newline 
 ...
 \newline 
 include('mysetup.py')
 \newline 
 ...
 \newline 
 alias('ls ls --color -l')
 \newline 
 ...
  etc.
 \layout Standard
 
 Also, put 
 \series bold 
 all
 \series default 
  aliases in here, out of the core code.
 \layout Standard
 
 The new system should allow for more seamless upgrading, so that:
 \layout Itemize
 
 It automatically recognizes when the config files need updating and does
  the upgrade.
 \layout Itemize
 
 It simply adds the new options to the user's config file without overwriting
  it.
  The current system is annoying since users need to manually re-sync their
  configuration after every update.
 \layout Itemize
 
 It detects obsolete options and informs the user to remove them from his
  config file.
 \layout Standard
 
 Here's a copy of Arnd Baecker suggestions on the matter:
 \layout Standard
 
 1.) upgrade: it might be nice to have an "auto" upgrade procedure: i.e.
  imagine that IPython is installed system-wide and gets upgraded, how does
  a user know, that an upgrade of the stuff in ~/.ipython is necessary ? So
  maybe one has to a keep a version number in ~/.ipython and if there is a
  mismatch with the started ipython, then invoke the upgrade procedure.
 \layout Standard
 
 2.) upgrade: I find that replacing the old files in ~/.ipython (after copying
  them to .old not optimal (for example, after every update, I have to change
  my color settings (and some others) in ~/.ipython/ipthonrc).
  So somehow keeping the old files and merging the new features would be
  nice.
  (but how to distinguish changes from version to version with changes made
  by the user ?) For, example, I would have to change in GnuplotMagic.py gnuplot_m
 ouse to 1 after every upgrade ...
 \layout Standard
 
 This is surely a minor point - also things will change during the "BIG"
  rewrite, but maybe this is a point to keep in mind for this ?
 \layout Standard
 
 3.) upgrade: old, sometimes obsolete files stay in the ~/.ipython subdirectory.
  (hmm, maybe one could move all these into some subdirectory, but which
  name for that (via version-number ?) ?)
 \layout Subsection
 
 Command line options
 \layout Standard
 
 It would be great to design the command-line processing system so that it
  can be dynamically modified in some easy way.
  This would allow systems based on IPython to include their own command-line
  processing to either extend or fully replace IPython's.
  Probably moving to the new optparse library (also known as optik) will
  make this a lot easier.
 \layout Section
 
 OS-dependent code
 \layout Standard
 
 Options which are OS-dependent (such as colors and aliases) should be loaded
  via include files.
  That is, the general file will have:
 \layout Standard
 
 
 \family typewriter 
 if os.name == 'posix':
 \newline 
 include('ipythonrc-posix.py')
 \newline 
 elif os.name == 'nt':
 \newline 
 include('ipythonrc-nt.py')...
 \layout Standard
 
 In the 
 \family typewriter 
 -posix
 \family default 
 , 
 \family typewriter 
 -nt
 \family default 
 , etc.
  files we'll set all os-specific options.
 \layout Section
 
 Merging with other shell systems
 \layout Standard
 
 This is listed before the big design issues, as it is something which should
  be kept in mind when that design is made.
 \layout Standard
 
 The following shell systems are out there and I think the whole design of
  IPython should try to be modular enough to make it possible to integrate
  its features into these.
  In all cases IPython should exist as a stand-alone, terminal based program.
  But it would be great if users of these other shells (some of them which
  have very nice features of their own, especially the graphical ones) could
  keep their environment but gain IPython's features.
 \layout List
 \labelwidthstring 00.00.0000
 
 IDLE This is the standard, distributed as part of Python.
  
 \layout List
 \labelwidthstring 00.00.0000
 
 pyrepl 
 \begin_inset LatexCommand \htmlurl{http://starship.python.net/crew/mwh/hacks/pyrepl.html}
 
 \end_inset 
 
 .
  This is a text (curses-based) shell-like replacement which doesn't have
  some of IPython's features, but has a crucially useful (and hard to implement)
  one: full multi-line editing.
  This turns the interactive interpreter into a true code testing and development
  environment.
  
 \layout List
 \labelwidthstring 00.00.0000
 
 PyCrust 
 \begin_inset LatexCommand \htmlurl{http://sourceforge.net/projects/pycrust}
 
 \end_inset 
 
 .
  Very nice, wxWindows based system.
 \layout List
 \labelwidthstring 00.00.0000
 
 PythonWin 
 \begin_inset LatexCommand \htmlurl{http://starship.python.net/crew/mhammond}
 
 \end_inset 
 
 .
  Similar to PyCrust in some respects, a very good and free Python development
  environment for Windows systems.
 \layout Section
 
 Class design
 \layout Standard
 
 This is the big one.
  Currently classes use each other in a very messy way, poking inside one
  another for data and methods.
  ipmaker() adds tons of stuff to the main __IP instance by hand, and the
  mix-ins used (Logger, Magic, etc) mean the final __IP instance has a million
  things in it.
  All that needs to be cleanly broken down with well defined interfaces amongst
  the different classes, and probably no mix-ins.
 \layout Standard
 
 The best approach is probably to have all the sub-systems which are currently
  mixins be fully independent classes which talk back only to the main instance
  (and 
 \series bold 
 not
 \series default 
  to each other).
  In the main instance there should be an object whose job is to handle communica
 tion with the sub-systems.
 \layout Standard
 
 I should probably learn a little UML and diagram this whole thing before
  I start coding.
 \layout Subsection
 
 Magic
 \layout Standard
 
 Now all methods which will become publicly available are called Magic.magic_name,
  the magic_ should go away.
  Then, Magic instead of being a mix-in should simply be an attribute of
  __IP:
 \layout Standard
 
 __IP.Magic = Magic()
 \layout Standard
 
 This will then give all the magic functions as __IP.Magic.name(), which is
  much cleaner.
  This will also force a better separation so that Magic doesn't poke inside
  __IP so much.
  In the constructor, Magic should get whatever information it needs to know
  about __IP (even if it means a pointer to __IP itself, but at least we'll
  know where it is.
  Right now since it's a mix-in, there's no way to know which variables belong
  to whom).
 \layout Standard
 
 Build a class MagicFunction so that adding new functions is a matter of:
 \layout Standard
 
 
 \family typewriter 
 my_magic = MagicFunction(category = 'System utilities')
 \newline 
 my_magic.__call__ = ...
 \layout Standard
 
 Features:
 \layout Itemize
 
 The class constructor should automatically register the functions and keep
  a table with category sections for easy sorting/viewing.
 \layout Itemize
 
 The object interface must allow automatic building of a GUI for them.
  This requires registering the options the command takes, the number of
  arguments, etc, in a formal way.
  The advantage of this approach is that it allows not only to add GUIs to
  the magics, but also for a much more intelligent building of docstrings,
  and better parsing of options and arguments.
 \layout Standard
 
 Also think through better an alias system for magics.
  Since the magic system is like a command shell inside ipython, the relation
  between these aliases and system aliases should be cleanly thought out.
 \layout Subsection
 
 Color schemes
 \layout Standard
 
 These should be loaded from some kind of resource file so they are easier
  to modify by the user.
 \layout Section
 
 Hooks
 \layout Standard
 
 IPython should have a modular system where functions can register themselves
  for certain tasks.
  Currently changing functionality requires overriding certain specific methods,
  there should be a clean API for this to be done.
 \layout Subsection
 
 whos hook
 \layout Standard
 
 This was a very nice suggestion from Alexander Schmolck <[e-mail]>:
 \layout Standard
 
 2.
  I think it would also be very helpful if there where some sort of hook
  for ``whos`` that let one customize display formaters depending on the
  object type.
 \layout Standard
 
 For example I'd rather have a whos that formats an array like:
 \layout Standard
 
 
 \family typewriter 
 Variable Type Data/Length
 \newline 
 ------------------------------ 
 \newline 
 a array size: 4x3 type: 'Float'
 \layout Standard
 
 than
 \layout Standard
 
 
 \family typewriter 
 Variable Type Data/Length 
 \newline 
 ------------------------------
 \newline 
 a array [[ 0.
  1.
  2.
  3<...> 8.
  9.
  10.
  11.]]
 \layout Section
 
 Parallel support
 \layout Standard
 
 For integration with graphical shells and other systems, it will be best
  if ipython is split into a kernel/client model, much like Mathematica works.
  This simultaneously opens the door for support of interactive parallel
  computing.
  Currenlty %bg provides a threads-based proof of concept, and Brian Granger's
  XGrid project is a much more realistic such system.
  The new design should integrates ideas as core elements.
  Some notes from Brian on this topic:
 \layout Standard
 
 1.
  How should the remote python server/kernel be designed? Multithreaded?
  Blocking? Connected/disconnected modes? Load balancing?
 \layout Standard
 
 2.
  What APi/protocol should the server/kernel expose to clients?
 \layout Standard
 
 3.
  How should the client classes (which the user uses to interact with the
  cluster) be designed?
 \layout Standard
 
 4.
  What API should the client classes expose?
 \layout Standard
 
 5.
  How should the client API be wrapped in a few simple magic functions?
 \layout Standard
 
 6.
  How should security be handled?
 \layout Standard
 
 7.
  How to work around the issues of the GIL and threads?
 \layout Standard
 
 I think the most important things to work out are the client API (#4) the
  server/kernel API/protocol (#2) and the magic function API (#5).
  We should let these determine the design and architecture of the components.
 \layout Standard
 
 One other thing.
  What is your impression of twisted? I have been looking at it and it looks
  like a _very_ powerful set of tools for this type of stuff.
  I am wondering if it might make sense to think about using twisted for
  this project.
 \layout Section
 
 Manuals
 \layout Standard
 
 The documentation should be generated from docstrings for the command line
  args and all the magic commands.
  Look into one of the simple text markup systems to see if we can get latex
  (for reLyXing later) out of this.
  Part of the build command would then be to make an update of the docs based
  on this, thus giving more complete manual (and guaranteed to be in sync
  with the code docstrings).
 \layout Standard
 
 [PARTLY DONE] At least now all magics are auto-documented, works farily
  well.
  Limited Latex formatting yet.
 \layout Subsection
 
 Integration with pydoc-help
 \layout Standard
 
 It should be possible to have access to the manual via the pydoc help system
  somehow.
  This might require subclassing the pydoc help, or figuring out how to add
  the IPython docs in the right form so that help() finds them.
 \layout Standard
 
 Some comments from Arnd and my reply on this topic:
 \layout Standard
 
 > ((Generally I would like to have the nice documentation > more easily
  accessable from within ipython ...
  > Many people just don't read documentation, even if it is > as good as
  the one of IPython ))
 \layout Standard
 
 That's an excellent point.
  I've added a note to this effect in new_design.
  Basically I'd like help() to naturally access the IPython docs.
  Since they are already there in html for the user, it's probably a matter
  of playing a bit with pydoc to tell it where to find them.
  It would definitely make for a much cleaner system.
  Right now the information on IPython is:
 \layout Standard
 
 -ipython --help at the command line: info on command line switches 
 \layout Standard
 
 -? at the ipython prompt: overview of IPython 
 \layout Standard
 
 -magic at the ipython prompt: overview of the magic system 
 \layout Standard
 
 -external docs (html/pdf)
 \layout Standard
 
 All that should be better integrated seamlessly in the help() system, so
  that you can simply say:
 \layout Standard
 
 help ipython -> full documentation access 
 \layout Standard
 
 help magic -> magic overview 
 \layout Standard
 
 help profile -> help on current profile 
 \layout Standard
 
 help -> normal python help access.
 \layout Section
 
 Graphical object browsers
 \layout Standard
 
 I'd like a system for graphically browsing through objects.
  
 \family typewriter 
 @obrowse
 \family default 
  should open a widged with all the things which 
 \family typewriter 
 @who
 \family default 
  lists, but cliking on each object would open a dedicated object viewer
  (also accessible as 
 \family typewriter 
 @oview <object>
 \family default 
 ).
  This object viewer could show a summary of what 
 \family typewriter 
 <object>?
 \family default 
  currently shows, but also colorize source code and show it via an html
  browser, show all attributes and methods of a given object (themselves
  openable in their own viewers, since in Python everything is an object),
  links to the parent classes, etc.
 \layout Standard
 
 The object viewer widget should be extensible, so that one can add methods
  to view certain types of objects in a special way (for example, plotting
  Numeric arrays via grace or gnuplot).
  This would be very useful when using IPython as part of an interactive
  complex system for working with certain types of data.
 \layout Standard
 
 I should look at what PyCrust has to offer along these lines, at least as
  a starting point.
 \layout Section
 
 Miscellaneous small things
 \layout Itemize
 
 Collect whatever variables matter from the environment in some globals for
  __IP, so we're not testing for them constantly (like $HOME, $TERM, etc.)
 \layout Section
 
 Session restoring
 \layout Standard
 
 I've convinced myself that session restore by log replay is too fragile
  and tricky to ever work reliably.
  Plus it can be dog slow.
  I'd rather have a way of saving/restoring the *current* memory state of
  IPython.
  I tried with pickle but failed (can't pickle modules).
  This seems the right way to do it to me, but it will have to wait until
  someone tells me of a robust way of dumping/reloading *all* of the user
  namespace in a file.
 \layout Standard
 
 Probably the best approach will be to pickle as much as possible and record
  what can not be pickled for manual reload (such as modules).
  This is not trivial to get to work reliably, so it's best left for after
  the code restructuring.
 \layout Standard
 
 The following issues exist (old notes, see above paragraph for my current
  take on the issue):
 \layout Itemize
 
 magic lines aren't properly re-executed when a log file is reloaded (and
  some of them, like clear or run, may change the environment).
  So session restore isn't 100% perfect.
 \layout Itemize
 
 auto-quote/parens lines aren't replayed either.
  All this could be done, but it needs some work.
  Basically it requires re-running the log through IPython itself, not through
  python.
 \layout Itemize
 
 _p variables aren't restored with a session.
  Fix: same as above.
 \layout Section
 
 Tips system
 \layout Standard
 
 It would be nice to have a tip() function which gives tips to users in some
  situations, but keeps track of already-given tips so they aren't given
  every time.
  This could be done by pickling a dict of given tips to IPYTHONDIR.
 \layout Section
 
 TAB completer
 \layout Standard
 
 Some suggestions from Arnd Baecker:
 \layout Standard
 
 a) For file related commands (ls, cat, ...) it would be nice to be able to
  TAB complete the files in the current directory.
  (once you started typing something which is uniquely a file, this leads
  to this effect, apart from going through the list of possible completions
  ...).
  (I know that this point is in your documentation.)
 \layout Standard
 
 More general, this might lead to something like command specific completion
  ? 
 \layout Standard
 
 Here's John Hunter's suggestion:
 \layout Standard
 
 The *right way to do it* would be to make intelligent or customizable choices
  about which namespace to add to the completion list depending on the string
  match up to the prompt, eg programmed completions.
  In the simplest implementation, one would only complete on files and directorie
 s if the line preceding the tab press matched 'cd ' or 'run ' (eg you don't
  want callable showing up in 'cd ca<TAB>')
 \layout Standard
 
 In a more advanced scenario, you might imaging that functions supplied the
  TAB namespace, and the user could configure a dictionary that mapped regular
  expressions to namespace providing functions (with sensible defaults).
  Something like
 \layout Standard
 
 completed = {
 \newline 
 '^cd
 \backslash 
 s+(.*)' : complete_files_and_dirs,
 \newline 
 '^run
 \backslash 
 s+(.*)' : complete_files_and_dirs,
 \newline 
 '^run
 \backslash 
 s+(-.*)' : complete_run_options,
 \newline 
 }
 \layout Standard
 
 I don't know if this is feasible, but I really like programmed completions,
  which I use extensively in tcsh.
  My feeling is that something like this is eminently doable in ipython.
 \layout Standard
 
 /JDH
 \layout Standard
 
 For something like this to work cleanly, the magic command system needs
  also a clean options framework, so all valid options for a given magic
  can be extracted programatically.
 \layout Section
 
 Debugger
 \layout Standard
 
 Current system uses a minimally tweaked pdb.
  Fine-tune it a bit, to provide at least:
 \layout Itemize
 
 Tab-completion in each stack frame.
  See email to Chris Hart for details.
 \layout Itemize
 
 Object information via ? at least.
  Break up magic_oinfo a bit so that pdb can call it without loading all
  of IPython.
  If possible, also have the other magics for object study: doc, source,
  pdef and pfile.
 \layout Itemize
 
 Shell access via !
 \layout Itemize
 
 Syntax highlighting in listings.
  Use py2html code, implement color schemes.
 \layout Section
 
 A Python-based system shell - pysh?
 \layout Standard
 
 Note: as of IPython 0.6.1, most of this functionality has actually been implemente
 d.
 \layout Standard
 
 This section is meant as a working draft for discussions on the possibility
  of having a python-based system shell.
  It is the result of my own thinking about these issues as much of discussions
  on the ipython lists.
  I apologize in advance for not giving individual credit to the various
  contributions, but right now I don't have the time to track down each message
  from the archives.
  So please consider this as the result of a collective effort by the ipython
  user community.
 \layout Standard
 
 While IPyhton is (and will remain) a python shell first, it does offer a
  fair amount of system access functionality:
 \layout Standard
 
 - ! and !! for direct system access,
 \layout Standard
 
 - magic commands which wrap various system commands,
 \layout Standard
 
 - @sc and @sx, for shell output capture into python variables,
 \layout Standard
 
 - @alias, for aliasing system commands.
 \layout Standard
 
 This has prompted many users, over time, to ask for a way of extending ipython
  to the point where it could be used as a full-time replacement over typical
  user shells like bash, csh or tcsh.
  While my interest in ipython is such that I'll concentrate my personal
  efforts on other fronts (debugging, architecture, improvements for scientific
  use, gui access), I will be happy to do anything which could make such
  a development possible.
  It would be the responsibility of someone else to maintain the code, but
  I would do all necessary architectural changes to ipython for such an extension
  to be feasible.
 \layout Standard
 
 I'll try to outline here what I see as the key issues which need to be taken
  into account.
  This document should be considered an evolving draft.
  Feel free to submit comments/improvements, even in the form of patches.
 \layout Standard
 
 In what follows, I'll represent the hypothetical python-based shell ('pysh'
  for now) prompt with '>>'.
 \layout Subsection
 
 Basic design principles
 \layout Standard
 
 I think the basic design guideline should be the following: a hypothetical
  python system shell should behave, as much as possible, like a normal shell
  that users are familiar with (bash, tcsh, etc).
  This means:
 \layout Standard
 
 1.
  System commands can be issued directly at the prompt with no special syntax:
 \layout Standard
 
 >> ls
 \layout Standard
 
 >> xemacs
 \layout Standard
 
 should just work like a user expects.
 \layout Standard
 
 2.
  The facilities of the python language should always be available, like
  they are in ipython:
 \layout Standard
 
 >> 3+4 
 \newline 
 7
 \layout Standard
 
 3.
  It should be possible to easily capture shell output into a variable.
  bash and friends use backquotes, I think using a command (@sc) like ipython
  currently does is an acceptable compromise.
 \layout Standard
 
 4.
  It should also be possible to expand python variables/commands in the middle
  of system commands.
  I thihk this will make it necessary to use $var for name expansions:
 \layout Standard
 
 >> var='hello' # var is a Python variable 
 \newline 
 >> print var hello # This is the result of a Python print command 
 \newline 
 >> echo $var hello # This calls the echo command, expanding 'var'.
 \layout Standard
 
 5.
  The above capabilities should remain possible for multi-line commands.
  One of the most annoying things I find about tcsh, is that I never quite
  remember the syntactic details of looping.
  I often want to do something at the shell which involves a simple loop,
  but I can never remember how to do it in tcsh.
  This often means I just write a quick throwaway python script to do it
  (Perl is great for this kind of quick things, but I've forgotten most its
  syntax as well).
 \layout Standard
 
 It should be possible to write code like:
 \layout Standard
 
 >> for ext in ['.jpg','.gif']: 
 \newline 
 ..
  ls file$ext
 \layout Standard
 
 And have it work as 'ls file.jpg;ls file.gif'.
 \layout Subsection
 
 Smaller details 
 \layout Standard
 
 If the above are considered as valid guiding principles for how such a python
  system shell should behave, then some smaller considerations and comments
  to keep in mind are listed below.
 \layout Standard
 
 - it's ok for shell builtins (in this case this includes the python language)
  to override system commands on the path.
  See tcsh's 'time' vs '/usr/bin/time'.
  This settles the 'print' issue and related.
 \layout Standard
 
 - pysh should take
 \layout Standard
 
 foo args
 \layout Standard
 
 as a command if (foo args is NOT valid python) and (foo is in $PATH).
 \layout Standard
 
 If the user types
 \layout Standard
 
 >> ./foo args
 \layout Standard
 
 it should be considered a system command always.
 \layout Standard
 
 - _, __ and ___ should automatically remember the previous 3 outputs captured
  from stdout.
  In parallel, there should be _e, __e and ___e for stderr.
  Whether capture is done as a single string or in list mode should be a
  user preference.
  If users have numbered prompts, ipython's full In/Out cache system should
  be available.
 \layout Standard
 
 But regardless of how variables are captured, the printout should be like
  that of a plain shell (without quotes or braces to indicate strings/lists).
  The everyday 'feel' of pysh should be more that of bash/tcsh than that
  of ipython.
 \layout Standard
 
 - filename completion first.
  Tab completion could work like in ipython, but with the order of priorities
  reversed: first files, then python names.
 \layout Standard
 
 - configuration via standard python files.
  Instead of 'setenv' you'd simply write into the os.environ[] dictionary.
  This assumes that IPython itself has been fixed to be configured via normal
  python files, instead of the current clunky ipythonrc format.
 \layout Standard
 
 - IPython can already configure the prompt in fairly generic ways.
  It should be able to generate almost any kind of prompt which bash/tcsh
  can (within reason).
 \layout Standard
 
 - Keep the Magics system.
  They provide a lightweight syntax for configuring and modifying the state
  of the user's session itself.
  Plus, they are an extensible system so why not give the users one more
  tool which is fairly flexible by nature? Finally, having the @magic optional
  syntax allows a user to always be able to access the shell's control system,
  regardless of name collisions with defined variables or system commands.
 \layout Standard
 
 But we need to move all magic functionality into a protected namespace,
  instead of the current messy name-mangling tricks (which don't scale well).
  
 \layout Section
 
 Future improvements
 \layout Itemize
 
 When from <mod> import * is used, first check the existing namespace and
  at least issue a warning on screen if names are overwritten.
 \layout Itemize
 
 Auto indent? Done, for users with readline support.
 \layout Subsection
 
 Better completion a la zsh
 \layout Standard
 
 This was suggested by Arnd:
 \layout Standard
 
 > >\SpecialChar ~
 \SpecialChar ~
 \SpecialChar ~
  More general, this might lead to something like
 \layout Standard
 
 > >\SpecialChar ~
 \SpecialChar ~
 \SpecialChar ~
  command specific completion ?
 \layout Standard
 
 >
 \layout Standard
 
 > I'm not sure what you mean here.
 \layout Standard
 
 \SpecialChar ~
 
 \layout Standard
 
 Sorry, that was not understandable, indeed ...
 \layout Standard
 
 I thought of something like
 \layout Standard
 
 \SpecialChar ~
 - cd and then use TAB to go through the list of directories
 \layout Standard
 
 \SpecialChar ~
 - ls and then TAB to consider all files and directories
 \layout Standard
 
 \SpecialChar ~
 - cat and TAB: only files (no directories ...)
 \layout Standard
 
 \SpecialChar ~
 
 \layout Standard
 
 For zsh things like this are established by defining in .zshrc
 \layout Standard
 
 \SpecialChar ~
 
 \layout Standard
 
 compctl -g '*.dvi' xdvi
 \layout Standard
 
 compctl -g '*.dvi' dvips
 \layout Standard
 
 compctl -g '*.tex' latex
 \layout Standard
 
 compctl -g '*.tex' tex
 \layout Standard
 
 ...
 \layout Section
 
 Outline of steps
 \layout Standard
 
 Here's a rough outline of the order in which to start implementing the various
  parts of the redesign.
  The first 'test of success' should be a clean pychecker run (not the mess
  we get right now).
 \layout Itemize
 
 Make Logger and Magic not be mixins but attributes of the main class.
  
 \begin_deeper 
 \layout Itemize
 
 Magic should have a pointer back to the main instance (even if this creates
  a recursive structure) so it can control it with minimal message-passing
  machinery.
  
 \layout Itemize
 
 Logger can be a standalone object, simply with a nice, clean interface.
 \layout Itemize
 
 Logger currently handles part of the prompt caching, but other parts of
  that are in the prompts class itself.
  Clean up.
 \end_deeper 
 \layout Itemize
 
 Change to python-based config system.
 \layout Itemize
 
 Move make_IPython() into the main shell class, as part of the constructor.
  Do this 
 \emph on 
 after 
 \emph default 
 the config system has been changed, debugging will be a lot easier then.
 \layout Itemize
 
 Merge the embeddable class and the normal one into one.
  After all, the standard ipython script 
 \emph on 
 is
 \emph default 
  a python program with IPython embedded in it.
  There's no need for two separate classes (
 \emph on 
 maybe
 \emph default 
  keep the old one around for the sake of backwards compatibility).
 \layout Section
 
 Ville Vainio's suggestions
 \layout Standard
 
 Some notes sent in by Ville Vainio 
 \family typewriter 
 <[e-mail]>
 \family default 
  on Tue, 29 Jun 2004.
  Keep here for reference, some of it replicates things already said above.
 \layout Standard
 
 Current ipython seems to "special case" lots of stuff - aliases, magics
  etc.
  It would seem to yield itself to a simpler and more extensible architecture,
  consisting of multple dictionaries, where just the order of search is determine
 d by profile/prefix.
  All the functionality would just be "pushed" to ipython core, i.e.
  the objects that represent the functionality are instantiated on "plugins"
  and they are registered with ipython core.
  i.e.
 \layout Standard
 
 def magic_f(options, args): pass
 \layout Standard
 
 m = MyMagic(magic_f) m.arghandler = stockhandlers.OptParseArgHandler m.options
  = ....
  # optparse options, for easy passing to magic_f and help display
 \layout Standard
 
 # note that arghandler takes a peek at the instance, sees options, and proceeds
  # accordingly.
  Various arg handlers can ask for arbitrary options.
  # some handler might optionally glob the filenames, search data folders
  for filenames etc.
 \layout Standard
 
 ipythonregistry.register(category = "magic", name = "mymagic", obj = m)
 \layout Standard
 
 I bet most of the current functionality could easily be added to such a
  registry by just instantiating e.g.
  "Magic" class and registering all the functions with some sensible default
  args.
  Supporting legacy stuff in general would be easy - just implement new handlers
  (arg and otherwise) for new stuff, and have the old handlers around forever
  / as long as is deemed appropriate.
  The 'python' namespace (locals() + globals()) should be special, of course.
 \layout Standard
 
 It should be easy to have arbitrary number of "categories" (like 'magic',
  'shellcommand','projectspecific_myproject', 'projectspecific_otherproject').
  It would only influence the order in which the completions are suggested,
  and in case of name collision which one is selected.
  Also, I think all completions should be shown, even the ones in "later"
  categories in the case of a match in an "earlier" category.
 \layout Standard
 
 The "functionality object" might also have a callable object 'expandarg',
  and ipython would run it (with the arg index) when tab completion is attempted
  after typing the function name.
  It would return the possible completions for that particular command...
  or None to "revert to default file completions".
  Such functionality could be useful in making ipython an "operating console"
  of a sort.
  I'm talking about:
 \layout Standard
 
 >> lscat reactor # list commands in category - reactor is "project specific"
  category
 \layout Standard
 
 r_operate
 \layout Standard
 
 >> r_operate <tab> start shutdown notify_meltdown evacuate
 \layout Standard
 
 >> r_operate shutdown <tab>
 \layout Standard
 
 1 2 5 6 # note that 3 and 4 are already shut down
 \layout Standard
 
 >> r_operate shutdown 2
 \layout Standard
 
 Shutting down..
  ok.
 \layout Standard
 
 >> r_operate start <tab>
 \layout Standard
 
 2 3 4 # 2 was shut down, can be started now
 \layout Standard
 
 >> r_operate start 2
 \layout Standard
 
 Starting....
  ok.
 \layout Standard
 
 I'm talking about having a super-configurable man-machine language here!
  Like cmd.Cmd on steroids, as a free addition to ipython!
 \the_end