Merge branch 'master' of git://github.com/ipython/ipython into qtfrontend and fix conflicts in setupbase.py.

Conflicts:
	setupbase.py

IPython/core/tests/test_autocall.py

@@ -0,0 +1,43 b''
+"""These kinds of tests are less than ideal, but at least they run.
+
+This was an old test that was being run interactively in the top-level tests/
+directory, which we are removing.  For now putting this here ensures at least
+we do run the test, though ultimately this functionality should all be tested
+with better-isolated tests that don't rely on the global instance in iptest.
+"""
+
+def doctest_autocall():
+    """
+    In [1]: def f1(a,b,c):
+       ...:     return a+b+c
+       ...: 
+
+    In [2]: def f2(a):
+       ...:     return a + a
+       ...:   
+
+    In [3]: ;f2 a b c
+    Out[3]: 'a b ca b c'
+
+    In [4]: assert _ == "a b ca b c"
+
+    In [5]: ,f1 a b c
+    Out[5]: 'abc'
+
+    In [6]: assert _ == 'abc'
+
+    In [7]: print _
+    abc
+
+    In [8]: /f1 1,2,3
+    Out[8]: 6
+
+    In [9]: assert _ == 6
+
+    In [10]: /f2 4
+    Out[10]: 8
+
+    In [11]: assert _ == 8
+
+    In [11]: del f1, f2
+    """

IPython/Shell.py

@@ -1,42 +1,42 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 A backwards compatibility layer for IPython.Shell.
 
 Previously, IPython had an IPython.Shell module.  IPython.Shell has been moved
 to IPython.core.shell and is being refactored.  This new module is provided
 for backwards compatability.  We strongly encourage everyone to start using
 the new code in IPython.core.shell.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 from warnings import warn
 
 msg = """
 This module (IPython.Shell) is deprecated.  The classes that were in this
 module have been replaced by:
 
 IPShell->IPython.core.iplib.InteractiveShell
 IPShellEmbed->IPython.core.embed.InteractiveShellEmbed
 
 Please migrate your code to use these classes instead.
 """
 
 warn(msg, category=DeprecationWarning, stacklevel=1)
 
 from IPython.core.iplib import InteractiveShell as IPShell
 from IPython.core.embed import InteractiveShellEmbed as IPShellEmbed
 
 def start(user_ns=None, embedded=False):
     """Return an instance of :class:`InteractiveShell`."""
     if embedded:
-        return InteractiveShellEmbed(user_ns=user_ns)
+        return IPShellEmbed(user_ns=user_ns)
     else:
-        return InteractiveShell(user_ns=user_ns)
+        return IPShell(user_ns=user_ns)
 

IPython/core/crashhandler.py

@@ -1,180 +1,179 b''
 # encoding: utf-8
 """sys.excepthook for IPython itself, leaves a detailed report on disk.
 
 Authors:
 
 * Fernando Perez
 * Brian E. Granger
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #  Copyright (C) 2008-2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 from pprint import pformat
 
 from IPython.core import ultratb
 from IPython.external.Itpl import itpl
 from IPython.utils.sysinfo import sys_info
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 # Template for the user message.
 _default_message_template = """\
 Oops, $self.app_name crashed. We do our best to make it stable, but...
 
 A crash report was automatically generated with the following information:
   - A verbatim copy of the crash traceback.
   - A copy of your input history during this session.
   - Data on your current $self.app_name configuration.
 
 It was left in the file named:
 \t'$self.crash_report_fname'
 If you can email this file to the developers, the information in it will help
 them in understanding and correcting the problem.
 
 You can mail it to: $self.contact_name at $self.contact_email
 with the subject '$self.app_name Crash Report'.
 
 If you want to do it now, the following command will work (under Unix):
 mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
 
 To ensure accurate tracking of this issue, please file a report about it at:
 $self.bug_tracker
 """
 
 
 class CrashHandler(object):
     """Customizable crash handlers for IPython applications.
 
     Instances of this class provide a :meth:`__call__` method which can be
     used as a ``sys.excepthook``.  The :meth:`__call__` signature is::
 
         def __call__(self, etype, evalue, etb)
     """
 
     message_template = _default_message_template
 
     def __init__(self, app, contact_name=None, contact_email=None, 
                  bug_tracker=None, show_crash_traceback=True, call_pdb=False):
         """Create a new crash handler
 
         Parameters
         ----------
         app :  Application
             A running :class:`Application` instance, which will be queried at 
             crash time for internal information.
 
         contact_name : str
             A string with the name of the person to contact.
 
         contact_email : str
             A string with the email address of the contact.
 
         bug_tracker : str
             A string with the URL for your project's bug tracker.
 
         show_crash_traceback : bool
             If false, don't print the crash traceback on stderr, only generate
             the on-disk report
 
         Non-argument instance attributes:
 
         These instances contain some non-argument attributes which allow for
         further customization of the crash handler's behavior. Please see the
         source for further details.
         """
         self.app = app
         self.app_name = self.app.name
         self.contact_name = contact_name
         self.contact_email = contact_email
         self.bug_tracker = bug_tracker
         self.crash_report_fname = "Crash_report_%s.txt" % self.app_name
         self.show_crash_traceback = show_crash_traceback
         self.section_sep = '\n\n'+'*'*75+'\n\n'
         self.call_pdb = call_pdb
         #self.call_pdb = True # dbg
 
     def __call__(self, etype, evalue, etb):
         """Handle an exception, call for compatible with sys.excepthook"""
 
         # Report tracebacks shouldn't use color in general (safer for users)
         color_scheme = 'NoColor'
 
         # Use this ONLY for developer debugging (keep commented out for release)
         #color_scheme = 'Linux'   # dbg
-        
         try:
             rptdir = self.app.ipython_dir
         except:
             rptdir = os.getcwd()
-        if not os.path.isdir(rptdir):
+        if rptdir is None or not os.path.isdir(rptdir):
             rptdir = os.getcwd()
         report_name = os.path.join(rptdir,self.crash_report_fname)
         # write the report filename into the instance dict so it can get
         # properly expanded out in the user message template
         self.crash_report_fname = report_name
         TBhandler = ultratb.VerboseTB(
             color_scheme=color_scheme,
             long_header=1,
             call_pdb=self.call_pdb,
         )
         if self.call_pdb:
             TBhandler(etype,evalue,etb)
             return
         else:
             traceback = TBhandler.text(etype,evalue,etb,context=31)
 
         # print traceback to screen
         if self.show_crash_traceback:
             print >> sys.stderr, traceback
 
         # and generate a complete report on disk
         try:
             report = open(report_name,'w')
         except:
             print >> sys.stderr, 'Could not create crash report on disk.'
             return
 
         # Inform user on stderr of what happened
         msg = itpl('\n'+'*'*70+'\n'+self.message_template)
         print >> sys.stderr, msg
 
         # Construct report on disk
         report.write(self.make_report(traceback))
         report.close()
         raw_input("Hit <Enter> to quit this message (your terminal may close):")
 
     def make_report(self,traceback):
         """Return a string containing a crash report."""
         
         sec_sep = self.section_sep
         
         report = ['*'*75+'\n\n'+'IPython post-mortem report\n\n']
         rpt_add = report.append
         rpt_add(sys_info())
         
         try:
             config = pformat(self.app.config)
             rpt_add(sec_sep)
             rpt_add('Application name: %s\n\n' % self.app_name)
             rpt_add('Current user configuration structure:\n\n')
             rpt_add(config)
         except:
             pass
         rpt_add(sec_sep+'Crash traceback:\n\n' + traceback)
 
         return ''.join(report)
 

IPython/core/magic.py

@@ -1,3700 +1,3708 b''
 # encoding: utf-8
 """Magic functions for InteractiveShell.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001 Janko Hauser <[e-mail]> and
 #  Copyright (C) 2001-2007 Fernando Perez <[e-mail]>
 #  Copyright (C) 2008-2009  The IPython Development Team
 
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import __builtin__
 import __future__
 import bdb
 import inspect
 import os
 import sys
 import shutil
 import re
 import time
 import textwrap
 import types
 from cStringIO import StringIO
 from getopt import getopt,GetoptError
 from pprint import pformat
 
 # cProfile was added in Python2.5
 try:
     import cProfile as profile
     import pstats
 except ImportError:
     # profile isn't bundled by default in Debian for license reasons
     try:
         import profile,pstats
     except ImportError:
         profile = pstats = None
 
 # print_function was added to __future__ in Python2.6, remove this when we drop
 # 2.5 compatibility
 if not hasattr(__future__,'CO_FUTURE_PRINT_FUNCTION'):
     __future__.CO_FUTURE_PRINT_FUNCTION = 65536
 
 import IPython
 from IPython.core import debugger, oinspect
 from IPython.core.error import TryNext
 from IPython.core.error import UsageError
 from IPython.core.fakemodule import FakeModule
 from IPython.core.macro import Macro
 from IPython.core.page import page
 from IPython.core.prefilter import ESC_MAGIC
 from IPython.lib.pylabtools import mpl_runner
 from IPython.lib.inputhook import enable_gui
 from IPython.external.Itpl import itpl, printpl
 from IPython.testing import decorators as testdec
 from IPython.utils.io import Term, file_read, nlprint
 from IPython.utils.path import get_py_filename
 from IPython.utils.process import arg_split, abbrev_cwd
 from IPython.utils.terminal import set_term_title
 from IPython.utils.text import LSString, SList, StringTypes
 from IPython.utils.timing import clock, clock2
 from IPython.utils.warn import warn, error
 from IPython.utils.ipstruct import Struct
 import IPython.utils.generics
 
 #-----------------------------------------------------------------------------
 # Utility functions
 #-----------------------------------------------------------------------------
 
 def on_off(tag):
     """Return an ON/OFF string for a 1/0 input. Simple utility function."""
     return ['OFF','ON'][tag]
 
 class Bunch: pass
 
 def compress_dhist(dh):
     head, tail = dh[:-10], dh[-10:]
 
     newhead = []
     done = set()
     for h in head:
         if h in done:
             continue
         newhead.append(h)
         done.add(h)
 
     return newhead + tail        
 
 
 #***************************************************************************
 # Main class implementing Magic functionality
 
 # XXX - for some odd reason, if Magic is made a new-style class, we get errors
 # on construction of the main InteractiveShell object.  Something odd is going
 # on with super() calls, Component and the MRO... For now leave it as-is, but
 # eventually this needs to be clarified.
 # BG: This is because InteractiveShell inherits from this, but is itself a 
 # Component. This messes up the MRO in some way. The fix is that we need to
 # make Magic a component that InteractiveShell does not subclass.
 
 class Magic:
     """Magic functions for InteractiveShell.
 
     Shell functions which can be reached as %function_name. All magic
     functions should accept a string, which they can parse for their own
     needs. This can make some functions easier to type, eg `%cd ../`
     vs. `%cd("../")`
 
     ALL definitions MUST begin with the prefix magic_. The user won't need it
     at the command line, but it is is needed in the definition. """
 
     # class globals
     auto_status = ['Automagic is OFF, % prefix IS needed for magic functions.',
                    'Automagic is ON, % prefix NOT needed for magic functions.']
 
     #......................................................................
     # some utility functions
 
     def __init__(self,shell):
         
         self.options_table = {}
         if profile is None:
             self.magic_prun = self.profile_missing_notice
         self.shell = shell
 
         # namespace for holding state we may need
         self._magic_state = Bunch()
 
     def profile_missing_notice(self, *args, **kwargs):
         error("""\
 The profile module could not be found. It has been removed from the standard
 python packages because of its non-free license. To use profiling, install the
 python-profiler package from non-free.""")
 
     def default_option(self,fn,optstr):
         """Make an entry in the options_table for fn, with value optstr"""
 
         if fn not in self.lsmagic():
             error("%s is not a magic function" % fn)
         self.options_table[fn] = optstr
 
     def lsmagic(self):
         """Return a list of currently available magic functions.
 
         Gives a list of the bare names after mangling (['ls','cd', ...], not
         ['magic_ls','magic_cd',...]"""
 
         # FIXME. This needs a cleanup, in the way the magics list is built.
         
         # magics in class definition
         class_magic = lambda fn: fn.startswith('magic_') and \
                       callable(Magic.__dict__[fn])
         # in instance namespace (run-time user additions)
         inst_magic =  lambda fn: fn.startswith('magic_') and \
                      callable(self.__dict__[fn])
         # and bound magics by user (so they can access self):
         inst_bound_magic =  lambda fn: fn.startswith('magic_') and \
                            callable(self.__class__.__dict__[fn])
         magics = filter(class_magic,Magic.__dict__.keys()) + \
                  filter(inst_magic,self.__dict__.keys()) + \
                  filter(inst_bound_magic,self.__class__.__dict__.keys())
         out = []
         for fn in set(magics):
             out.append(fn.replace('magic_','',1))
         out.sort()
         return out
     
     def extract_input_slices(self,slices,raw=False):
         """Return as a string a set of input history slices.
 
         Inputs:
 
           - slices: the set of slices is given as a list of strings (like
           ['1','4:8','9'], since this function is for use by magic functions
           which get their arguments as strings.
 
         Optional inputs:
 
           - raw(False): by default, the processed input is used.  If this is
           true, the raw input history is used instead.
 
         Note that slices can be called with two notations:
 
         N:M -> standard python form, means including items N...(M-1).
 
         N-M -> include items N..M (closed endpoint)."""
 
         if raw:
             hist = self.shell.input_hist_raw
         else:
             hist = self.shell.input_hist
 
         cmds = []
         for chunk in slices:
             if ':' in chunk:
                 ini,fin = map(int,chunk.split(':'))
             elif '-' in chunk:
                 ini,fin = map(int,chunk.split('-'))
                 fin += 1
             else:
                 ini = int(chunk)
                 fin = ini+1
             cmds.append(hist[ini:fin])
         return cmds
         
     def _ofind(self, oname, namespaces=None):
         """Find an object in the available namespaces.
 
         self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
 
         Has special code to detect magic functions.
         """
         oname = oname.strip()
         alias_ns = None
         if namespaces is None:
             # Namespaces to search in:
             # Put them in a list. The order is important so that we
             # find things in the same order that Python finds them.
             namespaces = [ ('Interactive', self.shell.user_ns),
                            ('IPython internal', self.shell.internal_ns),
                            ('Python builtin', __builtin__.__dict__),
                            ('Alias', self.shell.alias_manager.alias_table),
                            ]
             alias_ns = self.shell.alias_manager.alias_table
 
         # initialize results to 'null'
         found = False; obj = None;  ospace = None;  ds = None;
         ismagic = False; isalias = False; parent = None
 
         # We need to special-case 'print', which as of python2.6 registers as a
         # function but should only be treated as one if print_function was
         # loaded with a future import.  In this case, just bail.
         if (oname == 'print' and not (self.shell.compile.compiler.flags &
                                       __future__.CO_FUTURE_PRINT_FUNCTION)):
             return {'found':found, 'obj':obj, 'namespace':ospace,
                     'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
 
         # Look for the given name by splitting it in parts.  If the head is
         # found, then we look for all the remaining parts as members, and only
         # declare success if we can find them all.
         oname_parts = oname.split('.')
         oname_head, oname_rest = oname_parts[0],oname_parts[1:]
         for nsname,ns in namespaces:
             try:
                 obj = ns[oname_head]
             except KeyError:
                 continue
             else:
                 #print 'oname_rest:', oname_rest  # dbg
                 for part in oname_rest:
                     try:
                         parent = obj
                         obj = getattr(obj,part)
                     except:
                         # Blanket except b/c some badly implemented objects
                         # allow __getattr__ to raise exceptions other than
                         # AttributeError, which then crashes IPython.
                         break
                 else:
                     # If we finish the for loop (no break), we got all members
                     found = True
                     ospace = nsname
                     if ns == alias_ns:
                         isalias = True
                     break  # namespace loop
 
         # Try to see if it's magic
         if not found:
             if oname.startswith(ESC_MAGIC):
                 oname = oname[1:]
             obj = getattr(self,'magic_'+oname,None)
             if obj is not None:
                 found = True
                 ospace = 'IPython internal'
                 ismagic = True
 
         # Last try: special-case some literals like '', [], {}, etc:
         if not found and oname_head in ["''",'""','[]','{}','()']:
             obj = eval(oname_head)
             found = True
             ospace = 'Interactive'
             
         return {'found':found, 'obj':obj, 'namespace':ospace,
                 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
     
     def arg_err(self,func):
         """Print docstring if incorrect arguments were passed"""
         print 'Error in arguments:'
         print oinspect.getdoc(func)
 
     def format_latex(self,strng):
         """Format a string for latex inclusion."""
 
         # Characters that need to be escaped for latex:
         escape_re = re.compile(r'(%|_|\$|#|&)',re.MULTILINE)
         # Magic command names as headers:
         cmd_name_re = re.compile(r'^(%s.*?):' % ESC_MAGIC,
                                  re.MULTILINE)
         # Magic commands 
         cmd_re = re.compile(r'(?P<cmd>%s.+?\b)(?!\}\}:)' % ESC_MAGIC,
                             re.MULTILINE)
         # Paragraph continue
         par_re = re.compile(r'\\$',re.MULTILINE)
 
         # The "\n" symbol
         newline_re = re.compile(r'\\n')
 
         # Now build the string for output:
         #strng = cmd_name_re.sub(r'\n\\texttt{\\textsl{\\large \1}}:',strng)
         strng = cmd_name_re.sub(r'\n\\bigskip\n\\texttt{\\textbf{ \1}}:',
                                 strng)
         strng = cmd_re.sub(r'\\texttt{\g<cmd>}',strng)
         strng = par_re.sub(r'\\\\',strng)
         strng = escape_re.sub(r'\\\1',strng)
         strng = newline_re.sub(r'\\textbackslash{}n',strng)
         return strng
 
     def format_screen(self,strng):
         """Format a string for screen printing.
 
         This removes some latex-type format codes."""
         # Paragraph continue
         par_re = re.compile(r'\\$',re.MULTILINE)
         strng = par_re.sub('',strng)
         return strng
 
     def parse_options(self,arg_str,opt_str,*long_opts,**kw):
         """Parse options passed to an argument string.
 
         The interface is similar to that of getopt(), but it returns back a
         Struct with the options as keys and the stripped argument string still
         as a string.
 
         arg_str is quoted as a true sys.argv vector by using shlex.split.
         This allows us to easily expand variables, glob files, quote
         arguments, etc.
 
         Options:
           -mode: default 'string'. If given as 'list', the argument string is
           returned as a list (split on whitespace) instead of a string.
 
           -list_all: put all option values in lists. Normally only options
           appearing more than once are put in a list.
 
           -posix (True): whether to split the input line in POSIX mode or not,
           as per the conventions outlined in the shlex module from the
           standard library."""
  
         # inject default options at the beginning of the input line
         caller = sys._getframe(1).f_code.co_name.replace('magic_','')
         arg_str = '%s %s' % (self.options_table.get(caller,''),arg_str)
         
         mode = kw.get('mode','string')
         if mode not in ['string','list']:
             raise ValueError,'incorrect mode given: %s' % mode
         # Get options
         list_all = kw.get('list_all',0)
         posix = kw.get('posix', os.name == 'posix')
 
         # Check if we have more than one argument to warrant extra processing:
         odict = {}  # Dictionary with options
         args = arg_str.split()
         if len(args) >= 1:
             # If the list of inputs only has 0 or 1 thing in it, there's no
             # need to look for options
             argv = arg_split(arg_str,posix)
             # Do regular option processing
             try:
                 opts,args = getopt(argv,opt_str,*long_opts)
             except GetoptError,e:
                 raise UsageError('%s ( allowed: "%s" %s)' % (e.msg,opt_str, 
                                         " ".join(long_opts)))
             for o,a in opts:
                 if o.startswith('--'):
                     o = o[2:]
                 else:
                     o = o[1:]
                 try:
                     odict[o].append(a)
                 except AttributeError:
                     odict[o] = [odict[o],a]
                 except KeyError:
                     if list_all:
                         odict[o] = [a]
                     else:
                         odict[o] = a
 
         # Prepare opts,args for return
         opts = Struct(odict)
         if mode == 'string':
             args = ' '.join(args)
 
         return opts,args
     
     #......................................................................
     # And now the actual magic functions
 
     # Functions for IPython shell work (vars,funcs, config, etc)
     def magic_lsmagic(self, parameter_s = ''):
         """List currently available magic functions."""
         mesc = ESC_MAGIC
         print 'Available magic functions:\n'+mesc+\
               ('  '+mesc).join(self.lsmagic())
         print '\n' + Magic.auto_status[self.shell.automagic]
         return None
         
     def magic_magic(self, parameter_s = ''):
         """Print information about the magic function system.
         
         Supported formats: -latex, -brief, -rest        
         """
 
         mode = ''
         try:
             if parameter_s.split()[0] == '-latex':
                 mode = 'latex'
             if parameter_s.split()[0] == '-brief':
                 mode = 'brief'
             if parameter_s.split()[0] == '-rest':
                 mode = 'rest'
                 rest_docs = []
         except:
             pass
 
         magic_docs = []
         for fname in self.lsmagic():
             mname = 'magic_' + fname
             for space in (Magic,self,self.__class__):
                 try:
                     fn = space.__dict__[mname]
                 except KeyError:
                     pass
                 else:
                     break
             if mode == 'brief':
                 # only first line
                 if fn.__doc__:                    
                     fndoc = fn.__doc__.split('\n',1)[0]
                 else:
                     fndoc = 'No documentation'
             else:
                 if fn.__doc__:
                     fndoc = fn.__doc__.rstrip()       
                 else:
                     fndoc = 'No documentation'
                 
                 
             if mode == 'rest':
                 rest_docs.append('**%s%s**::\n\n\t%s\n\n' %(ESC_MAGIC,
                                                     fname,fndoc))
                 
             else:
                 magic_docs.append('%s%s:\n\t%s\n' %(ESC_MAGIC,
                                                     fname,fndoc))
                 
         magic_docs = ''.join(magic_docs)
 
         if mode == 'rest':
             return "".join(rest_docs)
         
         if mode == 'latex':
             print self.format_latex(magic_docs)
             return
         else:
             magic_docs = self.format_screen(magic_docs)
         if mode == 'brief':
             return magic_docs
         
         outmsg = """
 IPython's 'magic' functions
 ===========================
 
 The magic function system provides a series of functions which allow you to
 control the behavior of IPython itself, plus a lot of system-type
 features. All these functions are prefixed with a % character, but parameters
 are given without parentheses or quotes.
 
 NOTE: If you have 'automagic' enabled (via the command line option or with the
 %automagic function), you don't need to type in the % explicitly.  By default,
 IPython ships with automagic on, so you should only rarely need the % escape.
 
 Example: typing '%cd mydir' (without the quotes) changes you working directory
 to 'mydir', if it exists.
 
 You can define your own magic functions to extend the system. See the supplied
 ipythonrc and example-magic.py files for details (in your ipython
 configuration directory, typically $HOME/.ipython/).
 
 You can also define your own aliased names for magic functions. In your
 ipythonrc file, placing a line like:
 
   execute __IPYTHON__.magic_pf = __IPYTHON__.magic_profile
 
 will define %pf as a new name for %profile.
 
 You can also call magics in code using the magic() function, which IPython
 automatically adds to the builtin namespace.  Type 'magic?' for details.
 
 For a list of the available magic functions, use %lsmagic. For a description
 of any of them, type %magic_name?, e.g. '%cd?'.
 
 Currently the magic system has the following functions:\n"""
 
         mesc = ESC_MAGIC
         outmsg = ("%s\n%s\n\nSummary of magic functions (from %slsmagic):"
                   "\n\n%s%s\n\n%s" % (outmsg,
                                      magic_docs,mesc,mesc,
                                      ('  '+mesc).join(self.lsmagic()),
                                      Magic.auto_status[self.shell.automagic] ) )
 
         page(outmsg,screen_lines=self.shell.usable_screen_length)
   
 
     def magic_autoindent(self, parameter_s = ''):
         """Toggle autoindent on/off (if available)."""
 
         self.shell.set_autoindent()
         print "Automatic indentation is:",['OFF','ON'][self.shell.autoindent]
 
 
     def magic_automagic(self, parameter_s = ''):
         """Make magic functions callable without having to type the initial %.
 
         Without argumentsl toggles on/off (when off, you must call it as
         %automagic, of course).  With arguments it sets the value, and you can
         use any of (case insensitive):
 
          - on,1,True: to activate
 
          - off,0,False: to deactivate.
 
         Note that magic functions have lowest priority, so if there's a
         variable whose name collides with that of a magic fn, automagic won't
         work for that function (you get the variable instead). However, if you
         delete the variable (del var), the previously shadowed magic function
         becomes visible to automagic again."""
 
         arg = parameter_s.lower()
         if parameter_s in ('on','1','true'):
             self.shell.automagic = True
         elif parameter_s in ('off','0','false'):
             self.shell.automagic = False
         else:
             self.shell.automagic = not self.shell.automagic
         print '\n' + Magic.auto_status[self.shell.automagic]
 
     @testdec.skip_doctest
     def magic_autocall(self, parameter_s = ''):
         """Make functions callable without having to type parentheses.
 
         Usage:
 
            %autocall [mode]
 
         The mode can be one of: 0->Off, 1->Smart, 2->Full.  If not given, the
         value is toggled on and off (remembering the previous state).
 
         In more detail, these values mean:
 
         0 -> fully disabled
 
         1 -> active, but do not apply if there are no arguments on the line.
 
         In this mode, you get:
 
         In [1]: callable
         Out[1]: <built-in function callable>
 
         In [2]: callable 'hello'
         ------> callable('hello')
         Out[2]: False
 
         2 -> Active always.  Even if no arguments are present, the callable
         object is called:
 
         In [2]: float
         ------> float()
         Out[2]: 0.0
 
         Note that even with autocall off, you can still use '/' at the start of
         a line to treat the first argument on the command line as a function
         and add parentheses to it:
 
         In [8]: /str 43
         ------> str(43)
         Out[8]: '43'
 
         # all-random (note for auto-testing)
         """
 
         if parameter_s:
             arg = int(parameter_s)
         else:
             arg = 'toggle'
 
         if not arg in (0,1,2,'toggle'):
             error('Valid modes: (0->Off, 1->Smart, 2->Full')
             return
 
         if arg in (0,1,2):
             self.shell.autocall = arg
         else: # toggle
             if self.shell.autocall:
                 self._magic_state.autocall_save = self.shell.autocall
                 self.shell.autocall = 0
             else:
                 try:
                     self.shell.autocall = self._magic_state.autocall_save
                 except AttributeError:
                     self.shell.autocall = self._magic_state.autocall_save = 1
 
         print "Automatic calling is:",['OFF','Smart','Full'][self.shell.autocall]
 
     def magic_system_verbose(self, parameter_s = ''):
         """Set verbose printing of system calls.
 
         If called without an argument, act as a toggle"""
 
         if parameter_s:
             val = bool(eval(parameter_s))
         else:
             val = None
 
         if self.shell.system_verbose:
             self.shell.system_verbose = False
         else:
             self.shell.system_verbose = True
         print "System verbose printing is:",\
               ['OFF','ON'][self.shell.system_verbose]
 
 
     def magic_page(self, parameter_s=''):
         """Pretty print the object and display it through a pager.
         
         %page [options] OBJECT
 
         If no object is given, use _ (last output).
         
         Options:
 
           -r: page str(object), don't pretty-print it."""
 
         # After a function contributed by Olivier Aubert, slightly modified.
 
         # Process options/args
         opts,args = self.parse_options(parameter_s,'r')
         raw = 'r' in opts
 
         oname = args and args or '_'
         info = self._ofind(oname)
         if info['found']:
             txt = (raw and str or pformat)( info['obj'] )
             page(txt)
         else:
             print 'Object `%s` not found' % oname
 
     def magic_profile(self, parameter_s=''):
         """Print your currently active IPython profile."""
         if self.shell.profile:
             printpl('Current IPython profile: $self.shell.profile.')
         else:
             print 'No profile active.'
 
     def magic_pinfo(self, parameter_s='', namespaces=None):
         """Provide detailed information about an object.
     
         '%pinfo object' is just a synonym for object? or ?object."""
     
         #print 'pinfo par: <%s>' % parameter_s  # dbg
     
         
         # detail_level: 0 -> obj? , 1 -> obj??
         detail_level = 0
         # We need to detect if we got called as 'pinfo pinfo foo', which can
         # happen if the user types 'pinfo foo?' at the cmd line.
         pinfo,qmark1,oname,qmark2 = \
                re.match('(pinfo )?(\?*)(.*?)(\??$)',parameter_s).groups()
         if pinfo or qmark1 or qmark2:
             detail_level = 1
         if "*" in oname:
             self.magic_psearch(oname)
         else:
             self._inspect('pinfo', oname, detail_level=detail_level,
                           namespaces=namespaces)
 
     def magic_pdef(self, parameter_s='', namespaces=None):
         """Print the definition header for any callable object.
 
         If the object is a class, print the constructor information."""
         self._inspect('pdef',parameter_s, namespaces)
 
     def magic_pdoc(self, parameter_s='', namespaces=None):
         """Print the docstring for an object.
 
         If the given object is a class, it will print both the class and the
         constructor docstrings."""
         self._inspect('pdoc',parameter_s, namespaces)
 
     def magic_psource(self, parameter_s='', namespaces=None):
         """Print (or run through pager) the source code for an object."""
         self._inspect('psource',parameter_s, namespaces)
 
     def magic_pfile(self, parameter_s=''):
         """Print (or run through pager) the file where an object is defined.
 
         The file opens at the line where the object definition begins. IPython
         will honor the environment variable PAGER if set, and otherwise will
         do its best to print the file in a convenient form.
 
         If the given argument is not an object currently defined, IPython will
         try to interpret it as a filename (automatically adding a .py extension
         if needed). You can thus use %pfile as a syntax highlighting code
         viewer."""
 
         # first interpret argument as an object name
         out = self._inspect('pfile',parameter_s)
         # if not, try the input as a filename
         if out == 'not found':
             try:
                 filename = get_py_filename(parameter_s)
             except IOError,msg:
                 print msg
                 return
             page(self.shell.inspector.format(file(filename).read()))
 
     def _inspect(self,meth,oname,namespaces=None,**kw):
         """Generic interface to the inspector system.
 
         This function is meant to be called by pdef, pdoc & friends."""
 
         #oname = oname.strip()
         #print '1- oname: <%r>' % oname  # dbg
         try:
             oname = oname.strip().encode('ascii')
             #print '2- oname: <%r>' % oname  # dbg
         except UnicodeEncodeError:
             print 'Python identifiers can only contain ascii characters.'
             return 'not found'
             
         info = Struct(self._ofind(oname, namespaces))
         
         if info.found:
             try:
                 IPython.utils.generics.inspect_object(info.obj)
                 return
             except TryNext:
                 pass
             # Get the docstring of the class property if it exists.
             path = oname.split('.')
             root = '.'.join(path[:-1])
             if info.parent is not None:
                 try:
                     target = getattr(info.parent, '__class__') 
                     # The object belongs to a class instance. 
                     try: 
                         target = getattr(target, path[-1])
                         # The class defines the object. 
                         if isinstance(target, property):
                             oname = root + '.__class__.' + path[-1]
                             info = Struct(self._ofind(oname))
                     except AttributeError: pass
                 except AttributeError: pass
                         
             pmethod = getattr(self.shell.inspector,meth)
             formatter = info.ismagic and self.format_screen or None
             if meth == 'pdoc':
                 pmethod(info.obj,oname,formatter)
             elif meth == 'pinfo':
                 pmethod(info.obj,oname,formatter,info,**kw)
             else:
                 pmethod(info.obj,oname)
         else:
             print 'Object `%s` not found.' % oname
             return 'not found'  # so callers can take other action
         
     def magic_psearch(self, parameter_s=''):
         """Search for object in namespaces by wildcard.
 
         %psearch [options] PATTERN [OBJECT TYPE]
 
         Note: ? can be used as a synonym for %psearch, at the beginning or at
         the end: both a*? and ?a* are equivalent to '%psearch a*'.  Still, the
         rest of the command line must be unchanged (options come first), so
         for example the following forms are equivalent
 
         %psearch -i a* function
         -i a* function?
         ?-i a* function
 
         Arguments:
         
           PATTERN
 
           where PATTERN is a string containing * as a wildcard similar to its
           use in a shell.  The pattern is matched in all namespaces on the
           search path. By default objects starting with a single _ are not
           matched, many IPython generated objects have a single
           underscore. The default is case insensitive matching. Matching is
           also done on the attributes of objects and not only on the objects
           in a module.
 
           [OBJECT TYPE]
 
           Is the name of a python type from the types module. The name is
           given in lowercase without the ending type, ex. StringType is
           written string. By adding a type here only objects matching the
           given type are matched. Using all here makes the pattern match all
           types (this is the default).
 
         Options:
 
           -a: makes the pattern match even objects whose names start with a
           single underscore.  These names are normally ommitted from the
           search.
 
           -i/-c: make the pattern case insensitive/sensitive.  If neither of
           these options is given, the default is read from your ipythonrc
           file.  The option name which sets this value is
           'wildcards_case_sensitive'.  If this option is not specified in your
           ipythonrc file, IPython's internal default is to do a case sensitive
           search.
 
           -e/-s NAMESPACE: exclude/search a given namespace.  The pattern you
           specifiy can be searched in any of the following namespaces:
           'builtin', 'user', 'user_global','internal', 'alias', where
           'builtin' and 'user' are the search defaults.  Note that you should
           not use quotes when specifying namespaces.
 
           'Builtin' contains the python module builtin, 'user' contains all
           user data, 'alias' only contain the shell aliases and no python
           objects, 'internal' contains objects used by IPython.  The
           'user_global' namespace is only used by embedded IPython instances,
           and it contains module-level globals.  You can add namespaces to the
           search with -s or exclude them with -e (these options can be given
           more than once).
     
         Examples:
        
         %psearch a*            -> objects beginning with an a
         %psearch -e builtin a* -> objects NOT in the builtin space starting in a
         %psearch a* function   -> all functions beginning with an a
         %psearch re.e*         -> objects beginning with an e in module re
         %psearch r*.e*         -> objects that start with e in modules starting in r
         %psearch r*.* string   -> all strings in modules beginning with r
 
         Case sensitve search:
        
         %psearch -c a*         list all object beginning with lower case a
 
         Show objects beginning with a single _:
        
         %psearch -a _*         list objects beginning with a single underscore"""
         try:
             parameter_s = parameter_s.encode('ascii')
         except UnicodeEncodeError:
             print 'Python identifiers can only contain ascii characters.'
             return
 
         # default namespaces to be searched
         def_search = ['user','builtin']
 
         # Process options/args
         opts,args = self.parse_options(parameter_s,'cias:e:',list_all=True)
         opt = opts.get
         shell = self.shell
         psearch = shell.inspector.psearch
 
         # select case options
         if opts.has_key('i'):
             ignore_case = True
         elif opts.has_key('c'):
             ignore_case = False
         else:
             ignore_case = not shell.wildcards_case_sensitive
 
         # Build list of namespaces to search from user options
         def_search.extend(opt('s',[]))
         ns_exclude = ns_exclude=opt('e',[])
         ns_search = [nm for nm in def_search if nm not in ns_exclude]
         
         # Call the actual search
         try:
             psearch(args,shell.ns_table,ns_search,
                     show_all=opt('a'),ignore_case=ignore_case)
         except:
             shell.showtraceback()
         
     def magic_who_ls(self, parameter_s=''):
         """Return a sorted list of all interactive variables.
 
         If arguments are given, only variables of types matching these
         arguments are returned."""
 
         user_ns = self.shell.user_ns
         internal_ns = self.shell.internal_ns
         user_ns_hidden = self.shell.user_ns_hidden
         out = [ i for i in user_ns
                 if not i.startswith('_') \
                 and not (i in internal_ns or i in user_ns_hidden) ]
 
         typelist = parameter_s.split()
         if typelist:
             typeset = set(typelist)
             out = [i for i in out if type(i).__name__ in typeset]
 
         out.sort()
         return out
         
     def magic_who(self, parameter_s=''):
         """Print all interactive variables, with some minimal formatting.
 
         If any arguments are given, only variables whose type matches one of
         these are printed.  For example:
 
           %who function str
 
         will only list functions and strings, excluding all other types of
         variables.  To find the proper type names, simply use type(var) at a
         command line to see how python prints type names.  For example:
 
           In [1]: type('hello')\\
           Out[1]: <type 'str'>
 
         indicates that the type name for strings is 'str'.
 
         %who always excludes executed names loaded through your configuration
         file and things which are internal to IPython.
 
         This is deliberate, as typically you may load many modules and the
         purpose of %who is to show you only what you've manually defined."""
 
         varlist = self.magic_who_ls(parameter_s)
         if not varlist:
             if parameter_s:
                 print 'No variables match your requested type.'
             else:
                 print 'Interactive namespace is empty.'
             return
 
         # if we have variables, move on...
         count = 0
         for i in varlist:
             print i+'\t',
             count += 1
             if count > 8:
                 count = 0
                 print
         print
 
     def magic_whos(self, parameter_s=''):
         """Like %who, but gives some extra information about each variable.
 
         The same type filtering of %who can be applied here.
 
         For all variables, the type is printed. Additionally it prints:
         
           - For {},[],(): their length.
 
           - For numpy and Numeric arrays, a summary with shape, number of
           elements, typecode and size in memory.
 
           - Everything else: a string representation, snipping their middle if
           too long."""
         
         varnames = self.magic_who_ls(parameter_s)
         if not varnames:
             if parameter_s:
                 print 'No variables match your requested type.'
             else:
                 print 'Interactive namespace is empty.'
             return
 
         # if we have variables, move on...
 
         # for these types, show len() instead of data:
         seq_types = [types.DictType,types.ListType,types.TupleType]
 
         # for numpy/Numeric arrays, display summary info
         try:
             import numpy
         except ImportError:
             ndarray_type = None
         else:
             ndarray_type = numpy.ndarray.__name__
         try:
             import Numeric
         except ImportError:
             array_type = None
         else:
             array_type = Numeric.ArrayType.__name__
 
         # Find all variable names and types so we can figure out column sizes
         def get_vars(i):
             return self.shell.user_ns[i]
         
         # some types are well known and can be shorter
         abbrevs = {'IPython.core.macro.Macro' : 'Macro'}
         def type_name(v):
             tn = type(v).__name__
             return abbrevs.get(tn,tn)
             
         varlist = map(get_vars,varnames)
 
         typelist = []
         for vv in varlist:
             tt = type_name(vv)
 
             if tt=='instance':
                 typelist.append( abbrevs.get(str(vv.__class__),
                                              str(vv.__class__)))
             else:
                 typelist.append(tt)
 
         # column labels and # of spaces as separator
         varlabel = 'Variable'
         typelabel = 'Type'
         datalabel = 'Data/Info'
         colsep = 3
         # variable format strings
         vformat    = "$vname.ljust(varwidth)$vtype.ljust(typewidth)"
         vfmt_short = '$vstr[:25]<...>$vstr[-25:]'
         aformat    = "%s: %s elems, type `%s`, %s bytes"
         # find the size of the columns to format the output nicely
         varwidth = max(max(map(len,varnames)), len(varlabel)) + colsep
         typewidth = max(max(map(len,typelist)), len(typelabel)) + colsep
         # table header
         print varlabel.ljust(varwidth) + typelabel.ljust(typewidth) + \
               ' '+datalabel+'\n' + '-'*(varwidth+typewidth+len(datalabel)+1)
         # and the table itself
         kb = 1024
         Mb = 1048576  # kb**2
         for vname,var,vtype in zip(varnames,varlist,typelist):
             print itpl(vformat),
             if vtype in seq_types:
                 print len(var)
             elif vtype in [array_type,ndarray_type]:
                 vshape = str(var.shape).replace(',','').replace(' ','x')[1:-1]
                 if vtype==ndarray_type:
                     # numpy
                     vsize  = var.size
                     vbytes = vsize*var.itemsize
                     vdtype = var.dtype
                 else:
                     # Numeric
                     vsize  = Numeric.size(var)
                     vbytes = vsize*var.itemsize()
                     vdtype = var.typecode()
                     
                 if vbytes < 100000:
                     print aformat % (vshape,vsize,vdtype,vbytes)
                 else:
                     print aformat % (vshape,vsize,vdtype,vbytes),
                     if vbytes < Mb:
                         print '(%s kb)' % (vbytes/kb,)
                     else:
                         print '(%s Mb)' % (vbytes/Mb,)
             else:
                 try:
                     vstr = str(var)
                 except UnicodeEncodeError:
                     vstr = unicode(var).encode(sys.getdefaultencoding(),
                                                'backslashreplace')
                 vstr = vstr.replace('\n','\\n')
                 if len(vstr) < 50:
                     print vstr
                 else:
                     printpl(vfmt_short)
                 
     def magic_reset(self, parameter_s=''):
         """Resets the namespace by removing all names defined by the user.
 
         Input/Output history are left around in case you need them.
 
         Parameters
         ----------
           -y : force reset without asking for confirmation.
 
         Examples
         --------
         In [6]: a = 1
 
         In [7]: a
         Out[7]: 1
 
         In [8]: 'a' in _ip.user_ns
         Out[8]: True
 
         In [9]: %reset -f
 
         In [10]: 'a' in _ip.user_ns
         Out[10]: False
         """
 
         if parameter_s == '-f':
             ans = True
         else:
             ans = self.shell.ask_yes_no(
                 "Once deleted, variables cannot be recovered. Proceed (y/[n])? ")
         if not ans:
             print 'Nothing done.'
             return
         user_ns = self.shell.user_ns
         for i in self.magic_who_ls():
             del(user_ns[i])
             
         # Also flush the private list of module references kept for script
         # execution protection
         self.shell.clear_main_mod_cache()
 
     def magic_reset_selective(self, parameter_s=''):
         """Resets the namespace by removing names defined by the user.
 
         Input/Output history are left around in case you need them.
 
         %reset_selective [-f] regex
 
         No action is taken if regex is not included
         
         Options
           -f : force reset without asking for confirmation.
 
         Examples
         --------
+
+        We first fully reset the namespace so your output looks identical to
+        this example for pedagogical reasons; in practice you do not need a
+        full reset.
         
-        In [1]: a=1; b=2; c=3; b1m=4; b2m=5; b3m=6; b4m=7; b2s=8
+        In [1]: %reset -f
 
-        In [2]: who_ls
-        Out[2]: ['a', 'b', 'b1', 'b1m', 'b2m', 'b2s', 'b3m', 'b4m', 'c']
+        Now, with a clean namespace we can make a few variables and use
+        %reset_selective to only delete names that match our regexp:
 
-        In [3]: %reset_selective -f b[2-3]m
+        In [2]: a=1; b=2; c=3; b1m=4; b2m=5; b3m=6; b4m=7; b2s=8
 
-        In [4]: who_ls
-        Out[4]: ['a', 'b', 'b1', 'b1m', 'b2s', 'c']
+        In [3]: who_ls
+        Out[3]: ['a', 'b', 'b1m', 'b2m', 'b2s', 'b3m', 'b4m', 'c']
 
-        In [5]: %reset_selective -f d
+        In [4]: %reset_selective -f b[2-3]m
 
-        In [6]: who_ls
-        Out[6]: ['a', 'b', 'b1', 'b1m', 'b2s', 'c']
+        In [5]: who_ls
+        Out[5]: ['a', 'b', 'b1m', 'b2s', 'b4m', 'c']
 
-        In [7]: %reset_selective -f c
-        
-        In [8]: who_ls
-        Out[8]:['a', 'b', 'b1', 'b1m', 'b2s']  
+        In [6]: %reset_selective -f d
 
-        In [9]: %reset_selective -f b
-        
-        In [10]: who_ls        
-        Out[10]: ['a']
-        
+        In [7]: who_ls
+        Out[7]: ['a', 'b', 'b1m', 'b2s', 'b4m', 'c']
+
+        In [8]: %reset_selective -f c
+
+        In [9]: who_ls
+        Out[9]: ['a', 'b', 'b1m', 'b2s', 'b4m']
+
+        In [10]: %reset_selective -f b
+
+        In [11]: who_ls
+        Out[11]: ['a']
         """
         
         opts, regex = self.parse_options(parameter_s,'f')
         
         if opts.has_key('f'):
             ans = True
         else:
             ans = self.shell.ask_yes_no(
                 "Once deleted, variables cannot be recovered. Proceed (y/[n])? ")
         if not ans:
             print 'Nothing done.'
             return
         user_ns = self.shell.user_ns
         if not regex:
             print 'No regex pattern specified. Nothing done.'
             return
         else:
             try:
                 m = re.compile(regex)
             except TypeError:
                 raise TypeError('regex must be a string or compiled pattern')
             for i in self.magic_who_ls():
                 if m.search(i): 
                     del(user_ns[i])        
         
     def magic_logstart(self,parameter_s=''):
         """Start logging anywhere in a session.
 
         %logstart [-o|-r|-t] [log_name [log_mode]]
 
         If no name is given, it defaults to a file named 'ipython_log.py' in your
         current directory, in 'rotate' mode (see below).
 
         '%logstart name' saves to file 'name' in 'backup' mode.  It saves your
         history up to that point and then continues logging.
 
         %logstart takes a second optional parameter: logging mode. This can be one
         of (note that the modes are given unquoted):\\
           append: well, that says it.\\
           backup: rename (if exists) to name~ and start name.\\
           global: single logfile in your home dir, appended to.\\
           over  : overwrite existing log.\\
           rotate: create rotating logs name.1~, name.2~, etc.
 
         Options:
 
           -o: log also IPython's output.  In this mode, all commands which
           generate an Out[NN] prompt are recorded to the logfile, right after
           their corresponding input line.  The output lines are always
           prepended with a '#[Out]# ' marker, so that the log remains valid
           Python code.
 
           Since this marker is always the same, filtering only the output from
           a log is very easy, using for example a simple awk call:
 
             awk -F'#\\[Out\\]# ' '{if($2) {print $2}}' ipython_log.py
 
           -r: log 'raw' input.  Normally, IPython's logs contain the processed
           input, so that user lines are logged in their final form, converted
           into valid Python.  For example, %Exit is logged as
           '_ip.magic("Exit").  If the -r flag is given, all input is logged
           exactly as typed, with no transformations applied.
 
           -t: put timestamps before each input line logged (these are put in
           comments)."""
         
         opts,par = self.parse_options(parameter_s,'ort')
         log_output = 'o' in opts
         log_raw_input = 'r' in opts
         timestamp = 't' in opts
 
         logger = self.shell.logger
 
         # if no args are given, the defaults set in the logger constructor by
         # ipytohn remain valid
         if par:
             try:
                 logfname,logmode = par.split()
             except:
                 logfname = par
                 logmode = 'backup'
         else:
             logfname = logger.logfname
             logmode = logger.logmode
         # put logfname into rc struct as if it had been called on the command
         # line, so it ends up saved in the log header Save it in case we need
         # to restore it...
         old_logfile = self.shell.logfile
         if logfname:
             logfname = os.path.expanduser(logfname)
         self.shell.logfile = logfname
 
         loghead = '# IPython log file\n\n'
         try:
             started  = logger.logstart(logfname,loghead,logmode,
                                        log_output,timestamp,log_raw_input)
         except:
             self.shell.logfile = old_logfile
             warn("Couldn't start log: %s" % sys.exc_info()[1])
         else:
             # log input history up to this point, optionally interleaving
             # output if requested
 
             if timestamp:
                 # disable timestamping for the previous history, since we've
                 # lost those already (no time machine here).
                 logger.timestamp = False
 
             if log_raw_input:
                 input_hist = self.shell.input_hist_raw
             else:
                 input_hist = self.shell.input_hist
                 
             if log_output:
                 log_write = logger.log_write
                 output_hist = self.shell.output_hist
                 for n in range(1,len(input_hist)-1):
                     log_write(input_hist[n].rstrip())
                     if n in output_hist:
                         log_write(repr(output_hist[n]),'output')
             else:
                 logger.log_write(input_hist[1:])
             if timestamp:
                 # re-enable timestamping
                 logger.timestamp = True
                 
             print ('Activating auto-logging. '
                    'Current session state plus future input saved.')
             logger.logstate()
 
     def magic_logstop(self,parameter_s=''):
         """Fully stop logging and close log file.
 
         In order to start logging again, a new %logstart call needs to be made,
         possibly (though not necessarily) with a new filename, mode and other
         options."""
         self.logger.logstop()
 
     def magic_logoff(self,parameter_s=''):
         """Temporarily stop logging.
 
         You must have previously started logging."""
         self.shell.logger.switch_log(0)
         
     def magic_logon(self,parameter_s=''):
         """Restart logging.
 
         This function is for restarting logging which you've temporarily
         stopped with %logoff. For starting logging for the first time, you
         must use the %logstart function, which allows you to specify an
         optional log filename."""
         
         self.shell.logger.switch_log(1)
     
     def magic_logstate(self,parameter_s=''):
         """Print the status of the logging system."""
 
         self.shell.logger.logstate()
         
     def magic_pdb(self, parameter_s=''):
         """Control the automatic calling of the pdb interactive debugger.
 
         Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
         argument it works as a toggle.
 
         When an exception is triggered, IPython can optionally call the
         interactive pdb debugger after the traceback printout. %pdb toggles
         this feature on and off.
 
         The initial state of this feature is set in your ipythonrc
         configuration file (the variable is called 'pdb').
 
         If you want to just activate the debugger AFTER an exception has fired,
         without having to type '%pdb on' and rerunning your code, you can use
         the %debug magic."""
 
         par = parameter_s.strip().lower()
 
         if par:
             try:
                 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
             except KeyError:
                 print ('Incorrect argument. Use on/1, off/0, '
                        'or nothing for a toggle.')
                 return
         else:
             # toggle
             new_pdb = not self.shell.call_pdb
 
         # set on the shell
         self.shell.call_pdb = new_pdb
         print 'Automatic pdb calling has been turned',on_off(new_pdb)
 
     def magic_debug(self, parameter_s=''):
         """Activate the interactive debugger in post-mortem mode.
 
         If an exception has just occurred, this lets you inspect its stack
         frames interactively.  Note that this will always work only on the last
         traceback that occurred, so you must call this quickly after an
         exception that you wish to inspect has fired, because if another one
         occurs, it clobbers the previous one.
 
         If you want IPython to automatically do this on every exception, see
         the %pdb magic for more details.
         """
         self.shell.debugger(force=True)
 
     @testdec.skip_doctest
     def magic_prun(self, parameter_s ='',user_mode=1,
                    opts=None,arg_lst=None,prog_ns=None):
 
         """Run a statement through the python code profiler.
 
         Usage:
           %prun [options] statement
 
         The given statement (which doesn't require quote marks) is run via the
         python profiler in a manner similar to the profile.run() function.
         Namespaces are internally managed to work correctly; profile.run
         cannot be used in IPython because it makes certain assumptions about
         namespaces which do not hold under IPython.
 
         Options:
 
         -l <limit>: you can place restrictions on what or how much of the
         profile gets printed. The limit value can be:
 
           * A string: only information for function names containing this string
           is printed.
 
           * An integer: only these many lines are printed.
 
           * A float (between 0 and 1): this fraction of the report is printed
           (for example, use a limit of 0.4 to see the topmost 40% only).
 
         You can combine several limits with repeated use of the option. For
         example, '-l __init__ -l 5' will print only the topmost 5 lines of
         information about class constructors.
 
         -r: return the pstats.Stats object generated by the profiling. This
         object has all the information about the profile in it, and you can
         later use it for further analysis or in other functions.
 
        -s <key>: sort profile by given key. You can provide more than one key
         by using the option several times: '-s key1 -s key2 -s key3...'. The
         default sorting key is 'time'.
 
         The following is copied verbatim from the profile documentation
         referenced below:
 
         When more than one key is provided, additional keys are used as
         secondary criteria when the there is equality in all keys selected
         before them.
         
         Abbreviations can be used for any key names, as long as the
         abbreviation is unambiguous.  The following are the keys currently
         defined:
 
                 Valid Arg       Meaning
                   "calls"      call count
                   "cumulative" cumulative time
                   "file"       file name
                   "module"     file name
                   "pcalls"     primitive call count
                   "line"       line number
                   "name"       function name
                   "nfl"        name/file/line
                   "stdname"    standard name
                   "time"       internal time
 
         Note that all sorts on statistics are in descending order (placing
         most time consuming items first), where as name, file, and line number
         searches are in ascending order (i.e., alphabetical). The subtle
         distinction between "nfl" and "stdname" is that the standard name is a
         sort of the name as printed, which means that the embedded line
         numbers get compared in an odd way.  For example, lines 3, 20, and 40
         would (if the file names were the same) appear in the string order
         "20" "3" and "40".  In contrast, "nfl" does a numeric compare of the
         line numbers.  In fact, sort_stats("nfl") is the same as
         sort_stats("name", "file", "line").
 
         -T <filename>: save profile results as shown on screen to a text
         file. The profile is still shown on screen.
 
         -D <filename>: save (via dump_stats) profile statistics to given
         filename. This data is in a format understod by the pstats module, and
         is generated by a call to the dump_stats() method of profile
         objects. The profile is still shown on screen.
 
         If you want to run complete programs under the profiler's control, use
         '%run -p [prof_opts] filename.py [args to program]' where prof_opts
         contains profiler specific options as described here.
         
         You can read the complete documentation for the profile module with::
         
           In [1]: import profile; profile.help()
         """
 
         opts_def = Struct(D=[''],l=[],s=['time'],T=[''])
         # protect user quote marks
         parameter_s = parameter_s.replace('"',r'\"').replace("'",r"\'")
         
         if user_mode:  # regular user call
             opts,arg_str = self.parse_options(parameter_s,'D:l:rs:T:',
                                               list_all=1)
             namespace = self.shell.user_ns
         else:  # called to run a program by %run -p
             try:
                 filename = get_py_filename(arg_lst[0])
             except IOError,msg:
                 error(msg)
                 return
 
             arg_str = 'execfile(filename,prog_ns)'
             namespace = locals()
 
         opts.merge(opts_def)
         
         prof = profile.Profile()
         try:
             prof = prof.runctx(arg_str,namespace,namespace)
             sys_exit = ''
         except SystemExit:
             sys_exit = """*** SystemExit exception caught in code being profiled."""
 
         stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
 
         lims = opts.l
         if lims:
             lims = []  # rebuild lims with ints/floats/strings
             for lim in opts.l:
                 try:
                     lims.append(int(lim))
                 except ValueError:
                     try:
                         lims.append(float(lim))
                     except ValueError:
                         lims.append(lim)
                     
         # Trap output.
         stdout_trap = StringIO()
 
         if hasattr(stats,'stream'):
             # In newer versions of python, the stats object has a 'stream'
             # attribute to write into.
             stats.stream = stdout_trap
             stats.print_stats(*lims)
         else:
             # For older versions, we manually redirect stdout during printing
             sys_stdout = sys.stdout
             try:
                 sys.stdout = stdout_trap
                 stats.print_stats(*lims)
             finally:
                 sys.stdout = sys_stdout
                 
         output = stdout_trap.getvalue()
         output = output.rstrip()
 
         page(output,screen_lines=self.shell.usable_screen_length)
         print sys_exit,
 
         dump_file = opts.D[0]
         text_file = opts.T[0]
         if dump_file:
             prof.dump_stats(dump_file)
             print '\n*** Profile stats marshalled to file',\
                   `dump_file`+'.',sys_exit
         if text_file:
             pfile = file(text_file,'w')
             pfile.write(output)
             pfile.close()
             print '\n*** Profile printout saved to text file',\
                   `text_file`+'.',sys_exit
 
         if opts.has_key('r'):
             return stats
         else:
             return None
 
     @testdec.skip_doctest
     def magic_run(self, parameter_s ='',runner=None,
                   file_finder=get_py_filename):
         """Run the named file inside IPython as a program.
 
         Usage:\\
           %run [-n -i -t [-N<N>] -d [-b<N>] -p [profile options]] file [args]
         
         Parameters after the filename are passed as command-line arguments to
         the program (put in sys.argv). Then, control returns to IPython's
         prompt.
 
         This is similar to running at a system prompt:\\
           $ python file args\\
         but with the advantage of giving you IPython's tracebacks, and of
         loading all variables into your interactive namespace for further use
         (unless -p is used, see below).
 
         The file is executed in a namespace initially consisting only of
         __name__=='__main__' and sys.argv constructed as indicated. It thus
         sees its environment as if it were being run as a stand-alone program
         (except for sharing global objects such as previously imported
         modules). But after execution, the IPython interactive namespace gets
         updated with all variables defined in the program (except for __name__
         and sys.argv). This allows for very convenient loading of code for
         interactive work, while giving each program a 'clean sheet' to run in.
 
         Options:
         
         -n: __name__ is NOT set to '__main__', but to the running file's name
         without extension (as python does under import).  This allows running
         scripts and reloading the definitions in them without calling code
         protected by an ' if __name__ == "__main__" ' clause.
 
         -i: run the file in IPython's namespace instead of an empty one. This
         is useful if you are experimenting with code written in a text editor
         which depends on variables defined interactively.
 
         -e: ignore sys.exit() calls or SystemExit exceptions in the script
         being run.  This is particularly useful if IPython is being used to
         run unittests, which always exit with a sys.exit() call.  In such
         cases you are interested in the output of the test results, not in
         seeing a traceback of the unittest module.
 
         -t: print timing information at the end of the run.  IPython will give
         you an estimated CPU time consumption for your script, which under
         Unix uses the resource module to avoid the wraparound problems of
         time.clock().  Under Unix, an estimate of time spent on system tasks
         is also given (for Windows platforms this is reported as 0.0).
 
         If -t is given, an additional -N<N> option can be given, where <N>
         must be an integer indicating how many times you want the script to
         run.  The final timing report will include total and per run results.
 
         For example (testing the script uniq_stable.py):
 
             In [1]: run -t uniq_stable
 
             IPython CPU timings (estimated):\\
               User  :    0.19597 s.\\
               System:        0.0 s.\\
 
             In [2]: run -t -N5 uniq_stable
 
             IPython CPU timings (estimated):\\
             Total runs performed: 5\\
               Times :      Total       Per run\\
               User  :   0.910862 s,  0.1821724 s.\\
               System:        0.0 s,        0.0 s.
 
         -d: run your program under the control of pdb, the Python debugger.
         This allows you to execute your program step by step, watch variables,
         etc.  Internally, what IPython does is similar to calling:
         
           pdb.run('execfile("YOURFILENAME")')
 
         with a breakpoint set on line 1 of your file.  You can change the line
         number for this automatic breakpoint to be <N> by using the -bN option
         (where N must be an integer).  For example:
 
           %run -d -b40 myscript
 
         will set the first breakpoint at line 40 in myscript.py.  Note that
         the first breakpoint must be set on a line which actually does
         something (not a comment or docstring) for it to stop execution.
 
         When the pdb debugger starts, you will see a (Pdb) prompt.  You must
         first enter 'c' (without qoutes) to start execution up to the first
         breakpoint.
 
         Entering 'help' gives information about the use of the debugger.  You
         can easily see pdb's full documentation with "import pdb;pdb.help()"
         at a prompt.
 
         -p: run program under the control of the Python profiler module (which
         prints a detailed report of execution times, function calls, etc).
 
         You can pass other options after -p which affect the behavior of the
         profiler itself. See the docs for %prun for details.
 
         In this mode, the program's variables do NOT propagate back to the
         IPython interactive namespace (because they remain in the namespace
         where the profiler executes them).
 
         Internally this triggers a call to %prun, see its documentation for
         details on the options available specifically for profiling.
 
         There is one special usage for which the text above doesn't apply:
         if the filename ends with .ipy, the file is run as ipython script,
         just as if the commands were written on IPython prompt.
         """
 
         # get arguments and set sys.argv for program to be run.
         opts,arg_lst = self.parse_options(parameter_s,'nidtN:b:pD:l:rs:T:e',
                                           mode='list',list_all=1)
 
         try:
             filename = file_finder(arg_lst[0])
         except IndexError:
             warn('you must provide at least a filename.')
             print '\n%run:\n',oinspect.getdoc(self.magic_run)
             return
         except IOError,msg:
             error(msg)
             return
 
         if filename.lower().endswith('.ipy'):
             self.shell.safe_execfile_ipy(filename)
             return
         
         # Control the response to exit() calls made by the script being run
         exit_ignore = opts.has_key('e')
         
         # Make sure that the running script gets a proper sys.argv as if it
         # were run from a system shell.
         save_argv = sys.argv # save it for later restoring
         sys.argv = [filename]+ arg_lst[1:]  # put in the proper filename
 
         if opts.has_key('i'):
             # Run in user's interactive namespace
             prog_ns = self.shell.user_ns
             __name__save = self.shell.user_ns['__name__']
             prog_ns['__name__'] = '__main__'
             main_mod = self.shell.new_main_mod(prog_ns)
         else:
             # Run in a fresh, empty namespace
             if opts.has_key('n'):
                 name = os.path.splitext(os.path.basename(filename))[0]
             else:
                 name = '__main__'
 
             main_mod = self.shell.new_main_mod()
             prog_ns = main_mod.__dict__
             prog_ns['__name__'] = name
 
         # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
         # set the __file__ global in the script's namespace
         prog_ns['__file__'] = filename
 
         # pickle fix.  See iplib for an explanation.  But we need to make sure
         # that, if we overwrite __main__, we replace it at the end
         main_mod_name = prog_ns['__name__']
 
         if main_mod_name == '__main__':
             restore_main = sys.modules['__main__']
         else:
             restore_main = False
 
         # This needs to be undone at the end to prevent holding references to
         # every single object ever created.
         sys.modules[main_mod_name] = main_mod
         
         stats = None
         try:
             self.shell.savehist()
 
             if opts.has_key('p'):
                 stats = self.magic_prun('',0,opts,arg_lst,prog_ns)
             else:
                 if opts.has_key('d'):
                     deb = debugger.Pdb(self.shell.colors)
                     # reset Breakpoint state, which is moronically kept
                     # in a class
                     bdb.Breakpoint.next = 1
                     bdb.Breakpoint.bplist = {}
                     bdb.Breakpoint.bpbynumber = [None]
                     # Set an initial breakpoint to stop execution
                     maxtries = 10
                     bp = int(opts.get('b',[1])[0])
                     checkline = deb.checkline(filename,bp)
                     if not checkline:
                         for bp in range(bp+1,bp+maxtries+1):
                             if deb.checkline(filename,bp):
                                 break
                         else:
                             msg = ("\nI failed to find a valid line to set "
                                    "a breakpoint\n"
                                    "after trying up to line: %s.\n"
                                    "Please set a valid breakpoint manually "
                                    "with the -b option." % bp)
                             error(msg)
                             return
                     # if we find a good linenumber, set the breakpoint
                     deb.do_break('%s:%s' % (filename,bp))
                     # Start file run
                     print "NOTE: Enter 'c' at the",
                     print "%s prompt to start your script." % deb.prompt
                     try:
                         deb.run('execfile("%s")' % filename,prog_ns)
                         
                     except:
                         etype, value, tb = sys.exc_info()
                         # Skip three frames in the traceback: the %run one,
                         # one inside bdb.py, and the command-line typed by the
                         # user (run by exec in pdb itself).
                         self.shell.InteractiveTB(etype,value,tb,tb_offset=3)
                 else:
                     if runner is None:
                         runner = self.shell.safe_execfile
                     if opts.has_key('t'):
                         # timed execution
                         try:
                             nruns = int(opts['N'][0])
                             if nruns < 1:
                                 error('Number of runs must be >=1')
                                 return
                         except (KeyError):
                             nruns = 1
                         if nruns == 1:
                             t0 = clock2()
                             runner(filename,prog_ns,prog_ns,
                                    exit_ignore=exit_ignore)
                             t1 = clock2()
                             t_usr = t1[0]-t0[0]
                             t_sys = t1[1]-t0[1]
                             print "\nIPython CPU timings (estimated):"
                             print "  User  : %10s s." % t_usr
                             print "  System: %10s s." % t_sys
                         else:
                             runs = range(nruns)
                             t0 = clock2()
                             for nr in runs:
                                 runner(filename,prog_ns,prog_ns,
                                        exit_ignore=exit_ignore)
                             t1 = clock2()
                             t_usr = t1[0]-t0[0]
                             t_sys = t1[1]-t0[1]
                             print "\nIPython CPU timings (estimated):"
                             print "Total runs performed:",nruns
                             print "  Times : %10s    %10s" % ('Total','Per run')
                             print "  User  : %10s s, %10s s." % (t_usr,t_usr/nruns)
                             print "  System: %10s s, %10s s." % (t_sys,t_sys/nruns)
                             
                     else:
                         # regular execution
                         runner(filename,prog_ns,prog_ns,exit_ignore=exit_ignore)
 
                 if opts.has_key('i'):
                     self.shell.user_ns['__name__'] = __name__save
                 else:
                     # The shell MUST hold a reference to prog_ns so after %run
                     # exits, the python deletion mechanism doesn't zero it out
                     # (leaving dangling references).
                     self.shell.cache_main_mod(prog_ns,filename)
                     # update IPython interactive namespace
 
                     # Some forms of read errors on the file may mean the
                     # __name__ key was never set; using pop we don't have to
                     # worry about a possible KeyError.
                     prog_ns.pop('__name__', None)
 
                     self.shell.user_ns.update(prog_ns)
         finally:
             # It's a bit of a mystery why, but __builtins__ can change from
             # being a module to becoming a dict missing some key data after
             # %run.  As best I can see, this is NOT something IPython is doing
             # at all, and similar problems have been reported before:
             # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
             # Since this seems to be done by the interpreter itself, the best
             # we can do is to at least restore __builtins__ for the user on
             # exit.
             self.shell.user_ns['__builtins__'] = __builtin__
             
             # Ensure key global structures are restored
             sys.argv = save_argv
             if restore_main:
                 sys.modules['__main__'] = restore_main
             else:
                 # Remove from sys.modules the reference to main_mod we'd
                 # added.  Otherwise it will trap references to objects
                 # contained therein.
                 del sys.modules[main_mod_name]
 
             self.shell.reloadhist()
                 
         return stats
 
     @testdec.skip_doctest
     def magic_timeit(self, parameter_s =''):
         """Time execution of a Python statement or expression
 
         Usage:\\
           %timeit [-n<N> -r<R> [-t|-c]] statement
 
         Time execution of a Python statement or expression using the timeit
         module.
 
         Options:
         -n<N>: execute the given statement <N> times in a loop. If this value
         is not given, a fitting value is chosen. 
         
         -r<R>: repeat the loop iteration <R> times and take the best result.
         Default: 3
         
         -t: use time.time to measure the time, which is the default on Unix.
         This function measures wall time.
         
         -c: use time.clock to measure the time, which is the default on
         Windows and measures wall time. On Unix, resource.getrusage is used
         instead and returns the CPU user time.
 
         -p<P>: use a precision of <P> digits to display the timing result.
         Default: 3
 
         
         Examples:
 
           In [1]: %timeit pass
           10000000 loops, best of 3: 53.3 ns per loop
 
           In [2]: u = None
 
           In [3]: %timeit u is None
           10000000 loops, best of 3: 184 ns per loop
 
           In [4]: %timeit -r 4 u == None
           1000000 loops, best of 4: 242 ns per loop
 
           In [5]: import time
 
           In [6]: %timeit -n1 time.sleep(2)
           1 loops, best of 3: 2 s per loop
           
 
         The times reported by %timeit will be slightly higher than those
         reported by the timeit.py script when variables are accessed. This is
         due to the fact that %timeit executes the statement in the namespace
         of the shell, compared with timeit.py, which uses a single setup
         statement to import function or create variables. Generally, the bias
         does not matter as long as results from timeit.py are not mixed with
         those from %timeit."""
 
         import timeit
         import math
 
         # XXX: Unfortunately the unicode 'micro' symbol can cause problems in
         # certain terminals.  Until we figure out a robust way of
         # auto-detecting if the terminal can deal with it, use plain 'us' for
         # microseconds.  I am really NOT happy about disabling the proper
         # 'micro' prefix, but crashing is worse... If anyone knows what the
         # right solution for this is, I'm all ears...
         #
         # Note: using
         #
         # s = u'\xb5'
         # s.encode(sys.getdefaultencoding())
         #
         # is not sufficient, as I've seen terminals where that fails but
         # print s
         #
         # succeeds
         #
         # See bug: https://bugs.launchpad.net/ipython/+bug/348466
         
         #units = [u"s", u"ms",u'\xb5',"ns"]
         units = [u"s", u"ms",u'us',"ns"]
         
         scaling = [1, 1e3, 1e6, 1e9]
 
         opts, stmt = self.parse_options(parameter_s,'n:r:tcp:',
                                         posix=False)
         if stmt == "":
             return
         timefunc = timeit.default_timer
         number = int(getattr(opts, "n", 0))
         repeat = int(getattr(opts, "r", timeit.default_repeat))
         precision = int(getattr(opts, "p", 3))
         if hasattr(opts, "t"):
             timefunc = time.time
         if hasattr(opts, "c"):
             timefunc = clock
 
         timer = timeit.Timer(timer=timefunc)
         # this code has tight coupling to the inner workings of timeit.Timer,
         # but is there a better way to achieve that the code stmt has access
         # to the shell namespace?
 
         src = timeit.template % {'stmt': timeit.reindent(stmt, 8),
                                  'setup': "pass"}
         # Track compilation time so it can be reported if too long
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
         
         t0 = clock()
         code = compile(src, "<magic-timeit>", "exec")
         tc = clock()-t0
         
         ns = {}
         exec code in self.shell.user_ns, ns
         timer.inner = ns["inner"]
         
         if number == 0:
             # determine number so that 0.2 <= total time < 2.0
             number = 1
             for i in range(1, 10):
                 if timer.timeit(number) >= 0.2:
                     break
                 number *= 10
         
         best = min(timer.repeat(repeat, number)) / number
 
         if best > 0.0 and best < 1000.0:
             order = min(-int(math.floor(math.log10(best)) // 3), 3)
         elif best >= 1000.0:
             order = 0
         else:
             order = 3
         print u"%d loops, best of %d: %.*g %s per loop" % (number, repeat,
                                                           precision,
                                                           best * scaling[order],
                                                           units[order])
         if tc > tc_min:
             print "Compiler time: %.2f s" % tc
 
     @testdec.skip_doctest
     def magic_time(self,parameter_s = ''):
         """Time execution of a Python statement or expression.
 
         The CPU and wall clock times are printed, and the value of the
         expression (if any) is returned.  Note that under Win32, system time
         is always reported as 0, since it can not be measured.
 
         This function provides very basic timing functionality.  In Python
         2.3, the timeit module offers more control and sophistication, so this
         could be rewritten to use it (patches welcome).
         
         Some examples:
 
           In [1]: time 2**128
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
           Out[1]: 340282366920938463463374607431768211456L
 
           In [2]: n = 1000000
 
           In [3]: time sum(range(n))
           CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
           Wall time: 1.37
           Out[3]: 499999500000L
 
           In [4]: time print 'hello world'
           hello world
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
 
           Note that the time needed by Python to compile the given expression
           will be reported if it is more than 0.1s.  In this example, the
           actual exponentiation is done by Python at compilation time, so while
           the expression can take a noticeable amount of time to compute, that
           time is purely due to the compilation:
 
           In [5]: time 3**9999;
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00 s
 
           In [6]: time 3**999999;
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00 s
           Compiler : 0.78 s
           """
         
         # fail immediately if the given expression can't be compiled
         
         expr = self.shell.prefilter(parameter_s,False)
 
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
         
         try:
             mode = 'eval'
             t0 = clock()
             code = compile(expr,'<timed eval>',mode)
             tc = clock()-t0
         except SyntaxError:
             mode = 'exec'
             t0 = clock()
             code = compile(expr,'<timed exec>',mode)
             tc = clock()-t0
         # skew measurement as little as possible
         glob = self.shell.user_ns
         clk = clock2
         wtime = time.time
         # time execution
         wall_st = wtime()
         if mode=='eval':
             st = clk()
             out = eval(code,glob)
             end = clk()
         else:
             st = clk()
             exec code in glob
             end = clk()
             out = None
         wall_end = wtime()
         # Compute actual times and report
         wall_time = wall_end-wall_st
         cpu_user = end[0]-st[0]
         cpu_sys = end[1]-st[1]
         cpu_tot = cpu_user+cpu_sys
         print "CPU times: user %.2f s, sys: %.2f s, total: %.2f s" % \
               (cpu_user,cpu_sys,cpu_tot)
         print "Wall time: %.2f s" % wall_time
         if tc > tc_min:
             print "Compiler : %.2f s" % tc
         return out
 
     @testdec.skip_doctest
     def magic_macro(self,parameter_s = ''):
         """Define a set of input lines as a macro for future re-execution.
 
         Usage:\\
           %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
         
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed as the
           command line is used instead.
 
         This will define a global variable called `name` which is a string
         made of joining the slices and lines you specify (n1,n2,... numbers
         above) from your input history into a single string. This variable
         acts like an automatic function which re-executes those lines as if
         you had typed them. You just type 'name' at the prompt and the code
         executes.
 
         The notation for indicating number ranges is: n1-n2 means 'use line
         numbers n1,...n2' (the endpoint is included).  That is, '5-7' means
         using the lines numbered 5,6 and 7.
 
         Note: as a 'hidden' feature, you can also use traditional python slice
         notation, where N:M means numbers N through M-1.
 
         For example, if your history contains (%hist prints it):
         
           44: x=1
           45: y=3
           46: z=x+y
           47: print x
           48: a=5
           49: print 'x',x,'y',y
 
         you can create a macro with lines 44 through 47 (included) and line 49
         called my_macro with:
 
           In [55]: %macro my_macro 44-47 49
 
         Now, typing `my_macro` (without quotes) will re-execute all this code
         in one pass.
 
         You don't need to give the line-numbers in order, and any given line
         number can appear multiple times. You can assemble macros with any
         lines from your input history in any order.
 
         The macro is a simple object which holds its value in an attribute,
         but IPython's display system checks for macros and executes them as
         code instead of printing them when you type their name.
 
         You can view a macro's contents by explicitly printing it with:
         
           'print macro_name'.
 
         For one-off cases which DON'T contain magic function calls in them you
         can obtain similar results by explicitly executing slices from your
         input history with:
 
           In [60]: exec In[44:48]+In[49]"""
 
         opts,args = self.parse_options(parameter_s,'r',mode='list')
         if not args:
             macs = [k for k,v in self.shell.user_ns.items() if isinstance(v, Macro)]
             macs.sort()
             return macs
         if len(args) == 1:
             raise UsageError(
                 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
         name,ranges = args[0], args[1:]
         
         #print 'rng',ranges  # dbg
         lines = self.extract_input_slices(ranges,opts.has_key('r'))
         macro = Macro(lines)
         self.shell.define_macro(name, macro)
         print 'Macro `%s` created. To execute, type its name (without quotes).' % name
         print 'Macro contents:'
         print macro,
 
     def magic_save(self,parameter_s = ''):
         """Save a set of lines to a given filename.
 
         Usage:\\
           %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
         
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed as the
           command line is used instead.
 
         This function uses the same syntax as %macro for line extraction, but
         instead of creating a macro it saves the resulting string to the
         filename you specify.
 
         It adds a '.py' extension to the file if you don't do so yourself, and
         it asks for confirmation before overwriting existing files."""
 
         opts,args = self.parse_options(parameter_s,'r',mode='list')
         fname,ranges = args[0], args[1:]
         if not fname.endswith('.py'):
             fname += '.py'
         if os.path.isfile(fname):
             ans = raw_input('File `%s` exists. Overwrite (y/[N])? ' % fname)
             if ans.lower() not in ['y','yes']:
                 print 'Operation cancelled.'
                 return
         cmds = ''.join(self.extract_input_slices(ranges,opts.has_key('r')))
         f = file(fname,'w')
         f.write(cmds)
         f.close()
         print 'The following commands were written to file `%s`:' % fname
         print cmds
 
     def _edit_macro(self,mname,macro):
         """open an editor with the macro data in a file"""
         filename = self.shell.mktempfile(macro.value)
         self.shell.hooks.editor(filename)
 
         # and make a new macro object, to replace the old one
         mfile = open(filename)
         mvalue = mfile.read()
         mfile.close()
         self.shell.user_ns[mname] = Macro(mvalue)
 
     def magic_ed(self,parameter_s=''):
         """Alias to %edit."""
         return self.magic_edit(parameter_s)
 
     @testdec.skip_doctest
     def magic_edit(self,parameter_s='',last_call=['','']):
         """Bring up an editor and execute the resulting code.
 
         Usage:
           %edit [options] [args]
 
         %edit runs IPython's editor hook.  The default version of this hook is
         set to call the __IPYTHON__.rc.editor command.  This is read from your
         environment variable $EDITOR.  If this isn't found, it will default to
         vi under Linux/Unix and to notepad under Windows.  See the end of this
         docstring for how to change the editor hook.
 
         You can also set the value of this editor via the command line option
         '-editor' or in your ipythonrc file. This is useful if you wish to use
         specifically for IPython an editor different from your typical default
         (and for Windows users who typically don't set environment variables).
 
         This command allows you to conveniently edit multi-line code right in
         your IPython session.
         
         If called without arguments, %edit opens up an empty editor with a
         temporary file and will execute the contents of this file when you
         close it (don't forget to save it!).
 
 
         Options:
 
         -n <number>: open the editor at a specified line number.  By default,
         the IPython editor hook uses the unix syntax 'editor +N filename', but
         you can configure this by providing your own modified hook if your
         favorite editor supports line-number specifications with a different
         syntax.
         
         -p: this will call the editor with the same data as the previous time
         it was used, regardless of how long ago (in your current session) it
         was.
 
         -r: use 'raw' input.  This option only applies to input taken from the
         user's history.  By default, the 'processed' history is used, so that
         magics are loaded in their transformed version to valid Python.  If
         this option is given, the raw input as typed as the command line is
         used instead.  When you exit the editor, it will be executed by
         IPython's own processor.
         
         -x: do not execute the edited code immediately upon exit. This is
         mainly useful if you are editing programs which need to be called with
         command line arguments, which you can then do using %run.
 
 
         Arguments:
 
         If arguments are given, the following possibilites exist:
 
         - The arguments are numbers or pairs of colon-separated numbers (like
         1 4:8 9). These are interpreted as lines of previous input to be
         loaded into the editor. The syntax is the same of the %macro command.
 
         - If the argument doesn't start with a number, it is evaluated as a
         variable and its contents loaded into the editor. You can thus edit
         any string which contains python code (including the result of
         previous edits).
 
         - If the argument is the name of an object (other than a string),
         IPython will try to locate the file where it was defined and open the
         editor at the point where it is defined. You can use `%edit function`
         to load an editor exactly at the point where 'function' is defined,
         edit it and have the file be executed automatically.
 
         If the object is a macro (see %macro for details), this opens up your
         specified editor with a temporary file containing the macro's data.
         Upon exit, the macro is reloaded with the contents of the file.
 
         Note: opening at an exact line is only supported under Unix, and some
         editors (like kedit and gedit up to Gnome 2.8) do not understand the
         '+NUMBER' parameter necessary for this feature. Good editors like
         (X)Emacs, vi, jed, pico and joe all do.
 
         - If the argument is not found as a variable, IPython will look for a
         file with that name (adding .py if necessary) and load it into the
         editor. It will execute its contents with execfile() when you exit,
         loading any code in the file into your interactive namespace.
 
         After executing your code, %edit will return as output the code you
         typed in the editor (except when it was an existing file). This way
         you can reload the code in further invocations of %edit as a variable,
         via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
         the output.
 
         Note that %edit is also available through the alias %ed.
 
         This is an example of creating a simple function inside the editor and
         then modifying it. First, start up the editor:
 
         In [1]: ed
         Editing... done. Executing edited code...
         Out[1]: 'def foo():n    print "foo() was defined in an editing session"n'
 
         We can then call the function foo():
         
         In [2]: foo()
         foo() was defined in an editing session
 
         Now we edit foo.  IPython automatically loads the editor with the
         (temporary) file where foo() was previously defined:
         
         In [3]: ed foo
         Editing... done. Executing edited code...
 
         And if we call foo() again we get the modified version:
         
         In [4]: foo()
         foo() has now been changed!
 
         Here is an example of how to edit a code snippet successive
         times. First we call the editor:
 
         In [5]: ed
         Editing... done. Executing edited code...
         hello
         Out[5]: "print 'hello'n"
 
         Now we call it again with the previous output (stored in _):
 
         In [6]: ed _
         Editing... done. Executing edited code...
         hello world
         Out[6]: "print 'hello world'n"
 
         Now we call it with the output #8 (stored in _8, also as Out[8]):
 
         In [7]: ed _8
         Editing... done. Executing edited code...
         hello again
         Out[7]: "print 'hello again'n"
 
 
         Changing the default editor hook:
 
         If you wish to write your own editor hook, you can put it in a
         configuration file which you load at startup time.  The default hook
         is defined in the IPython.core.hooks module, and you can use that as a
         starting example for further modifications.  That file also has
         general instructions on how to set a new hook for use once you've
         defined it."""
         
         # FIXME: This function has become a convoluted mess.  It needs a
         # ground-up rewrite with clean, simple logic.
 
         def make_filename(arg):
             "Make a filename from the given args"
             try:
                 filename = get_py_filename(arg)
             except IOError:
                 if args.endswith('.py'):
                     filename = arg
                 else:
                     filename = None
             return filename
 
         # custom exceptions
         class DataIsObject(Exception): pass
 
         opts,args = self.parse_options(parameter_s,'prxn:')
         # Set a few locals from the options for convenience:
         opts_p = opts.has_key('p')
         opts_r = opts.has_key('r')
         
         # Default line number value
         lineno = opts.get('n',None)
 
         if opts_p:
             args = '_%s' % last_call[0]
             if not self.shell.user_ns.has_key(args):
                 args = last_call[1]
             
         # use last_call to remember the state of the previous call, but don't
         # let it be clobbered by successive '-p' calls.
         try:
             last_call[0] = self.shell.outputcache.prompt_count
             if not opts_p:
                 last_call[1] = parameter_s
         except:
             pass
 
         # by default this is done with temp files, except when the given
         # arg is a filename
         use_temp = 1
 
         if re.match(r'\d',args):
             # Mode where user specifies ranges of lines, like in %macro.
             # This means that you can't edit files whose names begin with
             # numbers this way. Tough.
             ranges = args.split()
             data = ''.join(self.extract_input_slices(ranges,opts_r))
         elif args.endswith('.py'):
             filename = make_filename(args)
             data = ''
             use_temp = 0
         elif args:
             try:
                 # Load the parameter given as a variable. If not a string,
                 # process it as an object instead (below)
 
                 #print '*** args',args,'type',type(args)  # dbg
                 data = eval(args,self.shell.user_ns)
                 if not type(data) in StringTypes:
                     raise DataIsObject
 
             except (NameError,SyntaxError):
                 # given argument is not a variable, try as a filename
                 filename = make_filename(args)
                 if filename is None:
                     warn("Argument given (%s) can't be found as a variable "
                          "or as a filename." % args)
                     return
 
                 data = ''
                 use_temp = 0
             except DataIsObject:
 
                 # macros have a special edit function
                 if isinstance(data,Macro):
                     self._edit_macro(args,data)
                     return
                                 
                 # For objects, try to edit the file where they are defined
                 try:
                     filename = inspect.getabsfile(data)
                     if 'fakemodule' in filename.lower() and inspect.isclass(data):                     
                         # class created by %edit? Try to find source
                         # by looking for method definitions instead, the
                         # __module__ in those classes is FakeModule.
                         attrs = [getattr(data, aname) for aname in dir(data)]
                         for attr in attrs:
                             if not inspect.ismethod(attr):
                                 continue
                             filename = inspect.getabsfile(attr)
                             if filename and 'fakemodule' not in filename.lower():
                                 # change the attribute to be the edit target instead
                                 data = attr 
                                 break
                     
                     datafile = 1
                 except TypeError:
                     filename = make_filename(args)
                     datafile = 1
                     warn('Could not find file where `%s` is defined.\n'
                          'Opening a file named `%s`' % (args,filename))
                 # Now, make sure we can actually read the source (if it was in
                 # a temp file it's gone by now).
                 if datafile:
                     try:
                         if lineno is None:
                             lineno = inspect.getsourcelines(data)[1]
                     except IOError:
                         filename = make_filename(args)
                         if filename is None:
                             warn('The file `%s` where `%s` was defined cannot '
                                  'be read.' % (filename,data))
                             return
                 use_temp = 0
         else:
             data = ''
 
         if use_temp:
             filename = self.shell.mktempfile(data)
             print 'IPython will make a temporary file named:',filename
 
         # do actual editing here
         print 'Editing...',
         sys.stdout.flush()
         try:
             # Quote filenames that may have spaces in them
             if ' ' in filename:
                 filename = "%s" % filename
             self.shell.hooks.editor(filename,lineno)
         except TryNext:
             warn('Could not open editor')
             return
         
         # XXX TODO: should this be generalized for all string vars?
         # For now, this is special-cased to blocks created by cpaste
         if args.strip() == 'pasted_block':
             self.shell.user_ns['pasted_block'] = file_read(filename)
         
         if opts.has_key('x'):  # -x prevents actual execution
             print
         else:
             print 'done. Executing edited code...'
             if opts_r:
                 self.shell.runlines(file_read(filename))
             else:
                 self.shell.safe_execfile(filename,self.shell.user_ns,
                                          self.shell.user_ns)
         
                                                      
         if use_temp:
             try:
                 return open(filename).read()
             except IOError,msg:
                 if msg.filename == filename:
                     warn('File not found. Did you forget to save?')
                     return
                 else:
                     self.shell.showtraceback()
 
     def magic_xmode(self,parameter_s = ''):
         """Switch modes for the exception handlers.
 
         Valid modes: Plain, Context and Verbose.
 
         If called without arguments, acts as a toggle."""
 
         def xmode_switch_err(name):
             warn('Error changing %s exception modes.\n%s' %
                  (name,sys.exc_info()[1]))
 
         shell = self.shell
         new_mode = parameter_s.strip().capitalize()
         try:
             shell.InteractiveTB.set_mode(mode=new_mode)
             print 'Exception reporting mode:',shell.InteractiveTB.mode
         except:
             xmode_switch_err('user')
 
         # threaded shells use a special handler in sys.excepthook
         if shell.isthreaded:
             try:
                 shell.sys_excepthook.set_mode(mode=new_mode)
             except:
                 xmode_switch_err('threaded')
             
     def magic_colors(self,parameter_s = ''):
         """Switch color scheme for prompts, info system and exception handlers.
 
         Currently implemented schemes: NoColor, Linux, LightBG.
 
         Color scheme names are not case-sensitive."""
 
         def color_switch_err(name):
             warn('Error changing %s color schemes.\n%s' %
                  (name,sys.exc_info()[1]))
             
         
         new_scheme = parameter_s.strip()
         if not new_scheme:
             raise UsageError(
                 "%colors: you must specify a color scheme. See '%colors?'")
             return
         # local shortcut
         shell = self.shell
 
         import IPython.utils.rlineimpl as readline
 
         if not readline.have_readline and sys.platform == "win32":
             msg = """\
 Proper color support under MS Windows requires the pyreadline library.
 You can find it at:
 http://ipython.scipy.org/moin/PyReadline/Intro
 Gary's readline needs the ctypes module, from:
 http://starship.python.net/crew/theller/ctypes
 (Note that ctypes is already part of Python versions 2.5 and newer).
 
 Defaulting color scheme to 'NoColor'"""
             new_scheme = 'NoColor'
             warn(msg)
         
         # readline option is 0
         if not shell.has_readline:
             new_scheme = 'NoColor'
             
         # Set prompt colors
         try:
             shell.outputcache.set_colors(new_scheme)
         except:
             color_switch_err('prompt')
         else:
             shell.colors = \
                        shell.outputcache.color_table.active_scheme_name
         # Set exception colors
         try:
             shell.InteractiveTB.set_colors(scheme = new_scheme)
             shell.SyntaxTB.set_colors(scheme = new_scheme)
         except:
             color_switch_err('exception')
 
         # threaded shells use a verbose traceback in sys.excepthook
         if shell.isthreaded:
             try:
                 shell.sys_excepthook.set_colors(scheme=new_scheme)
             except:
                 color_switch_err('system exception handler')
         
         # Set info (for 'object?') colors
         if shell.color_info:
             try:
                 shell.inspector.set_active_scheme(new_scheme)
             except:
                 color_switch_err('object inspector')
         else:
             shell.inspector.set_active_scheme('NoColor')
                 
     def magic_color_info(self,parameter_s = ''):
         """Toggle color_info.
 
         The color_info configuration parameter controls whether colors are
         used for displaying object details (by things like %psource, %pfile or
         the '?' system). This function toggles this value with each call.
 
         Note that unless you have a fairly recent pager (less works better
         than more) in your system, using colored object information displays
         will not work properly. Test it and see."""
         
         self.shell.color_info = not self.shell.color_info
         self.magic_colors(self.shell.colors)
         print 'Object introspection functions have now coloring:',
         print ['OFF','ON'][int(self.shell.color_info)]
 
     def magic_Pprint(self, parameter_s=''):
         """Toggle pretty printing on/off."""
         
         self.shell.pprint = 1 - self.shell.pprint
         print 'Pretty printing has been turned', \
               ['OFF','ON'][self.shell.pprint]
                 
     def magic_Exit(self, parameter_s=''):
         """Exit IPython without confirmation."""
 
         self.shell.ask_exit()
 
     # Add aliases as magics so all common forms work: exit, quit, Exit, Quit.
     magic_exit = magic_quit = magic_Quit = magic_Exit
 
     #......................................................................
     # Functions to implement unix shell-type things
 
     @testdec.skip_doctest
     def magic_alias(self, parameter_s = ''):
         """Define an alias for a system command.
 
         '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
 
         Then, typing 'alias_name params' will execute the system command 'cmd
         params' (from your underlying operating system).
 
         Aliases have lower precedence than magic functions and Python normal
         variables, so if 'foo' is both a Python variable and an alias, the
         alias can not be executed until 'del foo' removes the Python variable.
 
         You can use the %l specifier in an alias definition to represent the
         whole line when the alias is called.  For example:
 
           In [2]: alias all echo "Input in brackets: <%l>"
           In [3]: all hello world
           Input in brackets: <hello world>
 
         You can also define aliases with parameters using %s specifiers (one
         per parameter):
         
           In [1]: alias parts echo first %s second %s
           In [2]: %parts A B
           first A second B
           In [3]: %parts A
           Incorrect number of arguments: 2 expected.
           parts is an alias to: 'echo first %s second %s'
 
         Note that %l and %s are mutually exclusive.  You can only use one or
         the other in your aliases.
 
         Aliases expand Python variables just like system calls using ! or !! 
         do: all expressions prefixed with '$' get expanded.  For details of
         the semantic rules, see PEP-215:
         http://www.python.org/peps/pep-0215.html.  This is the library used by
         IPython for variable expansion.  If you want to access a true shell
         variable, an extra $ is necessary to prevent its expansion by IPython:
 
         In [6]: alias show echo
         In [7]: PATH='A Python string'
         In [8]: show $PATH
         A Python string
         In [9]: show $$PATH
         /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
 
         You can use the alias facility to acess all of $PATH.  See the %rehash
         and %rehashx functions, which automatically create aliases for the
         contents of your $PATH.
 
         If called with no parameters, %alias prints the current alias table."""
 
         par = parameter_s.strip()
         if not par:
             stored = self.db.get('stored_aliases', {} )
             aliases = sorted(self.shell.alias_manager.aliases)
             # for k, v in stored:
             #     atab.append(k, v[0])
 
             print "Total number of aliases:", len(aliases)
             return aliases
         
         # Now try to define a new one
         try:
             alias,cmd = par.split(None, 1)
         except:
             print oinspect.getdoc(self.magic_alias)
         else:
             self.shell.alias_manager.soft_define_alias(alias, cmd)
     # end magic_alias
 
     def magic_unalias(self, parameter_s = ''):
         """Remove an alias"""
 
         aname = parameter_s.strip()
         self.shell.alias_manager.undefine_alias(aname)
         stored = self.db.get('stored_aliases', {} )
         if aname in stored:
             print "Removing %stored alias",aname
             del stored[aname]
             self.db['stored_aliases'] = stored
             
 
     def magic_rehashx(self, parameter_s = ''):
         """Update the alias table with all executable files in $PATH.
 
         This version explicitly checks that every entry in $PATH is a file
         with execute access (os.X_OK), so it is much slower than %rehash.
 
         Under Windows, it checks executability as a match agains a
         '|'-separated string of extensions, stored in the IPython config
         variable win_exec_ext.  This defaults to 'exe|com|bat'.
         
         This function also resets the root module cache of module completer,
         used on slow filesystems.
         """
         from IPython.core.alias import InvalidAliasError
 
         # for the benefit of module completer in ipy_completers.py
         del self.db['rootmodules']
         
         path = [os.path.abspath(os.path.expanduser(p)) for p in 
             os.environ.get('PATH','').split(os.pathsep)]
         path = filter(os.path.isdir,path)
 
         syscmdlist = []
         # Now define isexec in a cross platform manner.
         if os.name == 'posix':
             isexec = lambda fname:os.path.isfile(fname) and \
                      os.access(fname,os.X_OK)
         else:
             try:
                 winext = os.environ['pathext'].replace(';','|').replace('.','')
             except KeyError:
                 winext = 'exe|com|bat|py'
             if 'py' not in winext:
                 winext += '|py'
             execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
             isexec = lambda fname:os.path.isfile(fname) and execre.match(fname)
         savedir = os.getcwd()
 
         # Now walk the paths looking for executables to alias.
         try:
             # write the whole loop for posix/Windows so we don't have an if in
             # the innermost part
             if os.name == 'posix':
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):
                         if isexec(ff):
                             try:
                                 # Removes dots from the name since ipython
                                 # will assume names with dots to be python.
                                 self.shell.alias_manager.define_alias(
                                     ff.replace('.',''), ff)
                             except InvalidAliasError:
                                 pass
                             else:
                                 syscmdlist.append(ff)
             else:
                 no_alias = self.shell.alias_manager.no_alias
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):
                         base, ext = os.path.splitext(ff)
                         if isexec(ff) and base.lower() not in no_alias:
                             if ext.lower() == '.exe':
                                 ff = base
                                 try:
                                     # Removes dots from the name since ipython
                                     # will assume names with dots to be python.
                                     self.shell.alias_manager.define_alias(
                                         base.lower().replace('.',''), ff)
                                 except InvalidAliasError:
                                     pass
                                 syscmdlist.append(ff)
             db = self.db
             db['syscmdlist'] = syscmdlist
         finally:
             os.chdir(savedir)
         
     def magic_pwd(self, parameter_s = ''):
         """Return the current working directory path."""
         return os.getcwd()
 
     def magic_cd(self, parameter_s=''):
         """Change the current working directory.
 
         This command automatically maintains an internal list of directories
         you visit during your IPython session, in the variable _dh. The
         command %dhist shows this history nicely formatted. You can also
         do 'cd -<tab>' to see directory history conveniently.
 
         Usage:
 
           cd 'dir': changes to directory 'dir'.
 
           cd -: changes to the last visited directory.
 
           cd -<n>: changes to the n-th directory in the directory history.
 
           cd --foo: change to directory that matches 'foo' in history
             
           cd -b <bookmark_name>: jump to a bookmark set by %bookmark
              (note: cd <bookmark_name> is enough if there is no
               directory <bookmark_name>, but a bookmark with the name exists.)
               'cd -b <tab>' allows you to tab-complete bookmark names. 
 
         Options:
 
         -q: quiet.  Do not print the working directory after the cd command is
         executed.  By default IPython's cd command does print this directory,
         since the default prompts do not display path information.
         
         Note that !cd doesn't work for this purpose because the shell where
         !command runs is immediately discarded after executing 'command'."""
 
         parameter_s = parameter_s.strip()
         #bkms = self.shell.persist.get("bookmarks",{})
 
         oldcwd = os.getcwd()
         numcd = re.match(r'(-)(\d+)$',parameter_s)
         # jump in directory history by number
         if numcd:
             nn = int(numcd.group(2))
             try:
                 ps = self.shell.user_ns['_dh'][nn]
             except IndexError:
                 print 'The requested directory does not exist in history.'
                 return
             else:
                 opts = {}
         elif parameter_s.startswith('--'):
             ps = None
             fallback = None
             pat = parameter_s[2:]
             dh = self.shell.user_ns['_dh']
             # first search only by basename (last component)
             for ent in reversed(dh):
                 if pat in os.path.basename(ent) and os.path.isdir(ent):
                     ps = ent
                     break
             
                 if fallback is None and pat in ent and os.path.isdir(ent):
                     fallback = ent
                 
             # if we have no last part match, pick the first full path match
             if ps is None:
                 ps = fallback
             
             if ps is None:
                 print "No matching entry in directory history"
                 return
             else:
                 opts = {}
                 
             
         else:
             #turn all non-space-escaping backslashes to slashes, 
             # for c:\windows\directory\names\
             parameter_s = re.sub(r'\\(?! )','/', parameter_s)            
             opts,ps = self.parse_options(parameter_s,'qb',mode='string')
         # jump to previous
         if ps == '-':
             try:
                 ps = self.shell.user_ns['_dh'][-2]
             except IndexError:
                 raise UsageError('%cd -: No previous directory to change to.')
         # jump to bookmark if needed
         else:
             if not os.path.isdir(ps) or opts.has_key('b'):
                 bkms = self.db.get('bookmarks', {})
             
                 if bkms.has_key(ps):
                     target = bkms[ps]
                     print '(bookmark:%s) -> %s' % (ps,target)
                     ps = target
                 else:
                     if opts.has_key('b'):
                         raise UsageError("Bookmark '%s' not found.  "
                               "Use '%%bookmark -l' to see your bookmarks." % ps)
             
         # at this point ps should point to the target dir
         if ps:
             try:                
                 os.chdir(os.path.expanduser(ps))
                 if self.shell.term_title:
                     set_term_title('IPython: ' + abbrev_cwd())
             except OSError:
                 print sys.exc_info()[1]
             else:
                 cwd = os.getcwd()
                 dhist = self.shell.user_ns['_dh']
                 if oldcwd != cwd:
                     dhist.append(cwd)
                     self.db['dhist'] = compress_dhist(dhist)[-100:]
                 
         else:
             os.chdir(self.shell.home_dir)
             if self.shell.term_title:
                 set_term_title('IPython: ' + '~')
             cwd = os.getcwd()
             dhist = self.shell.user_ns['_dh']
             
             if oldcwd != cwd:
                 dhist.append(cwd)
                 self.db['dhist'] = compress_dhist(dhist)[-100:]
         if not 'q' in opts and self.shell.user_ns['_dh']:
             print self.shell.user_ns['_dh'][-1]
 
 
     def magic_env(self, parameter_s=''):
         """List environment variables."""
         
         return os.environ.data
 
     def magic_pushd(self, parameter_s=''):
         """Place the current dir on stack and change directory.
         
         Usage:\\
           %pushd ['dirname']
         """
         
         dir_s = self.shell.dir_stack
         tgt = os.path.expanduser(parameter_s)
         cwd = os.getcwd().replace(self.home_dir,'~')
         if tgt:
             self.magic_cd(parameter_s)
         dir_s.insert(0,cwd)
         return self.magic_dirs()
 
     def magic_popd(self, parameter_s=''):
         """Change to directory popped off the top of the stack.
         """
         if not self.shell.dir_stack:
             raise UsageError("%popd on empty stack")
         top = self.shell.dir_stack.pop(0)
         self.magic_cd(top)
         print "popd ->",top
 
     def magic_dirs(self, parameter_s=''):
         """Return the current directory stack."""
 
         return self.shell.dir_stack
 
     def magic_dhist(self, parameter_s=''):
         """Print your history of visited directories.
 
         %dhist       -> print full history\\
         %dhist n     -> print last n entries only\\
         %dhist n1 n2 -> print entries between n1 and n2 (n1 not included)\\
 
         This history is automatically maintained by the %cd command, and
         always available as the global list variable _dh. You can use %cd -<n>
         to go to directory number <n>.
         
         Note that most of time, you should view directory history by entering
         cd -<TAB>.
         
         """
 
         dh = self.shell.user_ns['_dh']
         if parameter_s:
             try:
                 args = map(int,parameter_s.split())
             except:
                 self.arg_err(Magic.magic_dhist)
                 return
             if len(args) == 1:
                 ini,fin = max(len(dh)-(args[0]),0),len(dh)
             elif len(args) == 2:
                 ini,fin = args
             else:
                 self.arg_err(Magic.magic_dhist)
                 return
         else:
             ini,fin = 0,len(dh)
         nlprint(dh,
                 header = 'Directory history (kept in _dh)',
                 start=ini,stop=fin)
 
     @testdec.skip_doctest
     def magic_sc(self, parameter_s=''):
         """Shell capture - execute a shell command and capture its output.
 
         DEPRECATED. Suboptimal, retained for backwards compatibility.
         
         You should use the form 'var = !command' instead. Example:
          
          "%sc -l myfiles = ls ~" should now be written as
             
          "myfiles = !ls ~"
          
         myfiles.s, myfiles.l and myfiles.n still apply as documented
         below.
 
         --
         %sc [options] varname=command
 
         IPython will run the given command using commands.getoutput(), and
         will then update the user's interactive namespace with a variable
         called varname, containing the value of the call.  Your command can
         contain shell wildcards, pipes, etc.
 
         The '=' sign in the syntax is mandatory, and the variable name you
         supply must follow Python's standard conventions for valid names.
         
         (A special format without variable name exists for internal use)
 
         Options:
 
           -l: list output.  Split the output on newlines into a list before
           assigning it to the given variable.  By default the output is stored
           as a single string.
 
           -v: verbose.  Print the contents of the variable.
 
         In most cases you should not need to split as a list, because the
         returned value is a special type of string which can automatically
         provide its contents either as a list (split on newlines) or as a
         space-separated string.  These are convenient, respectively, either
         for sequential processing or to be passed to a shell command.
 
         For example:
 
         # all-random
         
             # Capture into variable a
             In [1]: sc a=ls *py
 
             # a is a string with embedded newlines
             In [2]: a
             Out[2]: 'setup.py\\nwin32_manual_post_install.py'
 
             # which can be seen as a list:
             In [3]: a.l
             Out[3]: ['setup.py', 'win32_manual_post_install.py']
 
             # or as a whitespace-separated string:
             In [4]: a.s
             Out[4]: 'setup.py win32_manual_post_install.py'
 
             # a.s is useful to pass as a single command line:
             In [5]: !wc -l $a.s
               146 setup.py
               130 win32_manual_post_install.py
               276 total
 
             # while the list form is useful to loop over:
             In [6]: for f in a.l:
               ...:      !wc -l $f
               ...:
             146 setup.py
             130 win32_manual_post_install.py
 
         Similiarly, the lists returned by the -l option are also special, in
         the sense that you can equally invoke the .s attribute on them to
         automatically get a whitespace-separated string from their contents:
 
             In [7]: sc -l b=ls *py
 
             In [8]: b
             Out[8]: ['setup.py', 'win32_manual_post_install.py']
 
             In [9]: b.s
             Out[9]: 'setup.py win32_manual_post_install.py'
 
         In summary, both the lists and strings used for ouptut capture have
         the following special attributes:
 
             .l (or .list) : value as list.
             .n (or .nlstr): value as newline-separated string.
             .s (or .spstr): value as space-separated string.
         """
 
         opts,args = self.parse_options(parameter_s,'lv')
         # Try to get a variable name and command to run
         try:
             # the variable name must be obtained from the parse_options
             # output, which uses shlex.split to strip options out.
             var,_ = args.split('=',1)
             var = var.strip()
             # But the the command has to be extracted from the original input
             # parameter_s, not on what parse_options returns, to avoid the
             # quote stripping which shlex.split performs on it.
             _,cmd = parameter_s.split('=',1)
         except ValueError:
             var,cmd = '',''
         # If all looks ok, proceed
         out,err = self.shell.getoutputerror(cmd)
         if err:
             print >> Term.cerr,err
         if opts.has_key('l'):
             out = SList(out.split('\n'))
         else:
             out = LSString(out)
         if opts.has_key('v'):
             print '%s ==\n%s' % (var,pformat(out))
         if var:
             self.shell.user_ns.update({var:out})
         else:
             return out
 
     def magic_sx(self, parameter_s=''):
         """Shell execute - run a shell command and capture its output.
 
         %sx command
 
         IPython will run the given command using commands.getoutput(), and
         return the result formatted as a list (split on '\\n').  Since the
         output is _returned_, it will be stored in ipython's regular output
         cache Out[N] and in the '_N' automatic variables.
 
         Notes:
 
         1) If an input line begins with '!!', then %sx is automatically
         invoked.  That is, while:
           !ls
         causes ipython to simply issue system('ls'), typing
           !!ls
         is a shorthand equivalent to:
           %sx ls
         
         2) %sx differs from %sc in that %sx automatically splits into a list,
         like '%sc -l'.  The reason for this is to make it as easy as possible
         to process line-oriented shell output via further python commands.
         %sc is meant to provide much finer control, but requires more
         typing.
 
         3) Just like %sc -l, this is a list with special attributes:
 
           .l (or .list) : value as list.
           .n (or .nlstr): value as newline-separated string.
           .s (or .spstr): value as whitespace-separated string.
 
         This is very useful when trying to use such lists as arguments to
         system commands."""
 
         if parameter_s:
             out,err = self.shell.getoutputerror(parameter_s)
             if err:
                 print >> Term.cerr,err
             return SList(out.split('\n'))
 
     def magic_bg(self, parameter_s=''):
         """Run a job in the background, in a separate thread.
 
         For example,
 
           %bg myfunc(x,y,z=1)
 
         will execute 'myfunc(x,y,z=1)' in a background thread.  As soon as the
         execution starts, a message will be printed indicating the job
         number.  If your job number is 5, you can use
 
           myvar = jobs.result(5)  or  myvar = jobs[5].result
 
         to assign this result to variable 'myvar'.
 
         IPython has a job manager, accessible via the 'jobs' object.  You can
         type jobs? to get more information about it, and use jobs.<TAB> to see
         its attributes.  All attributes not starting with an underscore are
         meant for public use.
 
         In particular, look at the jobs.new() method, which is used to create
         new jobs.  This magic %bg function is just a convenience wrapper
         around jobs.new(), for expression-based jobs.  If you want to create a
         new job with an explicit function object and arguments, you must call
         jobs.new() directly.
 
         The jobs.new docstring also describes in detail several important
         caveats associated with a thread-based model for background job
         execution.  Type jobs.new? for details.
 
         You can check the status of all jobs with jobs.status().
 
         The jobs variable is set by IPython into the Python builtin namespace.
         If you ever declare a variable named 'jobs', you will shadow this
         name.  You can either delete your global jobs variable to regain
         access to the job manager, or make a new name and assign it manually
         to the manager (stored in IPython's namespace).  For example, to
         assign the job manager to the Jobs name, use:
 
           Jobs = __builtins__.jobs"""
         
         self.shell.jobs.new(parameter_s,self.shell.user_ns)
         
     def magic_r(self, parameter_s=''):
         """Repeat previous input.
 
         Note: Consider using the more powerfull %rep instead!
         
         If given an argument, repeats the previous command which starts with
         the same string, otherwise it just repeats the previous input.
 
         Shell escaped commands (with ! as first character) are not recognized
         by this system, only pure python code and magic commands.
         """
 
         start = parameter_s.strip()
         esc_magic = ESC_MAGIC
         # Identify magic commands even if automagic is on (which means
         # the in-memory version is different from that typed by the user).
         if self.shell.automagic:
             start_magic = esc_magic+start
         else:
             start_magic = start
         # Look through the input history in reverse
         for n in range(len(self.shell.input_hist)-2,0,-1):
             input = self.shell.input_hist[n]
             # skip plain 'r' lines so we don't recurse to infinity
             if input != '_ip.magic("r")\n' and \
                    (input.startswith(start) or input.startswith(start_magic)):
                 #print 'match',`input`  # dbg
                 print 'Executing:',input,
                 self.shell.runlines(input)
                 return
         print 'No previous input matching `%s` found.' % start
 
     
     def magic_bookmark(self, parameter_s=''):
         """Manage IPython's bookmark system.
 
         %bookmark <name>       - set bookmark to current dir
         %bookmark <name> <dir> - set bookmark to <dir>
         %bookmark -l           - list all bookmarks
         %bookmark -d <name>    - remove bookmark
         %bookmark -r           - remove all bookmarks
 
         You can later on access a bookmarked folder with:
           %cd -b <name>
         or simply '%cd <name>' if there is no directory called <name> AND
         there is such a bookmark defined.
 
         Your bookmarks persist through IPython sessions, but they are
         associated with each profile."""
 
         opts,args = self.parse_options(parameter_s,'drl',mode='list')
         if len(args) > 2:
             raise UsageError("%bookmark: too many arguments")
 
         bkms = self.db.get('bookmarks',{})
             
         if opts.has_key('d'):
             try:
                 todel = args[0]
             except IndexError:
                 raise UsageError(
                     "%bookmark -d: must provide a bookmark to delete")
             else:
                 try:
                     del bkms[todel]
                 except KeyError:
                     raise UsageError(
                         "%%bookmark -d: Can't delete bookmark '%s'" % todel)
 
         elif opts.has_key('r'):
             bkms = {}
         elif opts.has_key('l'):
             bks = bkms.keys()
             bks.sort()
             if bks:
                 size = max(map(len,bks))
             else:
                 size = 0
             fmt = '%-'+str(size)+'s -> %s'
             print 'Current bookmarks:'
             for bk in bks:
                 print fmt % (bk,bkms[bk])
         else:
             if not args:
                 raise UsageError("%bookmark: You must specify the bookmark name")
             elif len(args)==1:
                 bkms[args[0]] = os.getcwd()
             elif len(args)==2:
                 bkms[args[0]] = args[1]
         self.db['bookmarks'] = bkms
 
     def magic_pycat(self, parameter_s=''):
         """Show a syntax-highlighted file through a pager.
 
         This magic is similar to the cat utility, but it will assume the file
         to be Python source and will show it with syntax highlighting. """
         
         try:
             filename = get_py_filename(parameter_s)
             cont = file_read(filename)
         except IOError:
             try:
                 cont = eval(parameter_s,self.user_ns)
             except NameError:
                 cont = None
         if cont is None:
             print "Error: no such file or variable"
             return
             
         page(self.shell.pycolorize(cont),
              screen_lines=self.shell.usable_screen_length)
 
     def _rerun_pasted(self):
         """ Rerun a previously pasted command.
         """
         b = self.user_ns.get('pasted_block', None)
         if b is None:
             raise UsageError('No previous pasted block available')
         print "Re-executing '%s...' (%d chars)"% (b.split('\n',1)[0], len(b))
         exec b in self.user_ns
 
     def _get_pasted_lines(self, sentinel):
         """ Yield pasted lines until the user enters the given sentinel value.
         """
         from IPython.core import iplib
         print "Pasting code; enter '%s' alone on the line to stop." % sentinel
         while True:
             l = iplib.raw_input_original(':')
             if l == sentinel:
                 return
             else:
                 yield l
 
     def _strip_pasted_lines_for_code(self, raw_lines):
         """ Strip non-code parts of a sequence of lines to return a block of
         code.
         """
         # Regular expressions that declare text we strip from the input:
         strip_re =  [r'^\s*In \[\d+\]:', # IPython input prompt
                      r'^\s*(\s?>)+', # Python input prompt
                      r'^\s*\.{3,}', # Continuation prompts
                      r'^\++',
                      ]
 
         strip_from_start = map(re.compile,strip_re)
         
         lines = []
         for l in raw_lines:
             for pat in strip_from_start: 
                 l = pat.sub('',l)
             lines.append(l)
                          
         block = "\n".join(lines) + '\n'
         #print "block:\n",block
         return block
 
     def _execute_block(self, block, par):
         """ Execute a block, or store it in a variable, per the user's request.
         """
         if not par:
             b = textwrap.dedent(block)
             self.user_ns['pasted_block'] = b
             exec b in self.user_ns
         else:
             self.user_ns[par] = SList(block.splitlines())
             print "Block assigned to '%s'" % par
 
     def magic_cpaste(self, parameter_s=''):
         """Allows you to paste & execute a pre-formatted code block from clipboard.
         
         You must terminate the block with '--' (two minus-signs) alone on the
         line. You can also provide your own sentinel with '%paste -s %%' ('%%' 
         is the new sentinel for this operation)
         
         The block is dedented prior to execution to enable execution of method
         definitions. '>' and '+' characters at the beginning of a line are
         ignored, to allow pasting directly from e-mails, diff files and
         doctests (the '...' continuation prompt is also stripped).  The
         executed block is also assigned to variable named 'pasted_block' for
         later editing with '%edit pasted_block'.
         
         You can also pass a variable name as an argument, e.g. '%cpaste foo'.
         This assigns the pasted block to variable 'foo' as string, without 
         dedenting or executing it (preceding >>> and + is still stripped)
         
         '%cpaste -r' re-executes the block previously entered by cpaste.
         
         Do not be alarmed by garbled output on Windows (it's a readline bug). 
         Just press enter and type -- (and press enter again) and the block 
         will be what was just pasted.
         
         IPython statements (magics, shell escapes) are not supported (yet).
 
         See also
         --------
         paste: automatically pull code from clipboard.
         """
         
         opts,args = self.parse_options(parameter_s,'rs:',mode='string')
         par = args.strip()
         if opts.has_key('r'):
             self._rerun_pasted()
             return
         
         sentinel = opts.get('s','--')
 
         block = self._strip_pasted_lines_for_code(
             self._get_pasted_lines(sentinel))
 
         self._execute_block(block, par)
 
     def magic_paste(self, parameter_s=''):
         """Allows you to paste & execute a pre-formatted code block from clipboard.
         
         The text is pulled directly from the clipboard without user
         intervention and printed back on the screen before execution (unless
         the -q flag is given to force quiet mode).
 
         The block is dedented prior to execution to enable execution of method
         definitions. '>' and '+' characters at the beginning of a line are
         ignored, to allow pasting directly from e-mails, diff files and
         doctests (the '...' continuation prompt is also stripped).  The
         executed block is also assigned to variable named 'pasted_block' for
         later editing with '%edit pasted_block'.
         
         You can also pass a variable name as an argument, e.g. '%paste foo'.
         This assigns the pasted block to variable 'foo' as string, without 
         dedenting or executing it (preceding >>> and + is still stripped)
 
         Options
         -------
         
           -r: re-executes the block previously entered by cpaste.
 
           -q: quiet mode: do not echo the pasted text back to the terminal.
         
         IPython statements (magics, shell escapes) are not supported (yet).
 
         See also
         --------
         cpaste: manually paste code into terminal until you mark its end.
         """
         opts,args = self.parse_options(parameter_s,'rq',mode='string')
         par = args.strip()
         if opts.has_key('r'):
             self._rerun_pasted()
             return
 
         text = self.shell.hooks.clipboard_get()
         block = self._strip_pasted_lines_for_code(text.splitlines())
 
         # By default, echo back to terminal unless quiet mode is requested
         if not opts.has_key('q'):
             write = self.shell.write
             write(self.shell.pycolorize(block))
             if not block.endswith('\n'):
                 write('\n')
             write("## -- End pasted text --\n")
             
         self._execute_block(block, par)
 
     def magic_quickref(self,arg):
         """ Show a quick reference sheet """
         import IPython.core.usage
         qr = IPython.core.usage.quick_reference + self.magic_magic('-brief')
         
         page(qr)
 
     def magic_doctest_mode(self,parameter_s=''):
         """Toggle doctest mode on and off.
 
         This mode allows you to toggle the prompt behavior between normal
         IPython prompts and ones that are as similar to the default IPython
         interpreter as possible.
 
         It also supports the pasting of code snippets that have leading '>>>'
         and '...' prompts in them.  This means that you can paste doctests from
         files or docstrings (even if they have leading whitespace), and the
         code will execute correctly.  You can then use '%history -tn' to see
         the translated history without line numbers; this will give you the
         input after removal of all the leading prompts and whitespace, which
         can be pasted back into an editor.
 
         With these features, you can switch into this mode easily whenever you
         need to do testing and changes to doctests, without having to leave
         your existing IPython session.
         """
 
         from IPython.utils.ipstruct import Struct
 
         # Shorthands
         shell = self.shell
         oc = shell.outputcache
         meta = shell.meta
         # dstore is a data store kept in the instance metadata bag to track any
         # changes we make, so we can undo them later.
         dstore = meta.setdefault('doctest_mode',Struct())
         save_dstore = dstore.setdefault
 
         # save a few values we'll need to recover later
         mode = save_dstore('mode',False)
         save_dstore('rc_pprint',shell.pprint)
         save_dstore('xmode',shell.InteractiveTB.mode)
         save_dstore('rc_separate_out',shell.separate_out)
         save_dstore('rc_separate_out2',shell.separate_out2)
         save_dstore('rc_prompts_pad_left',shell.prompts_pad_left)
         save_dstore('rc_separate_in',shell.separate_in)
 
         if mode == False:
             # turn on
             oc.prompt1.p_template = '>>> '
             oc.prompt2.p_template = '... '
             oc.prompt_out.p_template = ''
 
             # Prompt separators like plain python
             oc.input_sep = oc.prompt1.sep = ''
             oc.output_sep = ''
             oc.output_sep2 = ''
 
             oc.prompt1.pad_left = oc.prompt2.pad_left = \
                                   oc.prompt_out.pad_left = False
 
             shell.pprint = False
             
             shell.magic_xmode('Plain')
 
         else:
             # turn off
             oc.prompt1.p_template = shell.prompt_in1
             oc.prompt2.p_template = shell.prompt_in2
             oc.prompt_out.p_template = shell.prompt_out
 
             oc.input_sep = oc.prompt1.sep = dstore.rc_separate_in
 
             oc.output_sep = dstore.rc_separate_out
             oc.output_sep2 = dstore.rc_separate_out2
 
             oc.prompt1.pad_left = oc.prompt2.pad_left = \
                          oc.prompt_out.pad_left = dstore.rc_prompts_pad_left
 
             shell.pprint = dstore.rc_pprint
 
             shell.magic_xmode(dstore.xmode)
 
         # Store new mode and inform
         dstore.mode = bool(1-int(mode))
         print 'Doctest mode is:',
         print ['OFF','ON'][dstore.mode]
 
     def magic_gui(self, parameter_s=''):
         """Enable or disable IPython GUI event loop integration.
 
         %gui [-a] [GUINAME]
 
         This magic replaces IPython's threaded shells that were activated
         using the (pylab/wthread/etc.) command line flags.  GUI toolkits
         can now be enabled, disabled and swtiched at runtime and keyboard
         interrupts should work without any problems.  The following toolkits
         are supported:  wxPython, PyQt4, PyGTK, and Tk::
 
             %gui wx      # enable wxPython event loop integration
             %gui qt4|qt  # enable PyQt4 event loop integration
             %gui gtk     # enable PyGTK event loop integration
             %gui tk      # enable Tk event loop integration
             %gui         # disable all event loop integration
 
         WARNING:  after any of these has been called you can simply create
         an application object, but DO NOT start the event loop yourself, as
         we have already handled that.
 
         If you want us to create an appropriate application object add the
         "-a" flag to your command::
 
             %gui -a wx
 
         This is highly recommended for most users.
         """
         opts, arg = self.parse_options(parameter_s,'a')
         if arg=='': arg = None
         return enable_gui(arg, 'a' in opts)
 
     def magic_load_ext(self, module_str):
         """Load an IPython extension by its module name."""
         return self.load_extension(module_str)
 
     def magic_unload_ext(self, module_str):
         """Unload an IPython extension by its module name."""
         self.unload_extension(module_str)
 
     def magic_reload_ext(self, module_str):
         """Reload an IPython extension by its module name."""
         self.reload_extension(module_str)
 
     @testdec.skip_doctest
     def magic_install_profiles(self, s):
         """Install the default IPython profiles into the .ipython dir.
 
         If the default profiles have already been installed, they will not
         be overwritten. You can force overwriting them by using the ``-o``
         option::
 
             In [1]: %install_profiles -o
         """
         if '-o' in s:
             overwrite = True
         else:
             overwrite = False
         from IPython.config import profile
         profile_dir = os.path.split(profile.__file__)[0]
         ipython_dir = self.ipython_dir
         files = os.listdir(profile_dir)
 
         to_install = []
         for f in files:
             if f.startswith('ipython_config'):
                 src = os.path.join(profile_dir, f)
                 dst = os.path.join(ipython_dir, f)
                 if (not os.path.isfile(dst)) or overwrite:
                     to_install.append((f, src, dst))
         if len(to_install)>0:
             print "Installing profiles to: ", ipython_dir
             for (f, src, dst) in to_install:
                 shutil.copy(src, dst)
                 print "    %s" % f
 
     def magic_install_default_config(self, s):
         """Install IPython's default config file into the .ipython dir.
 
         If the default config file (:file:`ipython_config.py`) is already
         installed, it will not be overwritten. You can force overwriting
         by using the ``-o`` option::
 
             In [1]: %install_default_config
         """
         if '-o' in s:
             overwrite = True
         else:
             overwrite = False
         from IPython.config import default
         config_dir = os.path.split(default.__file__)[0]
         ipython_dir = self.ipython_dir
         default_config_file_name = 'ipython_config.py'
         src = os.path.join(config_dir, default_config_file_name)
         dst = os.path.join(ipython_dir, default_config_file_name)
         if (not os.path.isfile(dst)) or overwrite:
             shutil.copy(src, dst)
             print "Installing default config file: %s" % dst
 
     # Pylab support: simple wrappers that activate pylab, load gui input
     # handling and modify slightly %run
 
     @testdec.skip_doctest
     def _pylab_magic_run(self, parameter_s=''):
         Magic.magic_run(self, parameter_s,
                         runner=mpl_runner(self.shell.safe_execfile))
 
     _pylab_magic_run.__doc__ = magic_run.__doc__
 
     @testdec.skip_doctest
     def magic_pylab(self, s):
         """Load numpy and matplotlib to work interactively.
 
         %pylab [GUINAME]
 
         This function lets you activate pylab (matplotlib, numpy and
         interactive support) at any point during an IPython session.
 
         It will import at the top level numpy as np, pyplot as plt, matplotlib,
         pylab and mlab, as well as all names from numpy and pylab.
 
         Parameters
         ----------
         guiname : optional
           One of the valid arguments to the %gui magic ('qt', 'wx', 'gtk' or
           'tk').  If given, the corresponding Matplotlib backend is used,
           otherwise matplotlib's default (which you can override in your
           matplotlib config file) is used.
 
         Examples
         --------
         In this case, where the MPL default is TkAgg:
         In [2]: %pylab
 
         Welcome to pylab, a matplotlib-based Python environment.
         Backend in use: TkAgg
         For more information, type 'help(pylab)'.
 
         But you can explicitly request a different backend:
         In [3]: %pylab qt
 
         Welcome to pylab, a matplotlib-based Python environment.
         Backend in use: Qt4Agg
         For more information, type 'help(pylab)'.
         """
         self.shell.enable_pylab(s)
 
     def magic_tb(self, s):
         """Print the last traceback with the currently active exception mode.
 
         See %xmode for changing exception reporting modes."""
         self.shell.showtraceback()
 
 # end Magic

IPython/core/prefilter.py

@@ -1,1051 +1,1050 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 Prefiltering components.
 
 Prefilters transform user input before it is exec'd by Python.  These
 transforms are used to implement additional syntax such as !ls and %magic.
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 * Dan Milstein
 * Ville Vainio
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import __builtin__
 import codeop
 import re
 
 from IPython.core.alias import AliasManager
 from IPython.core.autocall import IPyAutocall
 from IPython.core.component import Component
 from IPython.core.splitinput import split_user_input
 from IPython.core.page import page
 
 from IPython.utils.traitlets import List, Int, Any, Str, CBool, Bool
 from IPython.utils.io import Term
 from IPython.utils.text import make_quoted_expr
 from IPython.utils.autoattr import auto_attr
 
 #-----------------------------------------------------------------------------
 # Global utilities, errors and constants
 #-----------------------------------------------------------------------------
 
 # Warning, these cannot be changed unless various regular expressions
 # are updated in a number of places.  Not great, but at least we told you.
 ESC_SHELL  = '!'
 ESC_SH_CAP = '!!'
 ESC_HELP   = '?'
 ESC_MAGIC  = '%'
 ESC_QUOTE  = ','
 ESC_QUOTE2 = ';'
 ESC_PAREN  = '/'
 
 
 class PrefilterError(Exception):
     pass
 
 
 # RegExp to identify potential function names
 re_fun_name = re.compile(r'[a-zA-Z_]([a-zA-Z0-9_.]*) *$')
 
 # RegExp to exclude strings with this start from autocalling.  In
 # particular, all binary operators should be excluded, so that if foo is
 # callable, foo OP bar doesn't become foo(OP bar), which is invalid.  The
 # characters '!=()' don't need to be checked for, as the checkPythonChars
 # routine explicitely does so, to catch direct calls and rebindings of
 # existing names.
 
 # Warning: the '-' HAS TO BE AT THE END of the first group, otherwise
 # it affects the rest of the group in square brackets.
 re_exclude_auto = re.compile(r'^[,&^\|\*/\+-]'
                              r'|^is |^not |^in |^and |^or ')
 
 # try to catch also methods for stuff in lists/tuples/dicts: off
 # (experimental). For this to work, the line_split regexp would need
 # to be modified so it wouldn't break things at '['. That line is
 # nasty enough that I shouldn't change it until I can test it _well_.
 #self.re_fun_name = re.compile (r'[a-zA-Z_]([a-zA-Z0-9_.\[\]]*) ?$')
 
 
 # Handler Check Utilities
 def is_shadowed(identifier, ip):
     """Is the given identifier defined in one of the namespaces which shadow
     the alias and magic namespaces?  Note that an identifier is different
     than ifun, because it can not contain a '.' character."""
     # This is much safer than calling ofind, which can change state
     return (identifier in ip.user_ns \
             or identifier in ip.internal_ns \
             or identifier in ip.ns_table['builtin'])
 
 
 #-----------------------------------------------------------------------------
 # The LineInfo class used throughout
 #-----------------------------------------------------------------------------
 
 
 class LineInfo(object):
     """A single line of input and associated info.
 
     Includes the following as properties: 
 
     line
       The original, raw line
     
     continue_prompt
       Is this line a continuation in a sequence of multiline input?
     
     pre
       The initial esc character or whitespace.
     
     pre_char
       The escape character(s) in pre or the empty string if there isn't one.
       Note that '!!' is a possible value for pre_char.  Otherwise it will
       always be a single character.
     
     pre_whitespace
       The leading whitespace from pre if it exists.  If there is a pre_char,
       this is just ''.
     
     ifun
       The 'function part', which is basically the maximal initial sequence
       of valid python identifiers and the '.' character.  This is what is
       checked for alias and magic transformations, used for auto-calling,
       etc.
     
     the_rest
       Everything else on the line.
     """
     def __init__(self, line, continue_prompt):
         self.line            = line
         self.continue_prompt = continue_prompt
         self.pre, self.ifun, self.the_rest = split_user_input(line)
 
         self.pre_char       = self.pre.strip()
         if self.pre_char:
             self.pre_whitespace = '' # No whitespace allowd before esc chars
         else: 
             self.pre_whitespace = self.pre
 
         self._oinfo = None
 
     def ofind(self, ip):
         """Do a full, attribute-walking lookup of the ifun in the various
         namespaces for the given IPython InteractiveShell instance.
 
         Return a dict with keys: found,obj,ospace,ismagic
 
         Note: can cause state changes because of calling getattr, but should
         only be run if autocall is on and if the line hasn't matched any
         other, less dangerous handlers.
 
         Does cache the results of the call, so can be called multiple times
         without worrying about *further* damaging state.
         """
         if not self._oinfo:
             # ip.shell._ofind is actually on the Magic class!
             self._oinfo = ip.shell._ofind(self.ifun)
         return self._oinfo
 
     def __str__(self):                                                         
         return "Lineinfo [%s|%s|%s]" %(self.pre, self.ifun, self.the_rest) 
 
 
 #-----------------------------------------------------------------------------
 # Main Prefilter manager
 #-----------------------------------------------------------------------------
 
 
 class PrefilterManager(Component):
     """Main prefilter component.
 
     The IPython prefilter is run on all user input before it is run.  The
     prefilter consumes lines of input and produces transformed lines of 
     input.
 
     The iplementation consists of two phases:
 
     1. Transformers
     2. Checkers and handlers
 
     Over time, we plan on deprecating the checkers and handlers and doing
     everything in the transformers.
 
     The transformers are instances of :class:`PrefilterTransformer` and have
     a single method :meth:`transform` that takes a line and returns a
     transformed line.  The transformation can be accomplished using any
     tool, but our current ones use regular expressions for speed.  We also
     ship :mod:`pyparsing` in :mod:`IPython.external` for use in transformers.
 
     After all the transformers have been run, the line is fed to the checkers,
     which are instances of :class:`PrefilterChecker`.  The line is passed to
     the :meth:`check` method, which either returns `None` or a 
     :class:`PrefilterHandler` instance.  If `None` is returned, the other
     checkers are tried.  If an :class:`PrefilterHandler` instance is returned,
     the line is passed to the :meth:`handle` method of the returned
     handler and no further checkers are tried.
     
     Both transformers and checkers have a `priority` attribute, that determines
     the order in which they are called.  Smaller priorities are tried first.
 
     Both transformers and checkers also have `enabled` attribute, which is
     a boolean that determines if the instance is used.
 
     Users or developers can change the priority or enabled attribute of
     transformers or checkers, but they must call the :meth:`sort_checkers`
     or :meth:`sort_transformers` method after changing the priority.
     """
 
     multi_line_specials = CBool(True, config=True)
 
     def __init__(self, parent, config=None):
         super(PrefilterManager, self).__init__(parent, config=config)
         self.init_transformers()
         self.init_handlers()
         self.init_checkers()
 
     @auto_attr
     def shell(self):
         return Component.get_instances(
             root=self.root,
             klass='IPython.core.iplib.InteractiveShell')[0]
 
     #-------------------------------------------------------------------------
     # API for managing transformers
     #-------------------------------------------------------------------------
 
     def init_transformers(self):
         """Create the default transformers."""
         self._transformers = []
         for transformer_cls in _default_transformers:
             transformer_cls(self, config=self.config)
 
     def sort_transformers(self):
         """Sort the transformers by priority.
 
         This must be called after the priority of a transformer is changed.
         The :meth:`register_transformer` method calls this automatically.
         """
         self._transformers.sort(cmp=lambda x,y: x.priority-y.priority)
 
     @property
     def transformers(self):
         """Return a list of checkers, sorted by priority."""
         return self._transformers
 
     def register_transformer(self, transformer):
         """Register a transformer instance."""
         if transformer not in self._transformers:
             self._transformers.append(transformer)
             self.sort_transformers()
 
     def unregister_transformer(self, transformer):
         """Unregister a transformer instance."""
         if transformer in self._transformers:
             self._transformers.remove(transformer)
 
     #-------------------------------------------------------------------------
     # API for managing checkers
     #-------------------------------------------------------------------------
 
     def init_checkers(self):
         """Create the default checkers."""
         self._checkers = []
         for checker in _default_checkers:
             checker(self, config=self.config)
 
     def sort_checkers(self):
         """Sort the checkers by priority.
 
         This must be called after the priority of a checker is changed.
         The :meth:`register_checker` method calls this automatically.
         """
         self._checkers.sort(cmp=lambda x,y: x.priority-y.priority)
 
     @property
     def checkers(self):
         """Return a list of checkers, sorted by priority."""
         return self._checkers
 
     def register_checker(self, checker):
         """Register a checker instance."""
         if checker not in self._checkers:
             self._checkers.append(checker)
             self.sort_checkers()
 
     def unregister_checker(self, checker):
         """Unregister a checker instance."""
         if checker in self._checkers:
             self._checkers.remove(checker)
 
     #-------------------------------------------------------------------------
     # API for managing checkers
     #-------------------------------------------------------------------------
 
     def init_handlers(self):
         """Create the default handlers."""
         self._handlers = {}
         self._esc_handlers = {}
         for handler in _default_handlers:
             handler(self, config=self.config)
 
     @property
     def handlers(self):
         """Return a dict of all the handlers."""
         return self._handlers
 
     def register_handler(self, name, handler, esc_strings):
         """Register a handler instance by name with esc_strings."""
         self._handlers[name] = handler
         for esc_str in esc_strings:
             self._esc_handlers[esc_str] = handler
 
     def unregister_handler(self, name, handler, esc_strings):
         """Unregister a handler instance by name with esc_strings."""
         try:
             del self._handlers[name]
         except KeyError:
             pass
         for esc_str in esc_strings:
             h = self._esc_handlers.get(esc_str)
             if h is handler:
                 del self._esc_handlers[esc_str]
 
     def get_handler_by_name(self, name):
         """Get a handler by its name."""
         return self._handlers.get(name)
 
     def get_handler_by_esc(self, esc_str):
         """Get a handler by its escape string."""
         return self._esc_handlers.get(esc_str)
 
     #-------------------------------------------------------------------------
     # Main prefiltering API
     #-------------------------------------------------------------------------
 
     def prefilter_line_info(self, line_info):
         """Prefilter a line that has been converted to a LineInfo object.
 
         This implements the checker/handler part of the prefilter pipe.
         """
         # print "prefilter_line_info: ", line_info
         handler = self.find_handler(line_info)
         return handler.handle(line_info)
 
     def find_handler(self, line_info):
         """Find a handler for the line_info by trying checkers."""
         for checker in self.checkers:
             if checker.enabled:
                 handler = checker.check(line_info)
                 if handler:
                     return handler
         return self.get_handler_by_name('normal')
 
     def transform_line(self, line, continue_prompt):
         """Calls the enabled transformers in order of increasing priority."""
         for transformer in self.transformers:
             if transformer.enabled:
                 line = transformer.transform(line, continue_prompt)
         return line
 
     def prefilter_line(self, line, continue_prompt=False):
         """Prefilter a single input line as text.
 
         This method prefilters a single line of text by calling the
         transformers and then the checkers/handlers.
         """
 
         # print "prefilter_line: ", line, continue_prompt
         # All handlers *must* return a value, even if it's blank ('').
 
         # Lines are NOT logged here. Handlers should process the line as
         # needed, update the cache AND log it (so that the input cache array
         # stays synced).
 
         # save the line away in case we crash, so the post-mortem handler can
         # record it
         self.shell._last_input_line = line
 
         if not line:
             # Return immediately on purely empty lines, so that if the user
             # previously typed some whitespace that started a continuation
             # prompt, he can break out of that loop with just an empty line.
             # This is how the default python prompt works.
 
             # Only return if the accumulated input buffer was just whitespace!
             if ''.join(self.shell.buffer).isspace():
                 self.shell.buffer[:] = []
             return ''
 
         # At this point, we invoke our transformers.
         if not continue_prompt or (continue_prompt and self.multi_line_specials):
             line = self.transform_line(line, continue_prompt)
 
         # Now we compute line_info for the checkers and handlers
         line_info = LineInfo(line, continue_prompt)
         
         # the input history needs to track even empty lines
         stripped = line.strip()
 
         normal_handler = self.get_handler_by_name('normal')
         if not stripped:
             if not continue_prompt:
                 self.shell.outputcache.prompt_count -= 1
 
             return normal_handler.handle(line_info)
 
         # special handlers are only allowed for single line statements
         if continue_prompt and not self.multi_line_specials:
             return normal_handler.handle(line_info)
 
         prefiltered = self.prefilter_line_info(line_info)
         # print "prefiltered line: %r" % prefiltered
         return prefiltered
 
     def prefilter_lines(self, lines, continue_prompt=False):
         """Prefilter multiple input lines of text.
 
         This is the main entry point for prefiltering multiple lines of
         input.  This simply calls :meth:`prefilter_line` for each line of
         input.
 
         This covers cases where there are multiple lines in the user entry,
         which is the case when the user goes back to a multiline history
         entry and presses enter.
         """
         llines = lines.rstrip('\n').split('\n')
         # We can get multiple lines in one shot, where multiline input 'blends'
         # into one line, in cases like recalling from the readline history
         # buffer.  We need to make sure that in such cases, we correctly
         # communicate downstream which line is first and which are continuation
         # ones.
         if len(llines) > 1:
             out = '\n'.join([self.prefilter_line(line, lnum>0)
                              for lnum, line in enumerate(llines) ])
         else:
             out = self.prefilter_line(llines[0], continue_prompt)
             
         return out
 
 #-----------------------------------------------------------------------------
 # Prefilter transformers
 #-----------------------------------------------------------------------------
 
 
 class PrefilterTransformer(Component):
     """Transform a line of user input."""
 
     priority = Int(100, config=True)
     shell = Any
     prefilter_manager = Any
     enabled = Bool(True, config=True)
 
     def __init__(self, parent, config=None):
         super(PrefilterTransformer, self).__init__(parent, config=config)
         self.prefilter_manager.register_transformer(self)
 
     @auto_attr
     def shell(self):
         return Component.get_instances(
             root=self.root,
             klass='IPython.core.iplib.InteractiveShell')[0]
 
     @auto_attr
     def prefilter_manager(self):
         return PrefilterManager.get_instances(root=self.root)[0]
 
     def transform(self, line, continue_prompt):
         """Transform a line, returning the new one."""
         return None
 
     def __repr__(self):
         return "<%s(priority=%r, enabled=%r)>" % (
             self.__class__.__name__, self.priority, self.enabled)
 
 
 _assign_system_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
                                r'\s*=\s*!(?P<cmd>.*)')
 
 
 class AssignSystemTransformer(PrefilterTransformer):
     """Handle the `files = !ls` syntax."""
 
     priority = Int(100, config=True)
 
     def transform(self, line, continue_prompt):
         m = _assign_system_re.match(line)
         if m is not None:
             cmd = m.group('cmd')
             lhs = m.group('lhs')
             expr = make_quoted_expr("sc -l =%s" % cmd)
             new_line = '%s = get_ipython().magic(%s)' % (lhs, expr)
             return new_line
         return line
 
 
 _assign_magic_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
                                r'\s*=\s*%(?P<cmd>.*)')
 
 class AssignMagicTransformer(PrefilterTransformer):
     """Handle the `a = %who` syntax."""
 
     priority = Int(200, config=True)
 
     def transform(self, line, continue_prompt):
         m = _assign_magic_re.match(line)
         if m is not None:
             cmd = m.group('cmd')
             lhs = m.group('lhs')
             expr = make_quoted_expr(cmd)
             new_line = '%s = get_ipython().magic(%s)' % (lhs, expr)
             return new_line
         return line
 
 
 _classic_prompt_re = re.compile(r'(^[ \t]*>>> |^[ \t]*\.\.\. )')
 
 class PyPromptTransformer(PrefilterTransformer):
     """Handle inputs that start with '>>> ' syntax."""
 
     priority = Int(50, config=True)
 
     def transform(self, line, continue_prompt):
 
         if not line or line.isspace() or line.strip() == '...':
             # This allows us to recognize multiple input prompts separated by
             # blank lines and pasted in a single chunk, very common when
             # pasting doctests or long tutorial passages.
             return ''
         m = _classic_prompt_re.match(line)
         if m:
             return line[len(m.group(0)):]
         else:
             return line
 
 
 _ipy_prompt_re = re.compile(r'(^[ \t]*In \[\d+\]: |^[ \t]*\ \ \ \.\.\.+: )')
 
 class IPyPromptTransformer(PrefilterTransformer):
     """Handle inputs that start classic IPython prompt syntax."""
 
     priority = Int(50, config=True)
 
     def transform(self, line, continue_prompt):
 
         if not line or line.isspace() or line.strip() == '...':
             # This allows us to recognize multiple input prompts separated by
             # blank lines and pasted in a single chunk, very common when
             # pasting doctests or long tutorial passages.
             return ''
         m = _ipy_prompt_re.match(line)
         if m:
             return line[len(m.group(0)):]
         else:
             return line
 
 #-----------------------------------------------------------------------------
 # Prefilter checkers
 #-----------------------------------------------------------------------------
 
 
 class PrefilterChecker(Component):
     """Inspect an input line and return a handler for that line."""
 
     priority = Int(100, config=True)
     shell = Any
     prefilter_manager = Any
     enabled = Bool(True, config=True)
 
     def __init__(self, parent, config=None):
         super(PrefilterChecker, self).__init__(parent, config=config)
         self.prefilter_manager.register_checker(self)
 
     @auto_attr
     def shell(self):
         return Component.get_instances(
             root=self.root,
             klass='IPython.core.iplib.InteractiveShell')[0]
 
     @auto_attr
     def prefilter_manager(self):
         return PrefilterManager.get_instances(root=self.root)[0]
 
     def check(self, line_info):
         """Inspect line_info and return a handler instance or None."""
         return None
 
     def __repr__(self):
         return "<%s(priority=%r, enabled=%r)>" % (
             self.__class__.__name__, self.priority, self.enabled)
 
 
 class EmacsChecker(PrefilterChecker):
 
     priority = Int(100, config=True)
     enabled = Bool(False, config=True)
 
     def check(self, line_info):
         "Emacs ipython-mode tags certain input lines."
         if line_info.line.endswith('# PYTHON-MODE'):
             return self.prefilter_manager.get_handler_by_name('emacs')
         else:
             return None
 
 
 class ShellEscapeChecker(PrefilterChecker):
 
     priority = Int(200, config=True)
 
     def check(self, line_info):
         if line_info.line.lstrip().startswith(ESC_SHELL):
             return self.prefilter_manager.get_handler_by_name('shell')
 
 
 class IPyAutocallChecker(PrefilterChecker):
 
     priority = Int(300, config=True)
 
     def check(self, line_info):
         "Instances of IPyAutocall in user_ns get autocalled immediately"
         obj = self.shell.user_ns.get(line_info.ifun, None)
         if isinstance(obj, IPyAutocall):
             obj.set_ip(self.shell)
             return self.prefilter_manager.get_handler_by_name('auto')
         else:
             return None
 
 
 class MultiLineMagicChecker(PrefilterChecker):
 
     priority = Int(400, config=True)
 
     def check(self, line_info):
         "Allow ! and !! in multi-line statements if multi_line_specials is on"
         # Note that this one of the only places we check the first character of
         # ifun and *not* the pre_char.  Also note that the below test matches
         # both ! and !!.    
         if line_info.continue_prompt \
             and self.prefilter_manager.multi_line_specials:
                 if line_info.ifun.startswith(ESC_MAGIC):
                     return self.prefilter_manager.get_handler_by_name('magic')
         else:
             return None
 
 
 class EscCharsChecker(PrefilterChecker):
 
     priority = Int(500, config=True)
 
     def check(self, line_info):
         """Check for escape character and return either a handler to handle it,
         or None if there is no escape char."""
         if line_info.line[-1] == ESC_HELP \
                and line_info.pre_char != ESC_SHELL \
                and line_info.pre_char != ESC_SH_CAP:
             # the ? can be at the end, but *not* for either kind of shell escape,
             # because a ? can be a vaild final char in a shell cmd
             return self.prefilter_manager.get_handler_by_name('help')
         else:
             # This returns None like it should if no handler exists
             return self.prefilter_manager.get_handler_by_esc(line_info.pre_char)
 
 
 class AssignmentChecker(PrefilterChecker):
 
     priority = Int(600, config=True)
 
     def check(self, line_info):
         """Check to see if user is assigning to a var for the first time, in
         which case we want to avoid any sort of automagic / autocall games.
     
         This allows users to assign to either alias or magic names true python
         variables (the magic/alias systems always take second seat to true
         python code).  E.g. ls='hi', or ls,that=1,2"""
         if line_info.the_rest:
             if line_info.the_rest[0] in '=,':
                 return self.prefilter_manager.get_handler_by_name('normal')
         else:
             return None
 
 
 class AutoMagicChecker(PrefilterChecker):
 
     priority = Int(700, config=True)
 
     def check(self, line_info):
         """If the ifun is magic, and automagic is on, run it.  Note: normal,
         non-auto magic would already have been triggered via '%' in
         check_esc_chars. This just checks for automagic.  Also, before
         triggering the magic handler, make sure that there is nothing in the
         user namespace which could shadow it."""
         if not self.shell.automagic or not hasattr(self.shell,'magic_'+line_info.ifun):
             return None
 
         # We have a likely magic method.  Make sure we should actually call it.
         if line_info.continue_prompt and not self.prefilter_manager.multi_line_specials:
             return None
 
         head = line_info.ifun.split('.',1)[0]
         if is_shadowed(head, self.shell):
             return None
 
         return self.prefilter_manager.get_handler_by_name('magic')
 
 
 class AliasChecker(PrefilterChecker):
 
     priority = Int(800, config=True)
 
     @auto_attr
     def alias_manager(self):
         return AliasManager.get_instances(root=self.root)[0]
 
     def check(self, line_info):
         "Check if the initital identifier on the line is an alias."
         # Note: aliases can not contain '.'
         head = line_info.ifun.split('.',1)[0]
         if line_info.ifun not in self.alias_manager \
                or head not in self.alias_manager \
                or is_shadowed(head, self.shell):
             return None
 
         return self.prefilter_manager.get_handler_by_name('alias')
 
 
 class PythonOpsChecker(PrefilterChecker):
 
     priority = Int(900, config=True)
 
     def check(self, line_info):
         """If the 'rest' of the line begins with a function call or pretty much
         any python operator, we should simply execute the line (regardless of
         whether or not there's a possible autocall expansion).  This avoids
         spurious (and very confusing) geattr() accesses."""
         if line_info.the_rest and line_info.the_rest[0] in '!=()<>,+*/%^&|':
             return self.prefilter_manager.get_handler_by_name('normal')
         else:
             return None
 
 
 class AutocallChecker(PrefilterChecker):
 
     priority = Int(1000, config=True)
 
     def check(self, line_info):
         "Check if the initial word/function is callable and autocall is on."
         if not self.shell.autocall:
             return None
 
         oinfo = line_info.ofind(self.shell) # This can mutate state via getattr
         if not oinfo['found']:
             return None
         
         if callable(oinfo['obj']) \
                and (not re_exclude_auto.match(line_info.the_rest)) \
                and re_fun_name.match(line_info.ifun):
             return self.prefilter_manager.get_handler_by_name('auto')
         else:
             return None
 
 
 #-----------------------------------------------------------------------------
 # Prefilter handlers
 #-----------------------------------------------------------------------------
 
 
 class PrefilterHandler(Component):
 
     handler_name = Str('normal')
     esc_strings = List([])
     shell = Any
     prefilter_manager = Any
 
     def __init__(self, parent, config=None):
         super(PrefilterHandler, self).__init__(parent, config=config)
         self.prefilter_manager.register_handler(
             self.handler_name,
             self,
             self.esc_strings
         )
 
     @auto_attr
     def shell(self):
         return Component.get_instances(
             root=self.root,
             klass='IPython.core.iplib.InteractiveShell')[0]
 
     @auto_attr
     def prefilter_manager(self):
         return PrefilterManager.get_instances(root=self.root)[0]
 
     def handle(self, line_info):
         # print "normal: ", line_info
         """Handle normal input lines. Use as a template for handlers."""
 
         # With autoindent on, we need some way to exit the input loop, and I
         # don't want to force the user to have to backspace all the way to
         # clear the line.  The rule will be in this case, that either two
         # lines of pure whitespace in a row, or a line of pure whitespace but
         # of a size different to the indent level, will exit the input loop.
         line = line_info.line
         continue_prompt = line_info.continue_prompt
 
         if (continue_prompt and
             self.shell.autoindent and
             line.isspace() and
             
             (0 < abs(len(line) - self.shell.indent_current_nsp) <= 2
              or
              not self.shell.buffer
              or
              (self.shell.buffer[-1]).isspace()
              )
             ):
             line = ''
 
         self.shell.log(line, line, continue_prompt)
         return line
 
     def __str__(self):
         return "<%s(name=%s)>" % (self.__class__.__name__, self.handler_name)
 
 
 class AliasHandler(PrefilterHandler):
 
     handler_name = Str('alias')
 
     @auto_attr
     def alias_manager(self):
         return AliasManager.get_instances(root=self.root)[0]
 
     def handle(self, line_info):
         """Handle alias input lines. """
         transformed = self.alias_manager.expand_aliases(line_info.ifun,line_info.the_rest)
         # pre is needed, because it carries the leading whitespace.  Otherwise
         # aliases won't work in indented sections.
         line_out = '%sget_ipython().system(%s)' % (line_info.pre_whitespace,
                                          make_quoted_expr(transformed))
         
         self.shell.log(line_info.line, line_out, line_info.continue_prompt)
         return line_out
 
 
 class ShellEscapeHandler(PrefilterHandler):
 
     handler_name = Str('shell')
     esc_strings = List([ESC_SHELL, ESC_SH_CAP])
 
     def handle(self, line_info):
         """Execute the line in a shell, empty return value"""
         magic_handler = self.prefilter_manager.get_handler_by_name('magic')
 
         line = line_info.line
         if line.lstrip().startswith(ESC_SH_CAP):
             # rewrite LineInfo's line, ifun and the_rest to properly hold the
             # call to %sx and the actual command to be executed, so
             # handle_magic can work correctly.  Note that this works even if
             # the line is indented, so it handles multi_line_specials
             # properly.
             new_rest = line.lstrip()[2:]
             line_info.line = '%ssx %s' % (ESC_MAGIC, new_rest)
             line_info.ifun = 'sx'
             line_info.the_rest = new_rest
             return magic_handler.handle(line_info)
         else:
             cmd = line.lstrip().lstrip(ESC_SHELL)
             line_out = '%sget_ipython().system(%s)' % (line_info.pre_whitespace,
                                              make_quoted_expr(cmd))
         # update cache/log and return
         self.shell.log(line, line_out, line_info.continue_prompt)
         return line_out
 
 
 class MagicHandler(PrefilterHandler):
 
     handler_name = Str('magic')
     esc_strings = List([ESC_MAGIC])
 
     def handle(self, line_info):
         """Execute magic functions."""
         ifun    = line_info.ifun
         the_rest = line_info.the_rest
         cmd = '%sget_ipython().magic(%s)' % (line_info.pre_whitespace,
                                    make_quoted_expr(ifun + " " + the_rest))
         self.shell.log(line_info.line, cmd, line_info.continue_prompt)
         return cmd
 
 
 class AutoHandler(PrefilterHandler):
 
     handler_name = Str('auto')
     esc_strings = List([ESC_PAREN, ESC_QUOTE, ESC_QUOTE2])
 
     def handle(self, line_info):
         """Hande lines which can be auto-executed, quoting if requested."""
         line    = line_info.line
         ifun    = line_info.ifun
         the_rest = line_info.the_rest
         pre     = line_info.pre
         continue_prompt = line_info.continue_prompt
         obj = line_info.ofind(self)['obj']
         #print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun,the_rest)  # dbg
 
         # This should only be active for single-line input!
         if continue_prompt:
             self.shell.log(line,line,continue_prompt)
             return line
 
         force_auto = isinstance(obj, IPyAutocall)
         auto_rewrite = True
         
         if pre == ESC_QUOTE:
             # Auto-quote splitting on whitespace
             newcmd = '%s("%s")' % (ifun,'", "'.join(the_rest.split()) )
         elif pre == ESC_QUOTE2:
             # Auto-quote whole string
             newcmd = '%s("%s")' % (ifun,the_rest)
         elif pre == ESC_PAREN:
             newcmd = '%s(%s)' % (ifun,",".join(the_rest.split()))
         else:
             # Auto-paren.
             # We only apply it to argument-less calls if the autocall
             # parameter is set to 2.  We only need to check that autocall is <
             # 2, since this function isn't called unless it's at least 1.
             if not the_rest and (self.shell.autocall < 2) and not force_auto:
                 newcmd = '%s %s' % (ifun,the_rest)
                 auto_rewrite = False
             else:
                 if not force_auto and the_rest.startswith('['):
                     if hasattr(obj,'__getitem__'):
                         # Don't autocall in this case: item access for an object
                         # which is BOTH callable and implements __getitem__.
                         newcmd = '%s %s' % (ifun,the_rest)
                         auto_rewrite = False
                     else:
                         # if the object doesn't support [] access, go ahead and
                         # autocall
                         newcmd = '%s(%s)' % (ifun.rstrip(),the_rest)
                 elif the_rest.endswith(';'):
                     newcmd = '%s(%s);' % (ifun.rstrip(),the_rest[:-1])
                 else:
                     newcmd = '%s(%s)' % (ifun.rstrip(), the_rest)
 
         if auto_rewrite:
             rw = self.shell.outputcache.prompt1.auto_rewrite() + newcmd
             
             try:
                 # plain ascii works better w/ pyreadline, on some machines, so
                 # we use it and only print uncolored rewrite if we have unicode
                 rw = str(rw)
                 print >>Term.cout, rw
             except UnicodeEncodeError:
                 print "-------------->" + newcmd
             
         # log what is now valid Python, not the actual user input (without the
         # final newline)
         self.shell.log(line,newcmd,continue_prompt)
         return newcmd
 
 
 class HelpHandler(PrefilterHandler):
 
     handler_name = Str('help')
     esc_strings = List([ESC_HELP])
 
     def handle(self, line_info):
         """Try to get some help for the object.
 
         obj? or ?obj   -> basic information.
         obj?? or ??obj -> more details.
         """
         normal_handler = self.prefilter_manager.get_handler_by_name('normal')
         line = line_info.line
         # We need to make sure that we don't process lines which would be
         # otherwise valid python, such as "x=1 # what?"
         try:
             codeop.compile_command(line)
         except SyntaxError:
             # We should only handle as help stuff which is NOT valid syntax
             if line[0]==ESC_HELP:
                 line = line[1:]
             elif line[-1]==ESC_HELP:
                 line = line[:-1]
             self.shell.log(line, '#?'+line, line_info.continue_prompt)
             if line:
                 #print 'line:<%r>' % line  # dbg
                 self.shell.magic_pinfo(line)
             else:
                 page(self.shell.usage, screen_lines=self.shell.usable_screen_length)
             return '' # Empty string is needed here!
         except:
             raise
             # Pass any other exceptions through to the normal handler
             return normal_handler.handle(line_info)
         else:
-            raise
             # If the code compiles ok, we should handle it normally
             return normal_handler.handle(line_info)
 
 
 class EmacsHandler(PrefilterHandler):
 
     handler_name = Str('emacs')
     esc_strings = List([])
 
     def handle(self, line_info):
         """Handle input lines marked by python-mode."""
 
         # Currently, nothing is done.  Later more functionality can be added
         # here if needed.
 
         # The input cache shouldn't be updated
         return line_info.line
 
 
 #-----------------------------------------------------------------------------
 # Defaults
 #-----------------------------------------------------------------------------
 
 
 _default_transformers = [
     AssignSystemTransformer,
     AssignMagicTransformer,
     PyPromptTransformer,
     IPyPromptTransformer,
 ]
 
 _default_checkers = [
     EmacsChecker,
     ShellEscapeChecker,
     IPyAutocallChecker,
     MultiLineMagicChecker,
     EscCharsChecker,
     AssignmentChecker,
     AutoMagicChecker,
     AliasChecker,
     PythonOpsChecker,
     AutocallChecker
 ]
 
 _default_handlers = [
     PrefilterHandler,
     AliasHandler,
     ShellEscapeHandler,
     MagicHandler,
     AutoHandler,
     HelpHandler,
     EmacsHandler
 ]

IPython/core/tests/test_handlers.py

@@ -1,202 +1,174 b''
-"""Test the various handlers which do the actual rewriting of the line."""
+"""Tests for input handlers.
+"""
+#-----------------------------------------------------------------------------
+# Module imports
+#-----------------------------------------------------------------------------
 
-from StringIO import StringIO
-import sys
-sys.path.append('..')
+# third party
+import nose.tools as nt
+
+# our own packages
+from IPython.core import autocall
+from IPython.testing import decorators as dec
+from IPython.testing.globalipapp import get_ipython
+
+#-----------------------------------------------------------------------------
+# Globals
+#-----------------------------------------------------------------------------
+
+# Get the public instance of IPython
+ip = get_ipython()
 
 failures = []
 num_tests = 0
 
+#-----------------------------------------------------------------------------
+# Test functions
+#-----------------------------------------------------------------------------
+
+class CallableIndexable(object):
+    def __getitem__(self, idx): return True
+    def __call__(self, *args, **kws): return True
+
+
+class Autocallable(autocall.IPyAutocall):
+    def __call__(self):
+        return "called"
+
+
 def run(tests):
     """Loop through a list of (pre, post) inputs, where pre is the string
     handed to ipython, and post is how that string looks after it's been
     transformed (i.e. ipython's notion of _i)"""
     for pre, post in tests:
         global num_tests
         num_tests += 1        
         ip.runlines(pre)
         ip.runlines('_i')  # Not sure why I need this...
         actual = ip.user_ns['_i']
         if actual != None:
             actual = actual.rstrip('\n')
         if actual != post:
             failures.append('Expected %r to become %r, found %r' % (
                 pre, post, actual))
 
 
-# Shutdown stdout/stderr so that ipython isn't noisy during tests.  Have to
-# do this *before* importing IPython below.
-#
-# NOTE: this means that, if you stick print statements into code as part of
-# debugging, you won't see the results (unless you comment out some of the
-# below).  I keep on doing this, so apparently it's easy.  Or I am an idiot.
-old_stdout = sys.stdout
-old_stderr = sys.stderr
-
-sys.stdout = StringIO()
-sys.stderr = StringIO()
-
-import IPython
-import IPython.ipapi
-
-IPython.Shell.start()
-ip = IPython.ipapi.get()
-
-class CallableIndexable(object):
-    def __getitem__(self, idx): return True
-    def __call__(self, *args, **kws): return True
-
-    
-try:
+def test_handlers():
     # alias expansion
-    
+
     # We're using 'true' as our syscall of choice because it doesn't
     # write anything to stdout.
 
     # Turn off actual execution of aliases, because it's noisy
     old_system_cmd = ip.system
     ip.system = lambda cmd: None
-    
-    
-    ip.IP.alias_table['an_alias'] = (0, 'true')
+
+
+    ip.alias_manager.alias_table['an_alias'] = (0, 'true')
     # These are useful for checking a particular recursive alias issue
-    ip.IP.alias_table['top'] = (0, 'd:/cygwin/top')
-    ip.IP.alias_table['d'] =   (0, 'true')
-    run([("an_alias",    '_ip.system("true ")'),     # alias
+    ip.alias_manager.alias_table['top'] = (0, 'd:/cygwin/top')
+    ip.alias_manager.alias_table['d'] =   (0, 'true')
+    run([("an_alias",    'get_ipython().system("true ")'),     # alias
          # Below: recursive aliases should expand whitespace-surrounded
          # chars, *not* initial chars which happen to be aliases:
-         ("top",         '_ip.system("d:/cygwin/top ")'),
+         ("top",         'get_ipython().system("d:/cygwin/top ")'),
          ])
     ip.system = old_system_cmd
 
-
     call_idx = CallableIndexable()
-    ip.to_user_ns('call_idx')
+    ip.user_ns['call_idx'] = call_idx
 
     # For many of the below, we're also checking that leading whitespace
     # turns off the esc char, which it should unless there is a continuation
     # line.
     run([('"no change"', '"no change"'),             # normal
-         ("!true",       '_ip.system("true")'),      # shell_escapes
-         ("!! true",     '_ip.magic("sx  true")'),   # shell_escapes + magic
-         ("!!true",      '_ip.magic("sx true")'),    # shell_escapes + magic
-         ("%lsmagic",    '_ip.magic("lsmagic ")'),   # magic
-         ("lsmagic",     '_ip.magic("lsmagic ")'),   # magic
-         ("a = b # PYTHON-MODE", '_i'),          # emacs -- avoids _in cache
+         ("!true",       'get_ipython().system("true")'),      # shell_escapes
+         ("!! true",     'get_ipython().magic("sx  true")'),   # shell_escapes + magic
+         ("!!true",      'get_ipython().magic("sx true")'),    # shell_escapes + magic
+         ("%lsmagic",    'get_ipython().magic("lsmagic ")'),   # magic
+         ("lsmagic",     'get_ipython().magic("lsmagic ")'),   # magic
+         #("a = b # PYTHON-MODE", '_i'),          # emacs -- avoids _in cache
 
          # post-esc-char whitespace goes inside
-         ("! true",   '_ip.system(" true")'),  
-
-         # Leading whitespace generally turns off escape characters
-         (" ! true",     ' ! true'),  
-         (" !true",      ' !true'),  
+         ("! true",   'get_ipython().system(" true")'),  
 
          # handle_help
 
          # These are weak tests -- just looking at what the help handlers
          # logs, which is not how it really does its work.  But it still
          # lets us check the key paths through the handler.
-        
+
          ("x=1 # what?", "x=1 # what?"), # no help if valid python
-         ("len?",  "#?len"),             # this is what help logs when it runs
-         ("len??", "#?len?"),                           
-         ("?len",  "#?len"),                            
          ])
 
     # multi_line_specials
-    ip.options.multi_line_specials = 0
+    ip.prefilter_manager.multi_line_specials = False
     # W/ multi_line_specials off, leading ws kills esc chars/autoexpansion
     run([
         ('if 1:\n    !true',    'if 1:\n    !true'),
         ('if 1:\n    lsmagic',  'if 1:\n    lsmagic'),
         ('if 1:\n    an_alias', 'if 1:\n    an_alias'),
         ])
 
-    ip.options.multi_line_specials = 1
+    ip.prefilter_manager.multi_line_specials = True
     # initial indents must be preserved.
     run([
-         ('if 1:\n    !true',    'if 1:\n    _ip.system("true")'),
-         ('if 1:\n    lsmagic',  'if 1:\n    _ip.magic("lsmagic ")'),
-         ('if 1:\n    an_alias', 'if 1:\n    _ip.system("true ")'),
+         ('if 1:\n    !true',    'if 1:\n    get_ipython().system("true")'),
+         ('if 2:\n    lsmagic',  'if 2:\n    get_ipython().magic("lsmagic ")'),
+         ('if 1:\n    an_alias', 'if 1:\n    get_ipython().system("true ")'),
          # Weird one
-         ('if 1:\n    !!true',   'if 1:\n    _ip.magic("sx true")'),
-         
+         ('if 1:\n    !!true',   'if 1:\n    get_ipython().magic("sx true")'),
 
-         # Even with m_l_s on, all esc_chars except ! are off
-         ('if 1:\n    %lsmagic', 'if 1:\n    %lsmagic'),
+         # Even with m_l_s on, autocall is off even with special chars
          ('if 1:\n    /fun 1 2', 'if 1:\n    /fun 1 2'), 
          ('if 1:\n    ;fun 1 2', 'if 1:\n    ;fun 1 2'),
          ('if 1:\n    ,fun 1 2', 'if 1:\n    ,fun 1 2'),
          ('if 1:\n    ?fun 1 2', 'if 1:\n    ?fun 1 2'),
          # What about !!  
          ])
 
-         
     # Objects which are instances of IPyAutocall are *always* autocalled
-    import IPython.ipapi
-    class Autocallable(IPython.ipapi.IPyAutocall):
-        def __call__(self):
-            return "called"
-    
     autocallable = Autocallable()
-    ip.to_user_ns('autocallable')
+    ip.user_ns['autocallable'] = autocallable
 
     # auto 
-    ip.options.autocall = 0
+    ip.magic('autocall 0')
     # Only explicit escapes or instances of IPyAutocallable should get
     # expanded
     run([
         ('len "abc"',       'len "abc"'),        
         ('autocallable',    'autocallable()'),     
         (",list 1 2 3",     'list("1", "2", "3")'),
         (";list 1 2 3",     'list("1 2 3")'),      
         ("/len range(1,4)", 'len(range(1,4))'),
         ])
-    ip.options.autocall = 1
+    ip.magic('autocall 1')
     run([
         (",list 1 2 3", 'list("1", "2", "3")'),
         (";list 1 2 3", 'list("1 2 3")'),      
         ("/len range(1,4)", 'len(range(1,4))'),
         ('len "abc"', 'len("abc")'),
         ('len "abc";', 'len("abc");'),  # ; is special -- moves out of parens
         # Autocall is turned off if first arg is [] and the object
         # is both callable and indexable.  Like so:
         ('len [1,2]', 'len([1,2])'),      # len doesn't support __getitem__...
         ('call_idx [1]', 'call_idx [1]'), # call_idx *does*..
         ('call_idx 1', 'call_idx(1)'),
         ('len', 'len '), # only at 2 does it auto-call on single args
         ])
-
-    ip.options.autocall = 2
+    ip.magic('autocall 2')
     run([
         (",list 1 2 3", 'list("1", "2", "3")'),
         (";list 1 2 3", 'list("1 2 3")'),      
         ("/len range(1,4)", 'len(range(1,4))'),
         ('len "abc"', 'len("abc")'),
         ('len "abc";', 'len("abc");'),
         ('len [1,2]', 'len([1,2])'),   
         ('call_idx [1]', 'call_idx [1]'),
         ('call_idx 1', 'call_idx(1)'),
         # This is what's different:
         ('len', 'len()'), # only at 2 does it auto-call on single args
         ])
-    ip.options.autocall = 1
-
-    # Ignoring handle_emacs, 'cause it doesn't do anything.
-finally:
-    sys.stdout = old_stdout
-    sys.stderr = old_stderr
-
-
-
-
-num_f = len(failures)
-#if verbose:
-#    print
-
+    ip.magic('autocall 1')
 
-print "%s tests run, %s failure%s" % (num_tests,
-                                      num_f,
-                                      num_f != 1 and "s" or "")
-for f in failures:
-    print f
+    nt.assert_equals(failures, [])

IPython/core/tests/test_iplib.py

@@ -1,245 +1,279 b''
 """Tests for the key iplib module, where the main ipython class is defined.
 """
 #-----------------------------------------------------------------------------
 # Module imports
 #-----------------------------------------------------------------------------
 
 # stdlib
 import os
 import shutil
 import tempfile
 
 # third party
 import nose.tools as nt
 
 # our own packages
 from IPython.testing import decorators as dec
 from IPython.testing.globalipapp import get_ipython
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # Get the public instance of IPython
 ip = get_ipython()
 
 #-----------------------------------------------------------------------------
 # Test functions
 #-----------------------------------------------------------------------------
 
 @dec.parametric
 def test_reset():
     """reset must clear most namespaces."""
     # The number of variables in the private user_ns_hidden is not zero, but it
     # should be constant regardless of what we do
     nvars_config_ns = len(ip.user_ns_hidden)
 
     # Check that reset runs without error
     ip.reset()
 
     # Once we've reset it (to clear of any junk that might have been there from
     # other tests, we can count how many variables are in the user's namespace
     nvars_user_ns = len(ip.user_ns)
 
     # Now add a few variables to user_ns, and check that reset clears them
     ip.user_ns['x'] = 1
     ip.user_ns['y'] = 1
     ip.reset()
     
     # Finally, check that all namespaces have only as many variables as we
     # expect to find in them:
     for ns in ip.ns_refs_table:
         if ns is ip.user_ns:
             nvars_expected = nvars_user_ns
         elif ns is ip.user_ns_hidden:
             nvars_expected = nvars_config_ns
         else:
             nvars_expected = 0
             
         yield nt.assert_equals(len(ns), nvars_expected)
 
 
 # Tests for reporting of exceptions in various modes, handling of SystemExit,
 # and %tb functionality.  This is really a mix of testing ultraTB and iplib.
 
 def doctest_tb_plain():
     """
 In [18]: xmode plain
 Exception reporting mode: Plain
 
 In [19]: run simpleerr.py
 Traceback (most recent call last):
   ...line 32, in <module>
     bar(mode)
   ...line 16, in bar
     div0()
   ...line 8, in div0
     x/y
 ZeroDivisionError: integer division or modulo by zero
     """
 
 
 def doctest_tb_context():
     """
 In [3]: xmode context
 Exception reporting mode: Context
 
 In [4]: run simpleerr.py
 ---------------------------------------------------------------------------
 ZeroDivisionError                         Traceback (most recent call last)
 <BLANKLINE>
 ... in <module>()
      30         mode = 'div'
      31 
 ---> 32     bar(mode)
      33 
      34 
 <BLANKLINE>
 ... in bar(mode)
      14     "bar"
      15     if mode=='div':
 ---> 16         div0()
      17     elif mode=='exit':
      18         try:
 <BLANKLINE>
 ... in div0()
       6     x = 1
       7     y = 0
 ----> 8     x/y
       9 
      10 def sysexit(stat, mode):
 <BLANKLINE>
 ZeroDivisionError: integer division or modulo by zero
 """
 
 
 def doctest_tb_verbose():
     """
 In [5]: xmode verbose
 Exception reporting mode: Verbose
 
 In [6]: run simpleerr.py
 ---------------------------------------------------------------------------
 ZeroDivisionError                         Traceback (most recent call last)
 <BLANKLINE>
 ... in <module>()
      30         mode = 'div'
      31 
 ---> 32     bar(mode)
         global bar = <function bar at ...>
         global mode = 'div'
      33 
      34 
 <BLANKLINE>
 ... in bar(mode='div')
      14     "bar"
      15     if mode=='div':
 ---> 16         div0()
         global div0 = <function div0 at ...>
      17     elif mode=='exit':
      18         try:
 <BLANKLINE>
 ... in div0()
       6     x = 1
       7     y = 0
 ----> 8     x/y
         x = 1
         y = 0
       9 
      10 def sysexit(stat, mode):
 <BLANKLINE>
 ZeroDivisionError: integer division or modulo by zero
       """
 
 
 def doctest_tb_sysexit():
     """
 In [17]: %xmode plain
 Exception reporting mode: Plain
 
 In [18]: %run simpleerr.py exit
 An exception has occurred, use %tb to see the full traceback.
 SystemExit: (1, 'Mode = exit')
 
 In [19]: %run simpleerr.py exit 2
 An exception has occurred, use %tb to see the full traceback.
 SystemExit: (2, 'Mode = exit')
 
 In [20]: %tb
 Traceback (most recent call last):
   File ... in <module>
     bar(mode)
   File ... line 22, in bar
     sysexit(stat, mode)
   File ... line 11, in sysexit
     raise SystemExit(stat, 'Mode = %s' % mode)
 SystemExit: (2, 'Mode = exit')
 
 In [21]: %xmode context
 Exception reporting mode: Context
 
 In [22]: %tb
 ---------------------------------------------------------------------------
 SystemExit                                Traceback (most recent call last)
 <BLANKLINE>
 ...<module>()
      30         mode = 'div'
      31 
 ---> 32     bar(mode)
      33 
      34 
 <BLANKLINE>
 ...bar(mode)
      20         except:
      21             stat = 1
 ---> 22         sysexit(stat, mode)
      23     else:
      24         raise ValueError('Unknown mode')
 <BLANKLINE>
 ...sysexit(stat, mode)
       9 
      10 def sysexit(stat, mode):
 ---> 11     raise SystemExit(stat, 'Mode = %s' % mode)
      12 
      13 def bar(mode):
 <BLANKLINE>
 SystemExit: (2, 'Mode = exit')
 
 In [23]: %xmode verbose
 Exception reporting mode: Verbose
 
 In [24]: %tb
 ---------------------------------------------------------------------------
 SystemExit                                Traceback (most recent call last)
 <BLANKLINE>
 ... in <module>()
      30         mode = 'div'
      31 
 ---> 32     bar(mode)
         global bar = <function bar at ...>
         global mode = 'exit'
      33 
      34 
 <BLANKLINE>
 ... in bar(mode='exit')
      20         except:
      21             stat = 1
 ---> 22         sysexit(stat, mode)
         global sysexit = <function sysexit at ...>
         stat = 2
         mode = 'exit'
      23     else:
      24         raise ValueError('Unknown mode')
 <BLANKLINE>
 ... in sysexit(stat=2, mode='exit')
       9 
      10 def sysexit(stat, mode):
 ---> 11     raise SystemExit(stat, 'Mode = %s' % mode)
         global SystemExit = undefined
         stat = 2
         mode = 'exit'
      12 
      13 def bar(mode):
 <BLANKLINE>
 SystemExit: (2, 'Mode = exit')
     """
+
+
+def test_runlines():
+    import textwrap
+    ip.runlines(['a = 10', 'a+=1'])
+    ip.runlines('assert a == 11\nassert 1')
+
+    nt.assert_equals(ip.user_ns['a'], 11)
+    complex = textwrap.dedent("""
+    if 1:
+        print "hello"
+        if 1:
+            print "world"
+        
+    if 2:
+        print "foo"
+
+    if 3:
+        print "bar"
+
+    if 4:
+        print "bar"
+    
+    """)
+    # Simply verifies that this kind of input is run
+    ip.runlines(complex)
+    
+
+def test_db():
+    """Test the internal database used for variable persistence."""
+    ip.db['__unittest_'] = 12
+    nt.assert_equals(ip.db['__unittest_'], 12)
+    del ip.db['__unittest_']
+    assert '__unittest_' not in ip.db

IPython/core/tests/test_magic.py

@@ -1,274 +1,357 b''
 """Tests for various magic functions.
 
 Needs to be run by nose (to make ipython session available).
 """
 from __future__ import absolute_import
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 import tempfile
 import types
 from cStringIO import StringIO
 
 import nose.tools as nt
 
 from IPython.utils.path import get_long_path_name
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 
 #-----------------------------------------------------------------------------
 # Test functions begin
 #-----------------------------------------------------------------------------
 def test_rehashx():
     # clear up everything
     _ip = get_ipython()
     _ip.alias_manager.alias_table.clear()
     del _ip.db['syscmdlist']
     
     _ip.magic('rehashx')
     # Practically ALL ipython development systems will have more than 10 aliases
 
     yield (nt.assert_true, len(_ip.alias_manager.alias_table) > 10)
     for key, val in _ip.alias_manager.alias_table.items():
         # we must strip dots from alias names
         nt.assert_true('.' not in key)
 
     # rehashx must fill up syscmdlist
     scoms = _ip.db['syscmdlist']
     yield (nt.assert_true, len(scoms) > 10)
 
 
 def test_magic_parse_options():
     """Test that we don't mangle paths when parsing magic options."""
     ip = get_ipython()
     path = 'c:\\x'
     opts = ip.parse_options('-f %s' % path,'f:')[0]
     # argv splitting is os-dependent
     if os.name == 'posix':
         expected = 'c:x'
     else:
         expected = path
     nt.assert_equals(opts['f'], expected)
 
     
 def doctest_hist_f():
     """Test %hist -f with temporary filename.
 
     In [9]: import tempfile
 
     In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
 
     In [11]: %hist -n -f $tfile 3
 
     In [13]: import os; os.unlink(tfile)
     """
 
 
 def doctest_hist_r():
     """Test %hist -r
 
     XXX - This test is not recording the output correctly.  For some reason, in
     testing mode the raw history isn't getting populated.  No idea why.
     Disabling the output checking for now, though at least we do run it.
 
     In [1]: 'hist' in _ip.lsmagic()
     Out[1]: True
 
     In [2]: x=1
 
     In [3]: %hist -r 2
     x=1 # random
     %hist -r 2
     """
 
 def doctest_hist_op():
     """Test %hist -op
 
     In [1]: class b:
        ...:         pass
        ...: 
 
     In [2]: class s(b):
        ...:         def __str__(self):
        ...:             return 's'
        ...: 
 
     In [3]: 
 
     In [4]: class r(b):
        ...:         def __repr__(self):
        ...:             return 'r'
        ...: 
 
     In [5]: class sr(s,r): pass
        ...: 
 
     In [6]: 
 
     In [7]: bb=b()
 
     In [8]: ss=s()
 
     In [9]: rr=r()
 
     In [10]: ssrr=sr()
 
     In [11]: bb
     Out[11]: <...b instance at ...>
 
     In [12]: ss
     Out[12]: <...s instance at ...>
 
     In [13]: 
 
     In [14]: %hist -op
     >>> class b:
     ...     pass
     ... 
     >>> class s(b):
     ...     def __str__(self):
     ...         return 's'
     ... 
     >>> 
     >>> class r(b):
     ...     def __repr__(self):
     ...         return 'r'
     ... 
     >>> class sr(s,r): pass
     >>> 
     >>> bb=b()
     >>> ss=s()
     >>> rr=r()
     >>> ssrr=sr()
     >>> bb
     <...b instance at ...>
     >>> ss
     <...s instance at ...>
     >>> 
     """
 
 def test_shist():
     # Simple tests of ShadowHist class - test generator.
     import os, shutil, tempfile
 
     from IPython.utils import pickleshare
     from IPython.core.history import ShadowHist
     
     tfile = tempfile.mktemp('','tmp-ipython-')
     
     db = pickleshare.PickleShareDB(tfile)
     s = ShadowHist(db)
     s.add('hello')
     s.add('world')
     s.add('hello')
     s.add('hello')
     s.add('karhu')
 
     yield nt.assert_equals,s.all(),[(1, 'hello'), (2, 'world'), (3, 'karhu')]
     
     yield nt.assert_equal,s.get(2),'world'
     
     shutil.rmtree(tfile)
 
     
 # XXX failing for now, until we get clearcmd out of quarantine.  But we should
 # fix this and revert the skip to happen only if numpy is not around.
 #@dec.skipif_not_numpy
 @dec.skipknownfailure
 def test_numpy_clear_array_undec():
     from IPython.extensions import clearcmd
 
     _ip.ex('import numpy as np')
     _ip.ex('a = np.empty(2)')
     yield (nt.assert_true, 'a' in _ip.user_ns)
     _ip.magic('clear array')
     yield (nt.assert_false, 'a' in _ip.user_ns)
     
 
 # Multiple tests for clipboard pasting
 @dec.parametric
 def test_paste():
     _ip = get_ipython()
     def paste(txt, flags='-q'):
         """Paste input text, by default in quiet mode"""
         hooks.clipboard_get = lambda : txt
         _ip.magic('paste '+flags)
 
     # Inject fake clipboard hook but save original so we can restore it later
     hooks = _ip.hooks
     user_ns = _ip.user_ns
     original_clip = hooks.clipboard_get
 
     try:
         # This try/except with an emtpy except clause is here only because
         # try/yield/finally is invalid syntax in Python 2.4.  This will be
         # removed when we drop 2.4-compatibility, and the emtpy except below
         # will be changed to a finally.
 
         # Run tests with fake clipboard function
         user_ns.pop('x', None)
         paste('x=1')
         yield nt.assert_equal(user_ns['x'], 1)
 
         user_ns.pop('x', None)
         paste('>>> x=2')
         yield nt.assert_equal(user_ns['x'], 2)
 
         paste("""
         >>> x = [1,2,3]
         >>> y = []
         >>> for i in x:
         ...     y.append(i**2)
         ...
         """)
         yield nt.assert_equal(user_ns['x'], [1,2,3])
         yield nt.assert_equal(user_ns['y'], [1,4,9])
 
         # Now, test that paste -r works
         user_ns.pop('x', None)
         yield nt.assert_false('x' in user_ns)
         _ip.magic('paste -r')
         yield nt.assert_equal(user_ns['x'], [1,2,3])
 
         # Also test paste echoing, by temporarily faking the writer
         w = StringIO()
         writer = _ip.write
         _ip.write = w.write
         code = """
         a = 100
         b = 200"""
         try:
             paste(code,'')
             out = w.getvalue()
         finally:
             _ip.write = writer
         yield nt.assert_equal(user_ns['a'], 100)
         yield nt.assert_equal(user_ns['b'], 200)
         yield nt.assert_equal(out, code+"\n## -- End pasted text --\n")
         
     finally:
         # This should be in a finally clause, instead of the bare except above.
         # Restore original hook
         hooks.clipboard_get = original_clip
 
 
 def test_time():
     _ip.magic('time None')
 
 
 def doctest_time():
     """
     In [10]: %time None
     CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
     Wall time: 0.00 s
     """
 
+
 def test_doctest_mode():
     "Toggle doctest_mode twice, it should be a no-op and run without error"
     _ip.magic('doctest_mode')
     _ip.magic('doctest_mode')
+
+
+def test_parse_options():
+    """Tests for basic options parsing in magics."""
+    # These are only the most minimal of tests, more should be added later.  At
+    # the very least we check that basic text/unicode calls work OK.
+    nt.assert_equal(_ip.parse_options('foo', '')[1], 'foo')
+    nt.assert_equal(_ip.parse_options(u'foo', '')[1], u'foo')
+
+    
+def test_dirops():
+    """Test various directory handling operations."""
+    curpath = lambda :os.path.splitdrive(os.getcwd())[1].replace('\\','/')
+
+    startdir = os.getcwd()
+    ipdir = _ip.ipython_dir
+    try:
+        _ip.magic('cd "%s"' % ipdir)
+        nt.assert_equal(curpath(), ipdir)
+        _ip.magic('cd -')
+        nt.assert_equal(curpath(), startdir)
+        _ip.magic('pushd "%s"' % ipdir)
+        nt.assert_equal(curpath(), ipdir)
+        _ip.magic('popd')
+        nt.assert_equal(curpath(), startdir)
+    finally:
+        os.chdir(startdir)
+
+
+def check_cpaste(code, should_fail=False):
+    """Execute code via 'cpaste' and ensure it was executed, unless
+    should_fail is set.
+    """
+    _ip.user_ns['code_ran'] = False
+
+    src = StringIO()
+    src.write('\n')
+    src.write(code)
+    src.write('\n--\n')
+    src.seek(0)
+
+    stdin_save = sys.stdin
+    sys.stdin = src
     
+    try:
+        _ip.magic('cpaste')
+    except:
+        if not should_fail:
+            raise AssertionError("Failure not expected : '%s'" %
+                                 code)
+    else:
+        assert _ip.user_ns['code_ran']
+        if should_fail:
+            raise AssertionError("Failure expected : '%s'" % code)
+    finally:
+        sys.stdin = stdin_save
+
+
+def test_cpaste():
+    """Test cpaste magic"""
+
+    def run():
+        """Marker function: sets a flag when executed.
+        """
+        _ip.user_ns['code_ran'] = True
+        return 'run' # return string so '+ run()' doesn't result in success
+
+    tests = {'pass': ["> > > run()",
+                      ">>> > run()",
+                      "+++ run()",
+                      "++ run()",
+                      "  >>> run()"],
+
+             'fail': ["+ + run()",
+                      " ++ run()"]}
+
+    _ip.user_ns['run'] = run
+
+    for code in tests['pass']:
+        check_cpaste(code)
+
+    for code in tests['fail']:
+        check_cpaste(code, should_fail=True)

IPython/core/tests/test_prefilter.py

@@ -1,59 +1,68 b''
 """Tests for input manipulation machinery."""
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 import nose.tools as nt
 
 from IPython.testing import tools as tt, decorators as dec
 from IPython.testing.globalipapp import get_ipython
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 ip = get_ipython()
 
 @dec.parametric
 def test_prefilter():
     """Test user input conversions"""
 
     # pairs of (raw, expected correct) input
     pairs = [ ('2+2','2+2'),
               ('>>> 2+2','2+2'),
               ('>>> # This is a comment\n'
                '... 2+2',
                '# This is a comment\n'
                '2+2'),
               # Some IPython input
               ('In [1]: 1', '1'),
               ('In [2]: for i in range(5):\n'
                '   ...:     print i,',
                'for i in range(5):\n'
                '    print i,'),
              ]
 
     for raw, correct in pairs:
         yield nt.assert_equals(ip.prefilter(raw), correct)
 
 @dec.parametric
 def test_autocall_binops():
     """See https://bugs.launchpad.net/ipython/+bug/315706"""
     ip.magic('autocall 2')
     f = lambda x: x
     ip.user_ns['f'] = f
     try:
         yield nt.assert_equals(ip.prefilter('f 1'),'f(1)')
         for t in ['f +1', 'f -1']:
             yield nt.assert_equals(ip.prefilter(t), t)
     finally:
         ip.magic('autocall 0')
         del ip.user_ns['f']
 
 @dec.parametric
 def test_issue114():
     """Check that multiline string literals don't expand as magic
     see http://github.com/ipython/ipython/issues/#issue/114"""
+
     template = '"""\n%s\n"""'
-    for mgk in ip.lsmagic():
-        raw = template % mgk
-        yield nt.assert_equals(ip.prefilter(raw), raw)
+    # Store the current value of multi_line_specials and turn it off before
+    # running test, since it could be true (case in which the test doesn't make
+    # sense, as multiline string literals *will* expand as magic in that case).
+    msp = ip.prefilter_manager.multi_line_specials
+    ip.prefilter_manager.multi_line_specials = False
+    try:
+        for mgk in ip.lsmagic():
+            raw = template % mgk
+            yield nt.assert_equals(ip.prefilter(raw), raw)
+    finally:
+        ip.prefilter_manager.multi_line_specials = msp

IPython/lib/irunner.py

@@ -1,441 +1,441 b''
 #!/usr/bin/env python
 """Module for interactively running scripts.
 
 This module implements classes for interactively running scripts written for
 any system with a prompt which can be matched by a regexp suitable for
 pexpect.  It can be used to run as if they had been typed up interactively, an
 arbitrary series of commands for the target system.
 
 The module includes classes ready for IPython (with the default prompts),
 plain Python and SAGE, but making a new one is trivial.  To see how to use it,
 simply run the module as a script:
 
 ./irunner.py --help
 
 
 This is an extension of Ken Schutte <kschutte-AT-csail.mit.edu>'s script
 contributed on the ipython-user list:
 
 http://scipy.net/pipermail/ipython-user/2006-May/001705.html
 
 
 NOTES:
 
  - This module requires pexpect, available in most linux distros, or which can
  be downloaded from
 
     http://pexpect.sourceforge.net
 
  - Because pexpect only works under Unix or Windows-Cygwin, this has the same
  limitations.  This means that it will NOT work under native windows Python.
 """
 
 # Stdlib imports
 import optparse
 import os
 import sys
 
 # Third-party modules.
 import pexpect
 
 # Global usage strings, to avoid indentation issues when typing it below.
 USAGE = """
 Interactive script runner, type: %s
 
 runner [opts] script_name
 """
 
 def pexpect_monkeypatch():
     """Patch pexpect to prevent unhandled exceptions at VM teardown.
 
     Calling this function will monkeypatch the pexpect.spawn class and modify
     its __del__ method to make it more robust in the face of failures that can
     occur if it is called when the Python VM is shutting down.
 
     Since Python may fire __del__ methods arbitrarily late, it's possible for
     them to execute during the teardown of the Python VM itself.  At this
     point, various builtin modules have been reset to None.  Thus, the call to
     self.close() will trigger an exception because it tries to call os.close(),
     and os is now None.
     """
     
     if pexpect.__version__[:3] >= '2.2':
         # No need to patch, fix is already the upstream version.
         return
     
     def __del__(self):
         """This makes sure that no system resources are left open.
         Python only garbage collects Python objects. OS file descriptors
         are not Python objects, so they must be handled explicitly.
         If the child file descriptor was opened outside of this class
         (passed to the constructor) then this does not close it.
         """
         if not self.closed:
             try:
                 self.close()
             except AttributeError:
                 pass
 
     pexpect.spawn.__del__ = __del__
 
 pexpect_monkeypatch()
 
 # The generic runner class
 class InteractiveRunner(object):
     """Class to run a sequence of commands through an interactive program."""
     
     def __init__(self,program,prompts,args=None,out=sys.stdout,echo=True):
         """Construct a runner.
 
         Inputs:
 
           - program: command to execute the given program.
 
           - prompts: a list of patterns to match as valid prompts, in the
           format used by pexpect.  This basically means that it can be either
           a string (to be compiled as a regular expression) or a list of such
           (it must be a true list, as pexpect does type checks).
 
           If more than one prompt is given, the first is treated as the main
           program prompt and the others as 'continuation' prompts, like
           python's.  This means that blank lines in the input source are
           ommitted when the first prompt is matched, but are NOT ommitted when
           the continuation one matches, since this is how python signals the
           end of multiline input interactively.
 
         Optional inputs:
 
           - args(None): optional list of strings to pass as arguments to the
           child program.
 
           - out(sys.stdout): if given, an output stream to be used when writing
           output.  The only requirement is that it must have a .write() method.
 
         Public members not parameterized in the constructor:
 
           - delaybeforesend(0): Newer versions of pexpect have a delay before
           sending each new input.  For our purposes here, it's typically best
           to just set this to zero, but if you encounter reliability problems
           or want an interactive run to pause briefly at each prompt, just
           increase this value (it is measured in seconds).  Note that this
           variable is not honored at all by older versions of pexpect.
         """
         
         self.program = program
         self.prompts = prompts
         if args is None: args = []
         self.args = args
         self.out = out
         self.echo = echo
         # Other public members which we don't make as parameters, but which
         # users may occasionally want to tweak
         self.delaybeforesend = 0
 
         # Create child process and hold on to it so we don't have to re-create
         # for every single execution call
         c = self.child = pexpect.spawn(self.program,self.args,timeout=None)
         c.delaybeforesend = self.delaybeforesend
         # pexpect hard-codes the terminal size as (24,80) (rows,columns).
         # This causes problems because any line longer than 80 characters gets
         # completely overwrapped on the printed outptut (even though
         # internally the code runs fine).  We reset this to 99 rows X 200
         # columns (arbitrarily chosen), which should avoid problems in all
         # reasonable cases.
         c.setwinsize(99,200)
 
     def close(self):
         """close child process"""
 
         self.child.close()
 
     def run_file(self,fname,interact=False,get_output=False):
         """Run the given file interactively.
 
         Inputs:
 
           -fname: name of the file to execute.
 
         See the run_source docstring for the meaning of the optional
         arguments."""
 
         fobj = open(fname,'r')
         try:
             out = self.run_source(fobj,interact,get_output)
         finally:
             fobj.close()
         if get_output:
             return out
 
     def run_source(self,source,interact=False,get_output=False):
         """Run the given source code interactively.
 
         Inputs:
 
           - source: a string of code to be executed, or an open file object we
           can iterate over.
 
         Optional inputs:
 
           - interact(False): if true, start to interact with the running
           program at the end of the script.  Otherwise, just exit.
 
           - get_output(False): if true, capture the output of the child process
           (filtering the input commands out) and return it as a string.
 
         Returns:
           A string containing the process output, but only if requested.
           """
 
         # if the source is a string, chop it up in lines so we can iterate
         # over it just as if it were an open file.
         if not isinstance(source,file):
             source = source.splitlines(True)
 
         if self.echo:
             # normalize all strings we write to use the native OS line
             # separators.
             linesep  = os.linesep
             stdwrite = self.out.write
             write    = lambda s: stdwrite(s.replace('\r\n',linesep))
         else:
             # Quiet mode, all writes are no-ops
             write = lambda s: None
             
         c = self.child
         prompts = c.compile_pattern_list(self.prompts)
         prompt_idx = c.expect_list(prompts)
 
         # Flag whether the script ends normally or not, to know whether we can
         # do anything further with the underlying process.
         end_normal = True
 
         # If the output was requested, store it in a list for return at the end
         if get_output:
             output = []
             store_output = output.append
         
         for cmd in source:
             # skip blank lines for all matches to the 'main' prompt, while the
             # secondary prompts do not
             if prompt_idx==0 and \
                    (cmd.isspace() or cmd.lstrip().startswith('#')):
                 write(cmd)
                 continue
 
             # write('AFTER: '+c.after)  # dbg
             write(c.after)
             c.send(cmd)
             try:
                 prompt_idx = c.expect_list(prompts)
             except pexpect.EOF:
                 # this will happen if the child dies unexpectedly
                 write(c.before)
                 end_normal = False
                 break
             
             write(c.before)
 
             # With an echoing process, the output we get in c.before contains
             # the command sent, a newline, and then the actual process output
             if get_output:
                 store_output(c.before[len(cmd+'\n'):])
                 #write('CMD: <<%s>>' % cmd)  # dbg
                 #write('OUTPUT: <<%s>>' % output[-1])  # dbg
 
         self.out.flush()
         if end_normal:
             if interact:
                 c.send('\n')
                 print '<< Starting interactive mode >>',
                 try:
                     c.interact()
                 except OSError:
                     # This is what fires when the child stops.  Simply print a
                     # newline so the system prompt is aligned.  The extra
                     # space is there to make sure it gets printed, otherwise
                     # OS buffering sometimes just suppresses it.
                     write(' \n')
                     self.out.flush()
         else:
             if interact:
                 e="Further interaction is not possible: child process is dead."
                 print >> sys.stderr, e
 
         # Leave the child ready for more input later on, otherwise select just
         # hangs on the second invocation.
         c.send('\n')
         
         # Return any requested output
         if get_output:
             return ''.join(output)
                 
     def main(self,argv=None):
         """Run as a command-line script."""
 
         parser = optparse.OptionParser(usage=USAGE % self.__class__.__name__)
         newopt = parser.add_option
         newopt('-i','--interact',action='store_true',default=False,
                help='Interact with the program after the script is run.')
 
         opts,args = parser.parse_args(argv)
 
         if len(args) != 1:
             print >> sys.stderr,"You must supply exactly one file to run."
             sys.exit(1)
 
         self.run_file(args[0],opts.interact)
 
 
 # Specific runners for particular programs
 class IPythonRunner(InteractiveRunner):
     """Interactive IPython runner.
 
     This initalizes IPython in 'nocolor' mode for simplicity.  This lets us
     avoid having to write a regexp that matches ANSI sequences, though pexpect
     does support them.  If anyone contributes patches for ANSI color support,
     they will be welcome.
 
     It also sets the prompts manually, since the prompt regexps for
     pexpect need to be matched to the actual prompts, so user-customized
     prompts would break this.
     """
     
     def __init__(self,program = 'ipython',args=None,out=sys.stdout,echo=True):
         """New runner, optionally passing the ipython command to use."""
         
         args0 = ['--colors','NoColor',
                  '-pi1','In [\\#]: ',
                  '-pi2','   .\\D.: ',
-                 '--noterm-title',
-                 '--no-auto-indent']
+                 '--no-term-title',
+                 '--no-autoindent']
         if args is None: args = args0
         else: args = args0 + args
         prompts = [r'In \[\d+\]: ',r'   \.*: ']
         InteractiveRunner.__init__(self,program,prompts,args,out,echo)
 
 
 class PythonRunner(InteractiveRunner):
     """Interactive Python runner."""
 
     def __init__(self,program='python',args=None,out=sys.stdout,echo=True):
         """New runner, optionally passing the python command to use."""
 
         prompts = [r'>>> ',r'\.\.\. ']
         InteractiveRunner.__init__(self,program,prompts,args,out,echo)
 
 
 class SAGERunner(InteractiveRunner):
     """Interactive SAGE runner.
     
     WARNING: this runner only works if you manually configure your SAGE copy
     to use 'colors NoColor' in the ipythonrc config file, since currently the
     prompt matching regexp does not identify color sequences."""
 
     def __init__(self,program='sage',args=None,out=sys.stdout,echo=True):
         """New runner, optionally passing the sage command to use."""
 
         prompts = ['sage: ',r'\s*\.\.\. ']
         InteractiveRunner.__init__(self,program,prompts,args,out,echo)
 
 
 class RunnerFactory(object):
     """Code runner factory.
 
     This class provides an IPython code runner, but enforces that only one
     runner is ever instantiated.  The runner is created based on the extension
     of the first file to run, and it raises an exception if a runner is later
     requested for a different extension type.
 
     This ensures that we don't generate example files for doctest with a mix of
     python and ipython syntax.
     """
 
     def __init__(self,out=sys.stdout):
         """Instantiate a code runner."""
         
         self.out = out
         self.runner = None
         self.runnerClass = None
 
     def _makeRunner(self,runnerClass):
         self.runnerClass = runnerClass
         self.runner = runnerClass(out=self.out)
         return self.runner
           
     def __call__(self,fname):
         """Return a runner for the given filename."""
 
         if fname.endswith('.py'):
             runnerClass = PythonRunner
         elif fname.endswith('.ipy'):
             runnerClass = IPythonRunner
         else:
             raise ValueError('Unknown file type for Runner: %r' % fname)
 
         if self.runner is None:
             return self._makeRunner(runnerClass)
         else:
             if runnerClass==self.runnerClass:
                 return self.runner
             else:
                 e='A runner of type %r can not run file %r' % \
                    (self.runnerClass,fname)
                 raise ValueError(e)
 
 
 # Global usage string, to avoid indentation issues if typed in a function def.
 MAIN_USAGE = """
 %prog [options] file_to_run
 
 This is an interface to the various interactive runners available in this
 module.  If you want to pass specific options to one of the runners, you need
 to first terminate the main options with a '--', and then provide the runner's
 options.  For example:
 
 irunner.py --python -- --help
 
 will pass --help to the python runner.  Similarly,
 
 irunner.py --ipython -- --interact script.ipy
 
 will run the script.ipy file under the IPython runner, and then will start to
 interact with IPython at the end of the script (instead of exiting).
 
 The already implemented runners are listed below; adding one for a new program
 is a trivial task, see the source for examples.
 
 WARNING: the SAGE runner only works if you manually configure your SAGE copy
 to use 'colors NoColor' in the ipythonrc config file, since currently the
 prompt matching regexp does not identify color sequences.
 """
 
 def main():
     """Run as a command-line script."""
 
     parser = optparse.OptionParser(usage=MAIN_USAGE)
     newopt = parser.add_option
     parser.set_defaults(mode='ipython')
     newopt('--ipython',action='store_const',dest='mode',const='ipython',
            help='IPython interactive runner (default).')
     newopt('--python',action='store_const',dest='mode',const='python',
            help='Python interactive runner.')
     newopt('--sage',action='store_const',dest='mode',const='sage',
            help='SAGE interactive runner.')
 
     opts,args = parser.parse_args()
     runners = dict(ipython=IPythonRunner,
                    python=PythonRunner,
                    sage=SAGERunner)
 
     try:
         ext = os.path.splitext(args[0])[-1]
     except IndexError:
         ext = ''
     modes = {'.ipy':'ipython',
              '.py':'python',
              '.sage':'sage'}
     mode = modes.get(ext,opts.mode)
     runners[mode]().main(args)
 
 if __name__ == '__main__':
     main()

IPython/lib/tests/test_irunner.py

@@ -1,168 +1,162 b''
-#!/usr/bin/env python
 """Test suite for the irunner module.
 
 Not the most elegant or fine-grained, but it does cover at least the bulk
 functionality."""
 
 # Global to make tests extra verbose and help debugging
 VERBOSE = True
 
 # stdlib imports
 import cStringIO as StringIO
 import sys
 import unittest
 
 # IPython imports
-from IPython import irunner
+from IPython.lib import irunner
 
 # Testing code begins
 class RunnerTestCase(unittest.TestCase):
 
     def setUp(self):
         self.out = StringIO.StringIO()
         #self.out = sys.stdout
 
     def _test_runner(self,runner,source,output):
         """Test that a given runner's input/output match."""
         
         runner.run_source(source)
         out = self.out.getvalue()
         #out = ''
         # this output contains nasty \r\n lineends, and the initial ipython
-        # banner.  clean it up for comparison
-        output_l = output.split()
-        out_l = out.split()
+        # banner.  clean it up for comparison, removing lines of whitespace
+        output_l = [l for l in output.splitlines() if l and not l.isspace()]
+        out_l = [l for l in out.splitlines() if l and not l.isspace()]
         mismatch  = 0
-        #if len(output_l) != len(out_l):
-        #    self.fail('mismatch in number of lines')
+        if len(output_l) != len(out_l):
+            self.fail('mismatch in number of lines')
         for n in range(len(output_l)):
             # Do a line-by-line comparison
             ol1 = output_l[n].strip()
             ol2 = out_l[n].strip()
             if ol1 != ol2:
                 mismatch += 1
                 if VERBOSE:
                     print '<<< line %s does not match:' % n
                     print repr(ol1)
                     print repr(ol2)
                     print '>>>'
         self.assert_(mismatch==0,'Number of mismatched lines: %s' %
                      mismatch)
 
     def testIPython(self):
         """Test the IPython runner."""
         source = """
 print 'hello, this is python'
-
 # some more code
 x=1;y=2
 x+y**2
 
 # An example of autocall functionality
 from math import *
 autocall 1
 cos pi
 autocall 0
 cos pi
 cos(pi)
 
 for i in range(5):
     print i,
 
 print "that's all folks!"
 
 %Exit
 """
         output = """\
 In [1]: print 'hello, this is python'
 hello, this is python
 
 
 # some more code
 In [2]: x=1;y=2
 
 In [3]: x+y**2
 Out[3]: 5
 
 
 # An example of autocall functionality
 In [4]: from math import *
 
 In [5]: autocall 1
 Automatic calling is: Smart
 
 In [6]: cos pi
 ------> cos(pi)
 Out[6]: -1.0
 
 In [7]: autocall 0
 Automatic calling is: OFF
 
 In [8]: cos pi
-------------------------------------------------------------
    File "<ipython console>", line 1
      cos pi
           ^
-<type 'exceptions.SyntaxError'>: invalid syntax
+SyntaxError: invalid syntax
 
 
 In [9]: cos(pi)
 Out[9]: -1.0
 
 
 In [10]: for i in range(5):
    ....:     print i,
    ....:
 0 1 2 3 4
 
 In [11]: print "that's all folks!"
 that's all folks!
 
 
 In [12]: %Exit
 """
         runner = irunner.IPythonRunner(out=self.out)
         self._test_runner(runner,source,output)
 
     def testPython(self):
         """Test the Python runner."""
         runner = irunner.PythonRunner(out=self.out)
         source = """
 print 'hello, this is python'
 
 # some more code
 x=1;y=2
 x+y**2
 
 from math import *
 cos(pi)
 
 for i in range(5):
     print i,
 
 print "that's all folks!"
         """
         output = """\
 >>> print 'hello, this is python'
 hello, this is python
 
 # some more code
 >>> x=1;y=2
 >>> x+y**2
 5
 
 >>> from math import *
 >>> cos(pi)
 -1.0
 
 >>> for i in range(5):
 ...     print i,
 ...
 0 1 2 3 4
 >>> print "that's all folks!"
 that's all folks!
 """
         self._test_runner(runner,source,output)
-        
-if __name__ == '__main__':
-    unittest.main()

IPython/testing/iptest.py

@@ -1,467 +1,446 b''
 # -*- coding: utf-8 -*-
 """IPython Test Suite Runner.
 
 This module provides a main entry point to a user script to test IPython
 itself from the command line. There are two ways of running this script:
 
 1. With the syntax `iptest all`.  This runs our entire test suite by
    calling this script (with different arguments) or trial recursively.  This
    causes modules and package to be tested in different processes, using nose
    or trial where appropriate.
 2. With the regular nose syntax, like `iptest -vvs IPython`.  In this form
    the script simply calls nose, but with special command line flags and
    plugins loaded.
 
 For now, this script requires that both nose and twisted are installed.  This
 will change in the future.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib
 import os
 import os.path as path
 import signal
 import sys
 import subprocess
 import tempfile
 import time
 import warnings
 
 # Note: monkeypatch!
 # We need to monkeypatch a small problem in nose itself first, before importing
 # it for actual use.  This should get into nose upstream, but its release cycle
 # is slow and we need it for our parametric tests to work correctly.
 from IPython.testing import nosepatch
 # Now, proceed to import nose itself
 import nose.plugins.builtin
 from nose.core import TestProgram
 
 # Our own imports
 from IPython.utils.path import get_ipython_module_path
 from IPython.utils.process import find_cmd, pycmd2argv
 from IPython.utils.sysinfo import sys_info
 
 from IPython.testing import globalipapp
 from IPython.testing.plugin.ipdoctest import IPythonDoctest
 
 pjoin = path.join
 
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 
 #-----------------------------------------------------------------------------
 # Warnings control
 #-----------------------------------------------------------------------------
 
 # Twisted generates annoying warnings with Python 2.6, as will do other code
 # that imports 'sets' as of today
 warnings.filterwarnings('ignore', 'the sets module is deprecated',
                         DeprecationWarning )
 
 # This one also comes from Twisted
 warnings.filterwarnings('ignore', 'the sha module is deprecated',
                         DeprecationWarning)
 
 # Wx on Fedora11 spits these out
 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
                         UserWarning)
 
 #-----------------------------------------------------------------------------
 # Logic for skipping doctests
 #-----------------------------------------------------------------------------
 
 def test_for(mod):
     """Test to see if mod is importable."""
     try:
         __import__(mod)
     except (ImportError, RuntimeError):
         # GTK reports Runtime error if it can't be initialized even if  it's
         # importable.
         return False
     else:
         return True
 
 # Global dict where we can store information on what we have and what we don't
 # have available at test run time
 have = {}
 
 have['curses'] = test_for('_curses')
 have['wx'] = test_for('wx')
 have['wx.aui'] = test_for('wx.aui')
 have['zope.interface'] = test_for('zope.interface')
 have['twisted'] = test_for('twisted')
 have['foolscap'] = test_for('foolscap')
-have['objc'] = test_for('objc')
 have['pexpect'] = test_for('pexpect')
 have['gtk'] = test_for('gtk')
 have['gobject'] = test_for('gobject')
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 def report():
     """Return a string with a summary report of test-related variables."""
 
     out = [ sys_info() ]
 
     avail = []
     not_avail = []
     
     for k, is_avail in have.items():
         if is_avail:
             avail.append(k)
         else:
             not_avail.append(k)
 
     if avail:
         out.append('\nTools and libraries available at test time:\n')
         avail.sort()
         out.append('   ' + ' '.join(avail)+'\n')
 
     if not_avail:
         out.append('\nTools and libraries NOT available at test time:\n')
         not_avail.sort()
         out.append('   ' + ' '.join(not_avail)+'\n')
         
     return ''.join(out)
 
 
 def make_exclude():
     """Make patterns of modules and packages to exclude from testing.
 
     For the IPythonDoctest plugin, we need to exclude certain patterns that
     cause testing problems.  We should strive to minimize the number of
     skipped modules, since this means untested code.
 
     These modules and packages will NOT get scanned by nose at all for tests.
     """
     # Simple utility to make IPython paths more readably, we need a lot of
     # these below
     ipjoin = lambda *paths: pjoin('IPython', *paths)
     
     exclusions = [ipjoin('external'),
-                  ipjoin('frontend', 'process', 'winprocess.py'),
                   # Deprecated old Shell and iplib modules, skip to avoid
                   # warnings
                   ipjoin('Shell'),
                   ipjoin('iplib'),
                   pjoin('IPython_doctest_plugin'),
                   ipjoin('quarantine'),
                   ipjoin('deathrow'),
                   ipjoin('testing', 'attic'),
                   # This guy is probably attic material
                   ipjoin('testing', 'mkdoctests'),
                   # Testing inputhook will need a lot of thought, to figure out
                   # how to have tests that don't lock up with the gui event
                   # loops in the picture
                   ipjoin('lib', 'inputhook'),
                   # Config files aren't really importable stand-alone
                   ipjoin('config', 'default'),
                   ipjoin('config', 'profile'),
                   ]
 
     if not have['wx']:
-        exclusions.append(ipjoin('gui'))
-        exclusions.append(ipjoin('frontend', 'wx'))
         exclusions.append(ipjoin('lib', 'inputhookwx'))
 
     if not have['gtk'] or not have['gobject']:
         exclusions.append(ipjoin('lib', 'inputhookgtk'))
 
-    if not have['wx.aui']:
-        exclusions.append(ipjoin('gui', 'wx', 'wxIPython'))
-
-    if not have['objc']:
-        exclusions.append(ipjoin('frontend', 'cocoa'))
-
     # These have to be skipped on win32 because the use echo, rm, cd, etc.
     # See ticket https://bugs.launchpad.net/bugs/366982
     if sys.platform == 'win32':
         exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
         exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
 
     if not have['pexpect']:
         exclusions.extend([ipjoin('scripts', 'irunner'),
                            ipjoin('lib', 'irunner')])
 
     # This is scary.  We still have things in frontend and testing that
     # are being tested by nose that use twisted.  We need to rethink
     # how we are isolating dependencies in testing.
     if not (have['twisted'] and have['zope.interface'] and have['foolscap']):
         exclusions.extend(
-            [ipjoin('frontend', 'asyncfrontendbase'),
-             ipjoin('frontend', 'prefilterfrontend'),
-             ipjoin('frontend', 'frontendbase'),
-             ipjoin('frontend', 'linefrontendbase'),
-             ipjoin('frontend', 'tests', 'test_linefrontend'),
-             ipjoin('frontend', 'tests', 'test_frontendbase'),
-             ipjoin('frontend', 'tests', 'test_prefilterfrontend'),
-             ipjoin('frontend', 'tests', 'test_asyncfrontendbase'),
-             ipjoin('testing', 'parametric'),
+            [ipjoin('testing', 'parametric'),
              ipjoin('testing', 'util'),
              ipjoin('testing', 'tests', 'test_decorators_trial'),
              ] )
 
     # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
     if sys.platform == 'win32':
         exclusions = [s.replace('\\','\\\\') for s in exclusions]
 
     return exclusions
 
 
 class IPTester(object):
     """Call that calls iptest or trial in a subprocess.
     """
     #: string, name of test runner that will be called
     runner = None
     #: list, parameters for test runner
     params = None
     #: list, arguments of system call to be made to call test runner
     call_args = None
     #: list, process ids of subprocesses we start (for cleanup)
     pids = None
     
     def __init__(self, runner='iptest', params=None):
         """Create new test runner."""
         p = os.path
         if runner == 'iptest':
             iptest_app = get_ipython_module_path('IPython.testing.iptest')
             self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
         elif runner == 'trial':
             # For trial, it needs to be installed system-wide
             self.runner = pycmd2argv(p.abspath(find_cmd('trial')))
         else:
             raise Exception('Not a valid test runner: %s' % repr(runner))
         if params is None:
             params = []
         if isinstance(params, str):
             params = [params]
         self.params = params
 
         # Assemble call
         self.call_args = self.runner+self.params
 
         # Store pids of anything we start to clean up on deletion, if possible
         # (on posix only, since win32 has no os.kill)
         self.pids = []
 
     if sys.platform == 'win32':
         def _run_cmd(self):
             # On Windows, use os.system instead of subprocess.call, because I
             # was having problems with subprocess and I just don't know enough
             # about win32 to debug this reliably.  Os.system may be the 'old
             # fashioned' way to do it, but it works just fine.  If someone
             # later can clean this up that's fine, as long as the tests run
             # reliably in win32.
             # What types of problems are you having. They may be related to
             # running Python in unboffered mode. BG.
             return os.system(' '.join(self.call_args))
     else:
         def _run_cmd(self):
             #print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
             subp = subprocess.Popen(self.call_args)
             self.pids.append(subp.pid)
             # If this fails, the pid will be left in self.pids and cleaned up
             # later, but if the wait call succeeds, then we can clear the
             # stored pid.
             retcode = subp.wait()
             self.pids.pop()
             return retcode
         
     def run(self):
         """Run the stored commands"""
         try:
             return self._run_cmd()
         except:
             import traceback
             traceback.print_exc()
             return 1  # signal failure
 
     def __del__(self):
         """Cleanup on exit by killing any leftover processes."""
 
         if not hasattr(os, 'kill'):
             return
         
         for pid in self.pids:
             try:
                 print 'Cleaning stale PID:', pid
                 os.kill(pid, signal.SIGKILL)
             except OSError:
                 # This is just a best effort, if we fail or the process was
                 # really gone, ignore it.
                 pass     
 
 
 def make_runners():
     """Define the top-level packages that need to be tested.
     """
 
     # Packages to be tested via nose, that only depend on the stdlib
     nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
                      'scripts', 'testing', 'utils' ]
     # The machinery in kernel needs twisted for real testing
     trial_pkg_names = []
 
-    if have['wx']:
-        nose_pkg_names.append('gui')
-
     # And add twisted ones if conditions are met
     if have['zope.interface'] and have['twisted'] and have['foolscap']:
         # We only list IPython.kernel for testing using twisted.trial as
         # nose and twisted.trial have conflicts that make the testing system
         # unstable.
         trial_pkg_names.append('kernel')
 
     # For debugging this code, only load quick stuff
     #nose_pkg_names = ['core', 'extensions']  # dbg
     #trial_pkg_names = []  # dbg 
 
     # Make fully qualified package names prepending 'IPython.' to our name lists
     nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
     trial_packages = ['IPython.%s' % m for m in trial_pkg_names ]
 
     # Make runners
     runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
     runners.extend([ (v, IPTester('trial', params=v)) for v in trial_packages ])
     
     return runners
 
 
 def run_iptest():
     """Run the IPython test suite using nose.
     
     This function is called when this script is **not** called with the form
     `iptest all`.  It simply calls nose with appropriate command line flags
     and accepts all of the standard nose arguments.
     """
 
     warnings.filterwarnings('ignore', 
         'This will be removed soon.  Use IPython.testing.util instead')
 
     argv = sys.argv + [ '--detailed-errors',  # extra info in tracebacks
                                                 
                         # Loading ipdoctest causes problems with Twisted, but
                         # our test suite runner now separates things and runs
                         # all Twisted tests with trial.
                         '--with-ipdoctest',
                         '--ipdoctest-tests','--ipdoctest-extension=txt',
                         
                         # We add --exe because of setuptools' imbecility (it
                         # blindly does chmod +x on ALL files).  Nose does the
                         # right thing and it tries to avoid executables,
                         # setuptools unfortunately forces our hand here.  This
                         # has been discussed on the distutils list and the
                         # setuptools devs refuse to fix this problem!
                         '--exe',
                         ]
 
     if nose.__version__ >= '0.11':
         # I don't fully understand why we need this one, but depending on what
         # directory the test suite is run from, if we don't give it, 0 tests
         # get run.  Specifically, if the test suite is run from the source dir
         # with an argument (like 'iptest.py IPython.core', 0 tests are run,
         # even if the same call done in this directory works fine).  It appears
         # that if the requested package is in the current dir, nose bails early
         # by default.  Since it's otherwise harmless, leave it in by default
         # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
         argv.append('--traverse-namespace')
 
     # Construct list of plugins, omitting the existing doctest plugin, which
     # ours replaces (and extends).
     plugins = [IPythonDoctest(make_exclude())]
     for p in nose.plugins.builtin.plugins:
         plug = p()
         if plug.name == 'doctest':
             continue
         plugins.append(plug)
 
     # We need a global ipython running in this process
     globalipapp.start_ipython()
     # Now nose can run
     TestProgram(argv=argv, plugins=plugins)
 
 
 def run_iptestall():
     """Run the entire IPython test suite by calling nose and trial.
     
     This function constructs :class:`IPTester` instances for all IPython
     modules and package and then runs each of them.  This causes the modules
     and packages of IPython to be tested each in their own subprocess using
     nose or twisted.trial appropriately.
     """
 
     runners = make_runners()
 
     # Run the test runners in a temporary dir so we can nuke it when finished
     # to clean up any junk files left over by accident.  This also makes it
     # robust against being run in non-writeable directories by mistake, as the
     # temp dir will always be user-writeable.
     curdir = os.getcwd()
     testdir = tempfile.gettempdir()
     os.chdir(testdir)
 
     # Run all test runners, tracking execution time
     failed = []
     t_start = time.time()
     try:
         for (name, runner) in runners:
             print '*'*70
             print 'IPython test group:',name
             res = runner.run()
             if res:
                 failed.append( (name, runner) )
     finally:
         os.chdir(curdir)
     t_end = time.time()
     t_tests = t_end - t_start
     nrunners = len(runners)
     nfail = len(failed)
     # summarize results
     print
     print '*'*70
     print 'Test suite completed for system with the following information:'
     print report()
     print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
     print
     print 'Status:'
     if not failed:
         print 'OK'
     else:
         # If anything went wrong, point out what command to rerun manually to
         # see the actual errors and individual summary
         print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
         for name, failed_runner in failed:
             print '-'*40
             print 'Runner failed:',name
             print 'You may wish to rerun this one individually, with:'
             print ' '.join(failed_runner.call_args)
             print
 
 
 def main():
     for arg in sys.argv[1:]:
         if arg.startswith('IPython'):
             # This is in-process
             run_iptest()
     else:
         # This starts subprocesses
         run_iptestall()
 
 
 if __name__ == '__main__':
     main()

IPython/utils/process.py

@@ -1,368 +1,367 b''
 # encoding: utf-8
 """
 Utilities for working with external processes.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 import shlex
 import subprocess
 
 from IPython.utils.terminal import set_term_title
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 
 class FindCmdError(Exception):
     pass
 
 
 def _find_cmd(cmd):
     """Find the full path to a command using which."""
     return os.popen('which %s' % cmd).read().strip()
 
 
 if os.name == 'posix':
     def _find_cmd(cmd):
         """Find the full path to a command using which."""
         return getoutputerror('/usr/bin/env which %s' % cmd)[0]
 
 
 if sys.platform == 'win32':
     def _find_cmd(cmd):
         """Find the full path to a .bat or .exe using the win32api module."""
         try:
             from win32api import SearchPath
         except ImportError:
             raise ImportError('you need to have pywin32 installed for this to work')
         else:
             PATH = os.environ['PATH']
             extensions = ['.exe', '.com', '.bat', '.py']
             path = None
             for ext in extensions:
                 try:
                     path = SearchPath(PATH,cmd + ext)[0]
                 except:
                     pass
             if path is None:
                 raise OSError("command %r not found" % cmd)
             else:
                 return path
 
 
 def find_cmd(cmd):
     """Find absolute path to executable cmd in a cross platform manner.
     
     This function tries to determine the full path to a command line program
     using `which` on Unix/Linux/OS X and `win32api` on Windows.  Most of the
     time it will use the version that is first on the users `PATH`.  If
     cmd is `python` return `sys.executable`.
 
     Warning, don't use this to find IPython command line programs as there
     is a risk you will find the wrong one.  Instead find those using the
     following code and looking for the application itself::
     
         from IPython.utils.path import get_ipython_module_path
         from IPython.utils.process import pycmd2argv
         argv = pycmd2argv(get_ipython_module_path('IPython.core.ipapp'))
 
     Parameters
     ----------
     cmd : str
         The command line program to look for.
     """
     if cmd == 'python':
         return os.path.abspath(sys.executable)
     try:
         path = _find_cmd(cmd)
     except OSError:
         raise FindCmdError('command could not be found: %s' % cmd)
     # which returns empty if not found
     if path == '':
         raise FindCmdError('command could not be found: %s' % cmd)
     return os.path.abspath(path)
 
 
 def pycmd2argv(cmd):
     r"""Take the path of a python command and return a list (argv-style).
 
     This only works on Python based command line programs and will find the
     location of the ``python`` executable using ``sys.executable`` to make
     sure the right version is used.
 
     For a given path ``cmd``, this returns [cmd] if cmd's extension is .exe,
     .com or .bat, and [, cmd] otherwise.  
 
     Parameters
     ----------
     cmd : string
       The path of the command.
 
     Returns
     -------
     argv-style list.
     """
     ext = os.path.splitext(cmd)[1]
     if ext in ['.exe', '.com', '.bat']:
         return [cmd]
     else:
         if sys.platform == 'win32':
             # The -u option here turns on unbuffered output, which is required
             # on Win32 to prevent wierd conflict and problems with Twisted.
             # Also, use sys.executable to make sure we are picking up the 
             # right python exe.
             return [sys.executable, '-u', cmd]
         else:
             return [sys.executable, cmd]
 
 
 def arg_split(s, posix=False):
     """Split a command line's arguments in a shell-like manner.
 
     This is a modified version of the standard library's shlex.split()
     function, but with a default of posix=False for splitting, so that quotes
     in inputs are respected."""
 
-    # XXX - there may be unicode-related problems here!!!  I'm not sure that
-    # shlex is truly unicode-safe, so it might be necessary to do
-    #
-    # s = s.encode(sys.stdin.encoding)
-    #
-    # first, to ensure that shlex gets a normal string.  Input from anyone who
-    # knows more about unicode and shlex than I would be good to have here...
+    # Unfortunately, python's shlex module is buggy with unicode input:
+    # http://bugs.python.org/issue1170
+    # At least encoding the input when it's unicode seems to help, but there
+    # may be more problems lurking.  Apparently this is fixed in python3.
+    if isinstance(s, unicode):
+        s = s.encode(sys.stdin.encoding)
     lex = shlex.shlex(s, posix=posix)
     lex.whitespace_split = True
     return list(lex)
 
 
 def system(cmd, verbose=0, debug=0, header=''):
     """Execute a system command, return its exit status.
 
     Options:
 
     - verbose (0): print the command to be executed.
 
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     stat = 0
     if verbose or debug: print header+cmd
     sys.stdout.flush()
     if not debug: stat = os.system(cmd)
     return stat
 
 
 def abbrev_cwd():
     """ Return abbreviated version of cwd, e.g. d:mydir """
     cwd = os.getcwd().replace('\\','/')
     drivepart = ''
     tail = cwd
     if sys.platform == 'win32':
         if len(cwd) < 4:
             return cwd
         drivepart,tail = os.path.splitdrive(cwd)
 
 
     parts = tail.split('/')
     if len(parts) > 2:
         tail = '/'.join(parts[-2:])
 
     return (drivepart + (
         cwd == '/' and '/' or tail))
 
 
 # This function is used by ipython in a lot of places to make system calls.
 # We need it to be slightly different under win32, due to the vagaries of
 # 'network shares'.  A win32 override is below.
 
 def shell(cmd, verbose=0, debug=0, header=''):
     """Execute a command in the system shell, always return None.
 
     Options:
 
     - verbose (0): print the command to be executed.
 
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: this is similar to system(), but it returns None so it can
     be conveniently used in interactive loops without getting the return value
     (typically 0) printed many times."""
 
     stat = 0
     if verbose or debug: print header+cmd
     # flush stdout so we don't mangle python's buffering
     sys.stdout.flush()
 
     if not debug:
         set_term_title("IPy " + cmd)
         os.system(cmd)
         set_term_title("IPy " + abbrev_cwd())
 
 # override shell() for win32 to deal with network shares
 if os.name in ('nt','dos'):
 
     shell_ori = shell
 
     def shell(cmd, verbose=0, debug=0, header=''):
         if os.getcwd().startswith(r"\\"):
             path = os.getcwd()
             # change to c drive (cannot be on UNC-share when issuing os.system,
             # as cmd.exe cannot handle UNC addresses)
             os.chdir("c:")
             # issue pushd to the UNC-share and then run the command
             try:
                 shell_ori('"pushd %s&&"'%path+cmd,verbose,debug,header)
             finally:
                 os.chdir(path)
         else:
             shell_ori(cmd,verbose,debug,header)
 
     shell.__doc__ = shell_ori.__doc__
 
     
 def getoutput(cmd, verbose=0, debug=0, header='', split=0):
     """Dummy substitute for perl's backquotes.
 
     Executes a command and returns the output.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, the output is returned as a list split on newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class.
 
     This is pretty much deprecated and rarely used, getoutputerror may be 
     what you need.
 
     """
 
     if verbose or debug: print header+cmd
     if not debug:
         pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout
         output = pipe.read()
         # stipping last \n is here for backwards compat.
         if output.endswith('\n'):
             output = output[:-1]
         if split:
             return output.split('\n')
         else:
             return output
 
 
 # for compatibility with older naming conventions
 xsys = system
 
 
 def getoutputerror(cmd, verbose=0, debug=0, header='', split=0):
     """Return (standard output,standard error) of executing cmd in a shell.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, each of stdout/err is returned as a list split on
     newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     if verbose or debug: print header+cmd
     if not cmd:
         if split:
             return [],[]
         else:
             return '',''
     if not debug:
         p = subprocess.Popen(cmd, shell=True,
                              stdin=subprocess.PIPE,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE,
                              close_fds=True)
         pin, pout, perr = (p.stdin, p.stdout, p.stderr)
 
         tout = pout.read().rstrip()
         terr = perr.read().rstrip()
         pin.close()
         pout.close()
         perr.close()
         if split:
             return tout.split('\n'),terr.split('\n')
         else:
             return tout,terr
 
 
 class SystemExec:
     """Access the system and getoutput functions through a stateful interface.
 
     Note: here we refer to the system and getoutput functions from this
     library, not the ones from the standard python library.
 
     This class offers the system and getoutput functions as methods, but the
     verbose, debug and header parameters can be set for the instance (at
     creation time or later) so that they don't need to be specified on each
     call.
 
     For efficiency reasons, there's no way to override the parameters on a
     per-call basis other than by setting instance attributes. If you need
     local overrides, it's best to directly call system() or getoutput().
 
     The following names are provided as alternate options:
      - xsys: alias to system
      - bq: alias to getoutput
 
     An instance can then be created as:
     >>> sysexec = SystemExec(verbose=1,debug=0,header='Calling: ')
     """
 
     def __init__(self, verbose=0, debug=0, header='', split=0):
         """Specify the instance's values for verbose, debug and header."""
         self.verbose = verbose
         self.debug = debug
         self.header = header
         self.split = split
 
     def system(self, cmd):
         """Stateful interface to system(), with the same keyword parameters."""
 
         system(cmd, self.verbose, self.debug, self.header)
 
     def shell(self, cmd):
         """Stateful interface to shell(), with the same keyword parameters."""
 
         shell(cmd, self.verbose, self.debug, self.header)
 
     xsys = system  # alias
 
     def getoutput(self, cmd):
         """Stateful interface to getoutput()."""
 
         return getoutput(cmd, self.verbose, self.debug, self.header, self.split)
 
     def getoutputerror(self, cmd):
         """Stateful interface to getoutputerror()."""
 
         return getoutputerror(cmd, self.verbose, self.debug, self.header, self.split)
 
     bq = getoutput  # alias
 
 

IPython/utils/tests/test_process.py

@@ -1,62 +1,68 b''
 # encoding: utf-8
 """
 Tests for platutils.py
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import sys
 
 import nose.tools as nt
 
-from IPython.utils.process import find_cmd, FindCmdError
+from IPython.utils.process import find_cmd, FindCmdError, arg_split
 from IPython.testing import decorators as dec
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
 def test_find_cmd_python():
     """Make sure we find sys.exectable for python."""
     nt.assert_equals(find_cmd('python'), sys.executable)
 
     
 @dec.skip_win32
 def test_find_cmd_ls():
     """Make sure we can find the full path to ls."""
     path = find_cmd('ls')
     nt.assert_true(path.endswith('ls'))
 
     
 def has_pywin32():
     try:
         import win32api
     except ImportError:
         return False
     return True
 
 
 @dec.onlyif(has_pywin32, "This test requires win32api to run")
 def test_find_cmd_pythonw():
     """Try to find pythonw on Windows."""
     path = find_cmd('pythonw')
     nt.assert_true(path.endswith('pythonw.exe'))
 
 
 @dec.onlyif(lambda : sys.platform != 'win32' or has_pywin32(),
             "This test runs on posix or in win32 with win32api installed")
 def test_find_cmd_fail():
     """Make sure that FindCmdError is raised if we can't find the cmd."""
     nt.assert_raises(FindCmdError,find_cmd,'asdfasdf')
 
     
-
+def test_arg_split():
+    """Ensure that argument lines are correctly split like in a shell."""
+    tests = [['hi', ['hi']],
+             [u'hi', [u'hi']],
+             ]
+    for argstr, argv in tests:
+        nt.assert_equal(arg_split(argstr), argv)

IPython/utils/tests/test_wildcard.py

@@ -1,100 +1,117 b''
-# -*- coding: UTF-8 -*-
-import sys, unittest
-sys.path.append ('..')
+"""Some tests for the wildcard utilities."""
 
-from IPython import wildcard
+#-----------------------------------------------------------------------------
+# Library imports
+#-----------------------------------------------------------------------------
+# Stdlib
+import sys
+import unittest
+
+# Our own
+from IPython.utils import wildcard
+
+#-----------------------------------------------------------------------------
+# Globals for test
+#-----------------------------------------------------------------------------
 
 class obj_t(object):
     pass
 
-root=obj_t()
-l=["arna","abel","ABEL","active","bob","bark","abbot"]
-q=["kate","loop","arne","vito","lucifer","koppel"]
+root = obj_t()
+l = ["arna","abel","ABEL","active","bob","bark","abbot"]
+q = ["kate","loop","arne","vito","lucifer","koppel"]
 for x in l:
-    o=obj_t()
+    o = obj_t()
     setattr(root,x,o)
     for y in q:
-        p=obj_t()
+        p = obj_t()
         setattr(o,y,p)
-root._apan=obj_t()
-root._apan.a=10
-root._apan._a=20
-root._apan.__a=20
-root.__anka=obj_t()
-root.__anka.a=10
-root.__anka._a=20
-root.__anka.__a=20
+root._apan = obj_t()
+root._apan.a = 10
+root._apan._a = 20
+root._apan.__a = 20
+root.__anka = obj_t()
+root.__anka.a = 10
+root.__anka._a = 20
+root.__anka.__a = 20
+
+root._APAN = obj_t()
+root._APAN.a = 10
+root._APAN._a = 20
+root._APAN.__a = 20
+root.__ANKA = obj_t()
+root.__ANKA.a = 10
+root.__ANKA._a = 20
+root.__ANKA.__a = 20
 
-root._APAN=obj_t()
-root._APAN.a=10
-root._APAN._a=20
-root._APAN.__a=20
-root.__ANKA=obj_t()
-root.__ANKA.a=10
-root.__ANKA._a=20
-root.__ANKA.__a=20
+#-----------------------------------------------------------------------------
+# Test cases
+#-----------------------------------------------------------------------------
 
 class Tests (unittest.TestCase):
     def test_case(self):
         ns=root.__dict__
         tests=[
          ("a*",     ["abbot","abel","active","arna",]),
          ("?b*.?o*",["abbot.koppel","abbot.loop","abel.koppel","abel.loop",]),
          ("_a*",    []),
          ("_*anka", ["__anka",]),
          ("_*a*",   ["__anka",]),
         ]
         for pat,res in tests:
             res.sort()
-            a=wildcard.list_namespace(ns,"all",pat,ignore_case=False,show_all=False).keys()
+            a=wildcard.list_namespace(ns,"all",pat,ignore_case=False,
+                                      show_all=False).keys()
             a.sort()
             self.assertEqual(a,res)
 
     def test_case_showall(self):
         ns=root.__dict__
         tests=[
          ("a*",     ["abbot","abel","active","arna",]),
          ("?b*.?o*",["abbot.koppel","abbot.loop","abel.koppel","abel.loop",]),
          ("_a*",    ["_apan"]),
          ("_*anka", ["__anka",]),
          ("_*a*",   ["__anka","_apan",]),
         ]
         for pat,res in tests:
             res.sort()
-            a=wildcard.list_namespace(ns,"all",pat,ignore_case=False,show_all=True).keys()
+            a=wildcard.list_namespace(ns,"all",pat,ignore_case=False,
+                                      show_all=True).keys()
             a.sort()
             self.assertEqual(a,res)
 
 
     def test_nocase(self):
         ns=root.__dict__
         tests=[
          ("a*",     ["abbot","abel","ABEL","active","arna",]),
-         ("?b*.?o*",["abbot.koppel","abbot.loop","abel.koppel","abel.loop","ABEL.koppel","ABEL.loop",]),
+         ("?b*.?o*",["abbot.koppel","abbot.loop","abel.koppel","abel.loop",
+                     "ABEL.koppel","ABEL.loop",]),
          ("_a*",    []),
          ("_*anka", ["__anka","__ANKA",]),
          ("_*a*",   ["__anka","__ANKA",]),
         ]
         for pat,res in tests:
             res.sort()
-            a=wildcard.list_namespace(ns,"all",pat,ignore_case=True,show_all=False).keys()
+            a=wildcard.list_namespace(ns,"all",pat,ignore_case=True,
+                                      show_all=False).keys()
             a.sort()
             self.assertEqual(a,res)
 
     def test_nocase_showall(self):
         ns=root.__dict__
         tests=[
          ("a*",     ["abbot","abel","ABEL","active","arna",]),
-         ("?b*.?o*",["abbot.koppel","abbot.loop","abel.koppel","abel.loop","ABEL.koppel","ABEL.loop",]),
+         ("?b*.?o*",["abbot.koppel","abbot.loop","abel.koppel","abel.loop",
+                     "ABEL.koppel","ABEL.loop",]),
          ("_a*",    ["_apan","_APAN"]),
          ("_*anka", ["__anka","__ANKA",]),
          ("_*a*",   ["__anka","__ANKA","_apan","_APAN"]),
         ]
         for pat,res in tests:
             res.sort()
-            a=wildcard.list_namespace(ns,"all",pat,ignore_case=True,show_all=True).keys()
+            a=wildcard.list_namespace(ns,"all",pat,ignore_case=True,
+                                      show_all=True).keys()
             a.sort()
             self.assertEqual(a,res)
-            
-if __name__ == '__main__':
-    unittest.main()
  No newline at end of file

setupbase.py

@@ -1,323 +1,313 b''
 # encoding: utf-8
 
 """
 This module defines the things that are used in setup.py for building IPython
 
 This includes:
 
     * The basic arguments to setup
     * Functions for finding things like packages, package data, etc.
     * A function for checking dependencies.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import os, sys
 
 from glob import glob
 
 from setupext import install_data_ext
 
 #-------------------------------------------------------------------------------
 # Useful globals and utility functions
 #-------------------------------------------------------------------------------
 
 # A few handy globals
 isfile = os.path.isfile
 pjoin = os.path.join
 
 def oscmd(s):
     print ">", s
     os.system(s)
 
 # A little utility we'll need below, since glob() does NOT allow you to do
 # exclusion on multiple endings!
 def file_doesnt_endwith(test,endings):
     """Return true if test is a file and its name does NOT end with any
     of the strings listed in endings."""
     if not isfile(test):
         return False
     for e in endings:
         if test.endswith(e):
             return False
     return True
 
 #---------------------------------------------------------------------------
 # Basic project information
 #---------------------------------------------------------------------------
 
 # release.py contains version, authors, license, url, keywords, etc.
 execfile(pjoin('IPython','core','release.py'))
 
 # Create a dict with the basic information
 # This dict is eventually passed to setup after additional keys are added.
 setup_args = dict(
       name             = name,
       version          = version,
       description      = description,
       long_description = long_description,
       author           = author,
       author_email     = author_email,
       url              = url,
       download_url     = download_url,
       license          = license,
       platforms        = platforms,
       keywords         = keywords,
       cmdclass         = {'install_data': install_data_ext},
       )
 
 
 #---------------------------------------------------------------------------
 # Find packages
 #---------------------------------------------------------------------------
 
 def add_package(packages,pname,config=False,tests=False,scripts=False,
                 others=None):
     """
     Add a package to the list of packages, including certain subpackages.
     """
     packages.append('.'.join(['IPython',pname]))
     if config:
         packages.append('.'.join(['IPython',pname,'config']))
     if tests:
         packages.append('.'.join(['IPython',pname,'tests']))
     if scripts:
         packages.append('.'.join(['IPython',pname,'scripts']))
     if others is not None:
         for o in others:
             packages.append('.'.join(['IPython',pname,o]))
 
 def find_packages():
     """
     Find all of IPython's packages.
     """
     packages = ['IPython']
     add_package(packages, 'config', tests=True, others=['default','profile'])
     add_package(packages, 'core', tests=True)
     add_package(packages, 'deathrow', tests=True)
     add_package(packages, 'extensions')
     add_package(packages, 'external')
-    add_package(packages, 'frontend', tests=True)
-    # Don't include the cocoa frontend for now as it is not stable
-    if sys.platform == 'darwin' and False:
-        add_package(packages, 'frontend.cocoa', tests=True, others=['plugin'])
-        add_package(packages, 'frontend.cocoa.examples')
-        add_package(packages, 'frontend.cocoa.examples.IPython1Sandbox')
-        add_package(packages, 'frontend.cocoa.examples.IPython1Sandbox.English.lproj')
-    add_package(packages, 'frontend.process')
+    add_package(packages, 'frontend')
     add_package(packages, 'frontend.qt')
     add_package(packages, 'frontend.qt.console')
-    add_package(packages, 'frontend.wx')
-    add_package(packages, 'gui')
-    add_package(packages, 'gui.wx')
     add_package(packages, 'kernel', config=False, tests=True, scripts=True)
     add_package(packages, 'kernel.core', config=False, tests=True)
     add_package(packages, 'lib', tests=True)
     add_package(packages, 'quarantine', tests=True)
     add_package(packages, 'scripts')
     add_package(packages, 'testing', tests=True)
     add_package(packages, 'testing.plugin', tests=False)
     add_package(packages, 'utils', tests=True)
     add_package(packages, 'zmq')
     return packages
 
 #---------------------------------------------------------------------------
 # Find package data
 #---------------------------------------------------------------------------
 
 def find_package_data():
     """
     Find IPython's package_data.
     """
     # This is not enough for these things to appear in an sdist.
     # We need to muck with the MANIFEST to get this to work
     package_data = {
         'IPython.config.userconfig' : ['*'],
         'IPython.testing' : ['*.txt']
     }
     return package_data
 
 
 #---------------------------------------------------------------------------
 # Find data files
 #---------------------------------------------------------------------------
 
 def make_dir_struct(tag,base,out_base):
     """Make the directory structure of all files below a starting dir.
 
     This is just a convenience routine to help build a nested directory
     hierarchy because distutils is too stupid to do this by itself.
 
     XXX - this needs a proper docstring!
     """
     
     # we'll use these a lot below
     lbase = len(base)
     pathsep = os.path.sep
     lpathsep = len(pathsep)
     
     out = []
     for (dirpath,dirnames,filenames) in os.walk(base):
         # we need to strip out the dirpath from the base to map it to the
         # output (installation) path.  This requires possibly stripping the
         # path separator, because otherwise pjoin will not work correctly
         # (pjoin('foo/','/bar') returns '/bar').
         
         dp_eff = dirpath[lbase:]
         if dp_eff.startswith(pathsep):
             dp_eff = dp_eff[lpathsep:]
         # The output path must be anchored at the out_base marker    
         out_path = pjoin(out_base,dp_eff)
         # Now we can generate the final filenames. Since os.walk only produces
         # filenames, we must join back with the dirpath to get full valid file
         # paths:
         pfiles = [pjoin(dirpath,f) for f in filenames]
         # Finally, generate the entry we need, which is a triple of (tag,output
         # path, files) for use as a data_files parameter in install_data.
         out.append((tag,out_path,pfiles))
 
     return out
     
 
 def find_data_files():
     """
     Find IPython's data_files.
 
     Most of these are docs.
     """
     
     docdirbase  = pjoin('share', 'doc', 'ipython')
     manpagebase = pjoin('share', 'man', 'man1')
     
     # Simple file lists can be made by hand
     manpages  = filter(isfile, glob(pjoin('docs','man','*.1.gz')))
     igridhelpfiles = filter(isfile, glob(pjoin('IPython','extensions','igrid_help.*')))
 
     # For nested structures, use the utility above
     example_files = make_dir_struct(
         'data',
         pjoin('docs','examples'),
         pjoin(docdirbase,'examples')
     )
     manual_files = make_dir_struct(
         'data',
         pjoin('docs','dist'),
         pjoin(docdirbase,'manual')
     )
 
     # And assemble the entire output list
     data_files = [ ('data',manpagebase, manpages),
                    ('data',pjoin(docdirbase,'extensions'),igridhelpfiles),
                    ] + manual_files + example_files
                  
     ## import pprint  # dbg
     ## print '*'*80
     ## print 'data files'
     ## pprint.pprint(data_files)
     ## print '*'*80
     
     return data_files
 
 
 def make_man_update_target(manpage):
     """Return a target_update-compliant tuple for the given manpage.
 
     Parameters
     ----------
     manpage : string
       Name of the manpage, must include the section number (trailing number).
 
     Example
     -------
 
     >>> make_man_update_target('ipython.1') #doctest: +NORMALIZE_WHITESPACE
     ('docs/man/ipython.1.gz',
      ['docs/man/ipython.1'],
      'cd docs/man && gzip -9c ipython.1 > ipython.1.gz')
     """
     man_dir = pjoin('docs', 'man')
     manpage_gz = manpage + '.gz'
     manpath = pjoin(man_dir, manpage)
     manpath_gz = pjoin(man_dir, manpage_gz)
     gz_cmd = ( "cd %(man_dir)s && gzip -9c %(manpage)s > %(manpage_gz)s" %
                locals() )
     return (manpath_gz, [manpath], gz_cmd)
     
 #---------------------------------------------------------------------------
 # Find scripts
 #---------------------------------------------------------------------------
 
 def find_scripts():
     """
     Find IPython's scripts.
     """
     kernel_scripts = pjoin('IPython','kernel','scripts')
     main_scripts = pjoin('IPython','scripts')
     scripts = [pjoin(kernel_scripts, 'ipengine'),
                pjoin(kernel_scripts, 'ipcontroller'),
                pjoin(kernel_scripts, 'ipcluster'),
                pjoin(main_scripts, 'ipython'),
                pjoin(main_scripts, 'ipythonx'),
                pjoin(main_scripts, 'ipython-wx'),
                pjoin(main_scripts, 'pycolor'),
                pjoin(main_scripts, 'irunner'),
                pjoin(main_scripts, 'iptest')
               ]
     
     # Script to be run by the windows binary installer after the default setup
     # routine, to add shortcuts and similar windows-only things.  Windows
     # post-install scripts MUST reside in the scripts/ dir, otherwise distutils
     # doesn't find them.
     if 'bdist_wininst' in sys.argv:
         if len(sys.argv) > 2 and ('sdist' in sys.argv or 'bdist_rpm' in sys.argv):
             print >> sys.stderr,"ERROR: bdist_wininst must be run alone. Exiting."
             sys.exit(1)
         scripts.append(pjoin('scripts','ipython_win_post_install.py'))
     
     return scripts
 
 #---------------------------------------------------------------------------
 # Verify all dependencies
 #---------------------------------------------------------------------------
 
 def check_for_dependencies():
     """Check for IPython's dependencies.
     
     This function should NOT be called if running under setuptools!
     """
     from setupext.setupext import (
         print_line, print_raw, print_status, print_message,
         check_for_zopeinterface, check_for_twisted,
         check_for_foolscap, check_for_pyopenssl,
         check_for_sphinx, check_for_pygments,
         check_for_nose, check_for_pexpect
     )
     print_line()
     print_raw("BUILDING IPYTHON")
     print_status('python', sys.version)
     print_status('platform', sys.platform)
     if sys.platform == 'win32':
         print_status('Windows version', sys.getwindowsversion())
     
     print_raw("")
     print_raw("OPTIONAL DEPENDENCIES")
 
     check_for_zopeinterface()
     check_for_twisted()
     check_for_foolscap()
     check_for_pyopenssl()
     check_for_sphinx()
     check_for_pygments()
     check_for_nose()
     check_for_pexpect()