Merge 'XDG', introducing basic XDG_CONFIG_HOME support

closes gh-278
closes gh-246
closes gh-48

IPython/core/application.py

@@ -1,453 +1,453 b''
 # encoding: utf-8
 """
 An application for IPython.
 
 All top-level applications should use the classes in this module for
 handling configuration and creating componenets.
 
 The job of an :class:`Application` is to create the master configuration 
 object and then create the configurable objects, passing the config to them.
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 
 Notes
 -----
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import logging
 import os
 import sys
 
 from IPython.core import release, crashhandler
 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
 from IPython.config.loader import (
     PyFileConfigLoader,
     ArgParseConfigLoader,
     Config,
 )
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 class ApplicationError(Exception):
     pass
 
 
 class BaseAppConfigLoader(ArgParseConfigLoader):
     """Default command line options for IPython based applications."""
 
     def _add_ipython_dir(self, parser):
         """Add the --ipython-dir option to the parser."""
         paa = parser.add_argument
         paa('--ipython-dir', 
             dest='Global.ipython_dir',type=unicode,
             help=
             """Set to override default location of the IPython directory
             IPYTHON_DIR, stored as Global.ipython_dir.  This can also be 
             specified through the environment variable IPYTHON_DIR.""",
             metavar='Global.ipython_dir')
 
     def _add_log_level(self, parser):
         """Add the --log-level option to the parser."""
         paa = parser.add_argument
         paa('--log-level',
             dest="Global.log_level",type=int,
             help='Set the log level (0,10,20,30,40,50).  Default is 30.',
             metavar='Global.log_level')
 
     def _add_arguments(self):
         self._add_ipython_dir(self.parser)
         self._add_log_level(self.parser)
 
 
 class Application(object):
     """Load a config, construct configurables and set them running.
 
     The configuration of an application can be done via three different Config
     objects, which are loaded and ultimately merged into a single one used
     from that point on by the app. These are:
 
     1. default_config: internal defaults, implemented in code.
     2. file_config: read from the filesystem.
     3. command_line_config: read from the system's command line flags.
 
     During initialization, 3 is actually read before 2, since at the
     command-line one may override the location of the file to be read.  But the
     above is the order in which the merge is made.
     """
 
     name = u'ipython'
     description = 'IPython: an enhanced interactive Python shell.'
     #: Usage message printed by argparse. If None, auto-generate
     usage = None
     #: The command line config loader.  Subclass of ArgParseConfigLoader.
     command_line_loader = BaseAppConfigLoader
     #: The name of the config file to load, determined at runtime
     config_file_name = None
     #: The name of the default config file. Track separately from the actual
     #: name because some logic happens only if we aren't using the default.
     default_config_file_name = u'ipython_config.py'
     default_log_level = logging.WARN
     #: Set by --profile option
     profile_name = None
-    #: User's ipython directory, typically ~/.ipython/
+    #: User's ipython directory, typically ~/.ipython or ~/.config/ipython/
     ipython_dir = None
     #: Internal defaults, implemented in code.
     default_config = None
     #: Read from the filesystem.
     file_config = None
     #: Read from the system's command line flags.
     command_line_config = None
     #: The final config that will be passed to the main object.
     master_config = None
     #: A reference to the argv to be used (typically ends up being sys.argv[1:])
     argv = None
     #: extra arguments computed by the command-line loader
     extra_args = None
     #: The class to use as the crash handler.
     crash_handler_class = crashhandler.CrashHandler
 
     # Private attributes
     _exiting = False
     _initialized = False
 
     def __init__(self, argv=None):
         self.argv = sys.argv[1:] if argv is None else argv
         self.init_logger()
 
     def init_logger(self):
         self.log = logging.getLogger(self.__class__.__name__)
         # This is used as the default until the command line arguments are read.
         self.log.setLevel(self.default_log_level)
         self._log_handler = logging.StreamHandler()
         self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
         self._log_handler.setFormatter(self._log_formatter)
         self.log.addHandler(self._log_handler)
 
     def _set_log_level(self, level):
         self.log.setLevel(level)
 
     def _get_log_level(self):
         return self.log.level
 
     log_level = property(_get_log_level, _set_log_level)
 
     def initialize(self):
         """Initialize the application.
 
         Loads all configuration information and sets all application state, but
         does not start any relevant processing (typically some kind of event
         loop).
 
         Once this method has been called, the application is flagged as
         initialized and the method becomes a no-op."""
         
         if self._initialized:
             return
 
         # The first part is protected with an 'attempt' wrapper, that will log
         # failures with the basic system traceback machinery.  Once our crash
         # handler is in place, we can let any subsequent exception propagate,
         # as our handler will log it with much better detail than the default.
         self.attempt(self.create_crash_handler)
 
         # Configuration phase
         # Default config (internally hardwired in application code)
         self.create_default_config()
         self.log_default_config()
         self.set_default_config_log_level()
 
         # Command-line config
         self.pre_load_command_line_config()
         self.load_command_line_config()
         self.set_command_line_config_log_level()
         self.post_load_command_line_config()
         self.log_command_line_config()
 
         # Find resources needed for filesystem access, using information from
         # the above two
         self.find_ipython_dir()
         self.find_resources()
         self.find_config_file_name()
         self.find_config_file_paths()
 
         # File-based config
         self.pre_load_file_config()
         self.load_file_config()
         self.set_file_config_log_level()
         self.post_load_file_config()
         self.log_file_config()
 
         # Merge all config objects into a single one the app can then use
         self.merge_configs()
         self.log_master_config()
 
         # Construction phase
         self.pre_construct()
         self.construct()
         self.post_construct()
 
         # Done, flag as such and
         self._initialized = True
 
     def start(self):
         """Start the application."""
         self.initialize()
         self.start_app()
 
     #-------------------------------------------------------------------------
     # Various stages of Application creation
     #-------------------------------------------------------------------------
 
     def create_crash_handler(self):
         """Create a crash handler, typically setting sys.excepthook to it."""
         self.crash_handler = self.crash_handler_class(self)
         sys.excepthook = self.crash_handler
 
     def create_default_config(self):
         """Create defaults that can't be set elsewhere.
 
         For the most part, we try to set default in the class attributes
         of Configurables.  But, defaults the top-level Application (which is
         not a HasTraits or Configurables) are not set in this way.  Instead
         we set them here.  The Global section is for variables like this that
         don't belong to a particular configurable.
         """
         c = Config()
         c.Global.ipython_dir = get_ipython_dir()
         c.Global.log_level = self.log_level
         self.default_config = c
 
     def log_default_config(self):
         self.log.debug('Default config loaded:')
         self.log.debug(repr(self.default_config))
 
     def set_default_config_log_level(self):
         try:
             self.log_level = self.default_config.Global.log_level
         except AttributeError:
             # Fallback to the default_log_level class attribute
             pass
 
     def create_command_line_config(self):
         """Create and return a command line config loader."""
         return self.command_line_loader(
             self.argv,
             description=self.description, 
             version=release.version,
             usage=self.usage
         )
 
     def pre_load_command_line_config(self):
         """Do actions just before loading the command line config."""
         pass
 
     def load_command_line_config(self):
         """Load the command line config."""
         loader = self.create_command_line_config()
         self.command_line_config = loader.load_config()
         self.extra_args = loader.get_extra_args()
 
     def set_command_line_config_log_level(self):
         try:
             self.log_level = self.command_line_config.Global.log_level
         except AttributeError:
             pass
 
     def post_load_command_line_config(self):
         """Do actions just after loading the command line config."""
         pass
 
     def log_command_line_config(self):
         self.log.debug("Command line config loaded:")
         self.log.debug(repr(self.command_line_config))
 
     def find_ipython_dir(self):
         """Set the IPython directory.
 
         This sets ``self.ipython_dir``, but the actual value that is passed to
         the application is kept in either ``self.default_config`` or
         ``self.command_line_config``.  This also adds ``self.ipython_dir`` to
         ``sys.path`` so config files there can be referenced by other config
         files.
         """
 
         try:
             self.ipython_dir = self.command_line_config.Global.ipython_dir
         except AttributeError:
             self.ipython_dir = self.default_config.Global.ipython_dir
         sys.path.append(os.path.abspath(self.ipython_dir))
         if not os.path.isdir(self.ipython_dir):
             os.makedirs(self.ipython_dir, mode=0777)
         self.log.debug("IPYTHON_DIR set to: %s" % self.ipython_dir)
 
     def find_resources(self):
         """Find other resources that need to be in place.
 
         Things like cluster directories need to be in place to find the
         config file.  These happen right after the IPython directory has
         been set.
         """
         pass
 
     def find_config_file_name(self):
         """Find the config file name for this application.
 
         This must set ``self.config_file_name`` to the filename of the
         config file to use (just the filename). The search paths for the
         config file are set in :meth:`find_config_file_paths` and then passed
         to the config file loader where they are resolved to an absolute path.
 
         If a profile has been set at the command line, this will resolve it.
         """
         try:
             self.config_file_name = self.command_line_config.Global.config_file
         except AttributeError:
             pass
         else:
             return
 
         try:
             self.profile_name = self.command_line_config.Global.profile
         except AttributeError:
             # Just use the default as there is no profile
             self.config_file_name = self.default_config_file_name
         else:
             # Use the default config file name and profile name if set
             # to determine the used config file name.
             name_parts = self.default_config_file_name.split('.')
             name_parts.insert(1, u'_' + self.profile_name + u'.')
             self.config_file_name = ''.join(name_parts)
 
     def find_config_file_paths(self):
         """Set the search paths for resolving the config file.
 
         This must set ``self.config_file_paths`` to a sequence of search
         paths to pass to the config file loader.
         """
         # Include our own profiles directory last, so that users can still find
         # our shipped copies of builtin profiles even if they don't have them
         # in their local ipython directory.
         prof_dir = os.path.join(get_ipython_package_dir(), 'config', 'profile')
         self.config_file_paths = (os.getcwd(), self.ipython_dir, prof_dir)
 
     def pre_load_file_config(self):
         """Do actions before the config file is loaded."""
         pass
 
     def load_file_config(self):
         """Load the config file.
         
         This tries to load the config file from disk.  If successful, the
         ``CONFIG_FILE`` config variable is set to the resolved config file
         location.  If not successful, an empty config is used.
         """
         self.log.debug("Attempting to load config file: %s" %
                        self.config_file_name)
         loader = PyFileConfigLoader(self.config_file_name,
                                     path=self.config_file_paths)
         try:
             self.file_config = loader.load_config()
             self.file_config.Global.config_file = loader.full_filename
         except IOError:
             # Only warn if the default config file was NOT being used.
             if not self.config_file_name==self.default_config_file_name:
                 self.log.warn("Config file not found, skipping: %s" %
                                self.config_file_name, exc_info=True)
             self.file_config = Config()
         except:
             self.log.warn("Error loading config file: %s" %
                           self.config_file_name, exc_info=True)
             self.file_config = Config()
 
     def set_file_config_log_level(self):
         # We need to keeep self.log_level updated.  But we only use the value
         # of the file_config if a value was not specified at the command
         # line, because the command line overrides everything.
         if not hasattr(self.command_line_config.Global, 'log_level'):
             try:
                 self.log_level = self.file_config.Global.log_level
             except AttributeError:
                 pass # Use existing value
 
     def post_load_file_config(self):
         """Do actions after the config file is loaded."""
         pass
 
     def log_file_config(self):
         if hasattr(self.file_config.Global, 'config_file'):
             self.log.debug("Config file loaded: %s" %
                            self.file_config.Global.config_file)
             self.log.debug(repr(self.file_config))
 
     def merge_configs(self):
         """Merge the default, command line and file config objects."""
         config = Config()
         config._merge(self.default_config)
         config._merge(self.file_config)
         config._merge(self.command_line_config)
 
         # XXX fperez - propose to Brian we rename master_config to simply
         # config, I think this is going to be heavily used in examples and
         # application code and the name is shorter/easier to find/remember.
         # For now, just alias it...
         self.master_config = config
         self.config = config
 
     def log_master_config(self):
         self.log.debug("Master config created:")
         self.log.debug(repr(self.master_config))
 
     def pre_construct(self):
         """Do actions after the config has been built, but before construct."""
         pass
 
     def construct(self):
         """Construct the main objects that make up this app."""
         self.log.debug("Constructing main objects for application")
 
     def post_construct(self):
         """Do actions after construct, but before starting the app."""
         pass
 
     def start_app(self):
         """Actually start the app."""
         self.log.debug("Starting application")
 
     #-------------------------------------------------------------------------
     # Utility methods
     #-------------------------------------------------------------------------
 
     def exit(self, exit_status=0):
         if self._exiting:
             pass
         else:
             self.log.debug("Exiting application: %s" % self.name)
             self._exiting = True
             sys.exit(exit_status)
 
     def attempt(self, func):
         try:
             func()
         except SystemExit:
             raise
         except:
             self.log.critical("Aborting application: %s" % self.name,
                               exc_info=True)
             self.exit(0)
 

IPython/core/magic.py

@@ -1,3510 +1,3510 b''
 # encoding: utf-8
 """Magic functions for InteractiveShell.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001 Janko Hauser <[e-mail]> and
 #  Copyright (C) 2001-2007 Fernando Perez <[e-mail]>
 #  Copyright (C) 2008-2009  The IPython Development Team
 
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import __builtin__
 import __future__
 import bdb
 import inspect
 import os
 import sys
 import shutil
 import re
 import time
 import textwrap
 import types
 from cStringIO import StringIO
 from getopt import getopt,GetoptError
 from pprint import pformat
 
 # cProfile was added in Python2.5
 try:
     import cProfile as profile
     import pstats
 except ImportError:
     # profile isn't bundled by default in Debian for license reasons
     try:
         import profile,pstats
     except ImportError:
         profile = pstats = None
 
 import IPython
 from IPython.core import debugger, oinspect
 from IPython.core.error import TryNext
 from IPython.core.error import UsageError
 from IPython.core.fakemodule import FakeModule
 from IPython.core.macro import Macro
 from IPython.core import page
 from IPython.core.prefilter import ESC_MAGIC
 from IPython.lib.pylabtools import mpl_runner
 from IPython.external.Itpl import itpl, printpl
 from IPython.testing import decorators as testdec
 from IPython.utils.io import file_read, nlprint
 import IPython.utils.io
 from IPython.utils.path import get_py_filename
 from IPython.utils.process import arg_split, abbrev_cwd
 from IPython.utils.terminal import set_term_title
 from IPython.utils.text import LSString, SList, StringTypes, format_screen
 from IPython.utils.timing import clock, clock2
 from IPython.utils.warn import warn, error
 from IPython.utils.ipstruct import Struct
 import IPython.utils.generics
 
 #-----------------------------------------------------------------------------
 # Utility functions
 #-----------------------------------------------------------------------------
 
 def on_off(tag):
     """Return an ON/OFF string for a 1/0 input. Simple utility function."""
     return ['OFF','ON'][tag]
 
 class Bunch: pass
 
 def compress_dhist(dh):
     head, tail = dh[:-10], dh[-10:]
 
     newhead = []
     done = set()
     for h in head:
         if h in done:
             continue
         newhead.append(h)
         done.add(h)
 
     return newhead + tail        
 
 
 #***************************************************************************
 # Main class implementing Magic functionality
 
 # XXX - for some odd reason, if Magic is made a new-style class, we get errors
 # on construction of the main InteractiveShell object.  Something odd is going
 # on with super() calls, Configurable and the MRO... For now leave it as-is, but
 # eventually this needs to be clarified.
 # BG: This is because InteractiveShell inherits from this, but is itself a 
 # Configurable. This messes up the MRO in some way. The fix is that we need to
 # make Magic a configurable that InteractiveShell does not subclass.
 
 class Magic:
     """Magic functions for InteractiveShell.
 
     Shell functions which can be reached as %function_name. All magic
     functions should accept a string, which they can parse for their own
     needs. This can make some functions easier to type, eg `%cd ../`
     vs. `%cd("../")`
 
     ALL definitions MUST begin with the prefix magic_. The user won't need it
     at the command line, but it is is needed in the definition. """
 
     # class globals
     auto_status = ['Automagic is OFF, % prefix IS needed for magic functions.',
                    'Automagic is ON, % prefix NOT needed for magic functions.']
 
     #......................................................................
     # some utility functions
 
     def __init__(self,shell):
         
         self.options_table = {}
         if profile is None:
             self.magic_prun = self.profile_missing_notice
         self.shell = shell
 
         # namespace for holding state we may need
         self._magic_state = Bunch()
 
     def profile_missing_notice(self, *args, **kwargs):
         error("""\
 The profile module could not be found. It has been removed from the standard
 python packages because of its non-free license. To use profiling, install the
 python-profiler package from non-free.""")
 
     def default_option(self,fn,optstr):
         """Make an entry in the options_table for fn, with value optstr"""
 
         if fn not in self.lsmagic():
             error("%s is not a magic function" % fn)
         self.options_table[fn] = optstr
 
     def lsmagic(self):
         """Return a list of currently available magic functions.
 
         Gives a list of the bare names after mangling (['ls','cd', ...], not
         ['magic_ls','magic_cd',...]"""
 
         # FIXME. This needs a cleanup, in the way the magics list is built.
         
         # magics in class definition
         class_magic = lambda fn: fn.startswith('magic_') and \
                       callable(Magic.__dict__[fn])
         # in instance namespace (run-time user additions)
         inst_magic =  lambda fn: fn.startswith('magic_') and \
                      callable(self.__dict__[fn])
         # and bound magics by user (so they can access self):
         inst_bound_magic =  lambda fn: fn.startswith('magic_') and \
                            callable(self.__class__.__dict__[fn])
         magics = filter(class_magic,Magic.__dict__.keys()) + \
                  filter(inst_magic,self.__dict__.keys()) + \
                  filter(inst_bound_magic,self.__class__.__dict__.keys())
         out = []
         for fn in set(magics):
             out.append(fn.replace('magic_','',1))
         out.sort()
         return out
     
     def extract_input_slices(self,slices,raw=False):
         """Return as a string a set of input history slices.
 
         Inputs:
 
           - slices: the set of slices is given as a list of strings (like
           ['1','4:8','9'], since this function is for use by magic functions
           which get their arguments as strings.
 
         Optional inputs:
 
           - raw(False): by default, the processed input is used.  If this is
           true, the raw input history is used instead.
 
         Note that slices can be called with two notations:
 
         N:M -> standard python form, means including items N...(M-1).
 
         N-M -> include items N..M (closed endpoint)."""
 
         if raw:
             hist = self.shell.history_manager.input_hist_raw
         else:
             hist = self.shell.history_manager.input_hist_parsed
 
         cmds = []
         for chunk in slices:
             if ':' in chunk:
                 ini,fin = map(int,chunk.split(':'))
             elif '-' in chunk:
                 ini,fin = map(int,chunk.split('-'))
                 fin += 1
             else:
                 ini = int(chunk)
                 fin = ini+1
             cmds.append(''.join(hist[ini:fin]))
         return cmds
             
     def arg_err(self,func):
         """Print docstring if incorrect arguments were passed"""
         print 'Error in arguments:'
         print oinspect.getdoc(func)
 
     def format_latex(self,strng):
         """Format a string for latex inclusion."""
 
         # Characters that need to be escaped for latex:
         escape_re = re.compile(r'(%|_|\$|#|&)',re.MULTILINE)
         # Magic command names as headers:
         cmd_name_re = re.compile(r'^(%s.*?):' % ESC_MAGIC,
                                  re.MULTILINE)
         # Magic commands 
         cmd_re = re.compile(r'(?P<cmd>%s.+?\b)(?!\}\}:)' % ESC_MAGIC,
                             re.MULTILINE)
         # Paragraph continue
         par_re = re.compile(r'\\$',re.MULTILINE)
 
         # The "\n" symbol
         newline_re = re.compile(r'\\n')
 
         # Now build the string for output:
         #strng = cmd_name_re.sub(r'\n\\texttt{\\textsl{\\large \1}}:',strng)
         strng = cmd_name_re.sub(r'\n\\bigskip\n\\texttt{\\textbf{ \1}}:',
                                 strng)
         strng = cmd_re.sub(r'\\texttt{\g<cmd>}',strng)
         strng = par_re.sub(r'\\\\',strng)
         strng = escape_re.sub(r'\\\1',strng)
         strng = newline_re.sub(r'\\textbackslash{}n',strng)
         return strng
 
     def parse_options(self,arg_str,opt_str,*long_opts,**kw):
         """Parse options passed to an argument string.
 
         The interface is similar to that of getopt(), but it returns back a
         Struct with the options as keys and the stripped argument string still
         as a string.
 
         arg_str is quoted as a true sys.argv vector by using shlex.split.
         This allows us to easily expand variables, glob files, quote
         arguments, etc.
 
         Options:
           -mode: default 'string'. If given as 'list', the argument string is
           returned as a list (split on whitespace) instead of a string.
 
           -list_all: put all option values in lists. Normally only options
           appearing more than once are put in a list.
 
           -posix (True): whether to split the input line in POSIX mode or not,
           as per the conventions outlined in the shlex module from the
           standard library."""
  
         # inject default options at the beginning of the input line
         caller = sys._getframe(1).f_code.co_name.replace('magic_','')
         arg_str = '%s %s' % (self.options_table.get(caller,''),arg_str)
         
         mode = kw.get('mode','string')
         if mode not in ['string','list']:
             raise ValueError,'incorrect mode given: %s' % mode
         # Get options
         list_all = kw.get('list_all',0)
         posix = kw.get('posix', os.name == 'posix')
 
         # Check if we have more than one argument to warrant extra processing:
         odict = {}  # Dictionary with options
         args = arg_str.split()
         if len(args) >= 1:
             # If the list of inputs only has 0 or 1 thing in it, there's no
             # need to look for options
             argv = arg_split(arg_str,posix)
             # Do regular option processing
             try:
                 opts,args = getopt(argv,opt_str,*long_opts)
             except GetoptError,e:
                 raise UsageError('%s ( allowed: "%s" %s)' % (e.msg,opt_str, 
                                         " ".join(long_opts)))
             for o,a in opts:
                 if o.startswith('--'):
                     o = o[2:]
                 else:
                     o = o[1:]
                 try:
                     odict[o].append(a)
                 except AttributeError:
                     odict[o] = [odict[o],a]
                 except KeyError:
                     if list_all:
                         odict[o] = [a]
                     else:
                         odict[o] = a
 
         # Prepare opts,args for return
         opts = Struct(odict)
         if mode == 'string':
             args = ' '.join(args)
 
         return opts,args
     
     #......................................................................
     # And now the actual magic functions
 
     # Functions for IPython shell work (vars,funcs, config, etc)
     def magic_lsmagic(self, parameter_s = ''):
         """List currently available magic functions."""
         mesc = ESC_MAGIC
         print 'Available magic functions:\n'+mesc+\
               ('  '+mesc).join(self.lsmagic())
         print '\n' + Magic.auto_status[self.shell.automagic]
         return None
         
     def magic_magic(self, parameter_s = ''):
         """Print information about the magic function system.
         
         Supported formats: -latex, -brief, -rest        
         """
 
         mode = ''
         try:
             if parameter_s.split()[0] == '-latex':
                 mode = 'latex'
             if parameter_s.split()[0] == '-brief':
                 mode = 'brief'
             if parameter_s.split()[0] == '-rest':
                 mode = 'rest'
                 rest_docs = []
         except:
             pass
 
         magic_docs = []
         for fname in self.lsmagic():
             mname = 'magic_' + fname
             for space in (Magic,self,self.__class__):
                 try:
                     fn = space.__dict__[mname]
                 except KeyError:
                     pass
                 else:
                     break
             if mode == 'brief':
                 # only first line
                 if fn.__doc__:                    
                     fndoc = fn.__doc__.split('\n',1)[0]
                 else:
                     fndoc = 'No documentation'
             else:
                 if fn.__doc__:
                     fndoc = fn.__doc__.rstrip()       
                 else:
                     fndoc = 'No documentation'
                 
                 
             if mode == 'rest':
                 rest_docs.append('**%s%s**::\n\n\t%s\n\n' %(ESC_MAGIC,
                                                     fname,fndoc))
                 
             else:
                 magic_docs.append('%s%s:\n\t%s\n' %(ESC_MAGIC,
                                                     fname,fndoc))
                 
         magic_docs = ''.join(magic_docs)
 
         if mode == 'rest':
             return "".join(rest_docs)
         
         if mode == 'latex':
             print self.format_latex(magic_docs)
             return
         else:
             magic_docs = format_screen(magic_docs)
         if mode == 'brief':
             return magic_docs
         
         outmsg = """
 IPython's 'magic' functions
 ===========================
 
 The magic function system provides a series of functions which allow you to
 control the behavior of IPython itself, plus a lot of system-type
 features. All these functions are prefixed with a % character, but parameters
 are given without parentheses or quotes.
 
 NOTE: If you have 'automagic' enabled (via the command line option or with the
 %automagic function), you don't need to type in the % explicitly.  By default,
 IPython ships with automagic on, so you should only rarely need the % escape.
 
 Example: typing '%cd mydir' (without the quotes) changes you working directory
 to 'mydir', if it exists.
 
 You can define your own magic functions to extend the system. See the supplied
 ipythonrc and example-magic.py files for details (in your ipython
-configuration directory, typically $HOME/.ipython/).
+configuration directory, typically $HOME/.config/ipython on Linux or $HOME/.ipython elsewhere).
 
 You can also define your own aliased names for magic functions. In your
 ipythonrc file, placing a line like:
 
   execute __IPYTHON__.magic_pf = __IPYTHON__.magic_profile
 
 will define %pf as a new name for %profile.
 
 You can also call magics in code using the magic() function, which IPython
 automatically adds to the builtin namespace.  Type 'magic?' for details.
 
 For a list of the available magic functions, use %lsmagic. For a description
 of any of them, type %magic_name?, e.g. '%cd?'.
 
 Currently the magic system has the following functions:\n"""
 
         mesc = ESC_MAGIC
         outmsg = ("%s\n%s\n\nSummary of magic functions (from %slsmagic):"
                   "\n\n%s%s\n\n%s" % (outmsg,
                                      magic_docs,mesc,mesc,
                                      ('  '+mesc).join(self.lsmagic()),
                                      Magic.auto_status[self.shell.automagic] ) )
         page.page(outmsg)
 
     def magic_automagic(self, parameter_s = ''):
         """Make magic functions callable without having to type the initial %.
 
         Without argumentsl toggles on/off (when off, you must call it as
         %automagic, of course).  With arguments it sets the value, and you can
         use any of (case insensitive):
 
          - on,1,True: to activate
 
          - off,0,False: to deactivate.
 
         Note that magic functions have lowest priority, so if there's a
         variable whose name collides with that of a magic fn, automagic won't
         work for that function (you get the variable instead). However, if you
         delete the variable (del var), the previously shadowed magic function
         becomes visible to automagic again."""
 
         arg = parameter_s.lower()
         if parameter_s in ('on','1','true'):
             self.shell.automagic = True
         elif parameter_s in ('off','0','false'):
             self.shell.automagic = False
         else:
             self.shell.automagic = not self.shell.automagic
         print '\n' + Magic.auto_status[self.shell.automagic]
 
     @testdec.skip_doctest
     def magic_autocall(self, parameter_s = ''):
         """Make functions callable without having to type parentheses.
 
         Usage:
 
            %autocall [mode]
 
         The mode can be one of: 0->Off, 1->Smart, 2->Full.  If not given, the
         value is toggled on and off (remembering the previous state).
 
         In more detail, these values mean:
 
         0 -> fully disabled
 
         1 -> active, but do not apply if there are no arguments on the line.
 
         In this mode, you get:
 
         In [1]: callable
         Out[1]: <built-in function callable>
 
         In [2]: callable 'hello'
         ------> callable('hello')
         Out[2]: False
 
         2 -> Active always.  Even if no arguments are present, the callable
         object is called:
 
         In [2]: float
         ------> float()
         Out[2]: 0.0
 
         Note that even with autocall off, you can still use '/' at the start of
         a line to treat the first argument on the command line as a function
         and add parentheses to it:
 
         In [8]: /str 43
         ------> str(43)
         Out[8]: '43'
 
         # all-random (note for auto-testing)
         """
 
         if parameter_s:
             arg = int(parameter_s)
         else:
             arg = 'toggle'
 
         if not arg in (0,1,2,'toggle'):
             error('Valid modes: (0->Off, 1->Smart, 2->Full')
             return
 
         if arg in (0,1,2):
             self.shell.autocall = arg
         else: # toggle
             if self.shell.autocall:
                 self._magic_state.autocall_save = self.shell.autocall
                 self.shell.autocall = 0
             else:
                 try:
                     self.shell.autocall = self._magic_state.autocall_save
                 except AttributeError:
                     self.shell.autocall = self._magic_state.autocall_save = 1
 
         print "Automatic calling is:",['OFF','Smart','Full'][self.shell.autocall]
 
 
     def magic_page(self, parameter_s=''):
         """Pretty print the object and display it through a pager.
         
         %page [options] OBJECT
 
         If no object is given, use _ (last output).
         
         Options:
 
           -r: page str(object), don't pretty-print it."""
 
         # After a function contributed by Olivier Aubert, slightly modified.
 
         # Process options/args
         opts,args = self.parse_options(parameter_s,'r')
         raw = 'r' in opts
 
         oname = args and args or '_'
         info = self._ofind(oname)
         if info['found']:
             txt = (raw and str or pformat)( info['obj'] )
             page.page(txt)
         else:
             print 'Object `%s` not found' % oname
 
     def magic_profile(self, parameter_s=''):
         """Print your currently active IPython profile."""
         if self.shell.profile:
             printpl('Current IPython profile: $self.shell.profile.')
         else:
             print 'No profile active.'
 
     def magic_pinfo(self, parameter_s='', namespaces=None):
         """Provide detailed information about an object.
     
         '%pinfo object' is just a synonym for object? or ?object."""
     
         #print 'pinfo par: <%s>' % parameter_s  # dbg
     
         
         # detail_level: 0 -> obj? , 1 -> obj??
         detail_level = 0
         # We need to detect if we got called as 'pinfo pinfo foo', which can
         # happen if the user types 'pinfo foo?' at the cmd line.
         pinfo,qmark1,oname,qmark2 = \
                re.match('(pinfo )?(\?*)(.*?)(\??$)',parameter_s).groups()
         if pinfo or qmark1 or qmark2:
             detail_level = 1
         if "*" in oname:
             self.magic_psearch(oname)
         else:
             self.shell._inspect('pinfo', oname, detail_level=detail_level,
                                 namespaces=namespaces)
 
     def magic_pinfo2(self, parameter_s='', namespaces=None):
         """Provide extra detailed information about an object.
     
         '%pinfo2 object' is just a synonym for object?? or ??object."""
         self.shell._inspect('pinfo', parameter_s, detail_level=1,
                             namespaces=namespaces)
 
     @testdec.skip_doctest
     def magic_pdef(self, parameter_s='', namespaces=None):
         """Print the definition header for any callable object.
 
         If the object is a class, print the constructor information.
         
         Examples
         --------
         ::
         
           In [3]: %pdef urllib.urlopen
           urllib.urlopen(url, data=None, proxies=None)
         """
         self._inspect('pdef',parameter_s, namespaces)
 
     def magic_pdoc(self, parameter_s='', namespaces=None):
         """Print the docstring for an object.
 
         If the given object is a class, it will print both the class and the
         constructor docstrings."""
         self._inspect('pdoc',parameter_s, namespaces)
 
     def magic_psource(self, parameter_s='', namespaces=None):
         """Print (or run through pager) the source code for an object."""
         self._inspect('psource',parameter_s, namespaces)
 
     def magic_pfile(self, parameter_s=''):
         """Print (or run through pager) the file where an object is defined.
 
         The file opens at the line where the object definition begins. IPython
         will honor the environment variable PAGER if set, and otherwise will
         do its best to print the file in a convenient form.
 
         If the given argument is not an object currently defined, IPython will
         try to interpret it as a filename (automatically adding a .py extension
         if needed). You can thus use %pfile as a syntax highlighting code
         viewer."""
 
         # first interpret argument as an object name
         out = self._inspect('pfile',parameter_s)
         # if not, try the input as a filename
         if out == 'not found':
             try:
                 filename = get_py_filename(parameter_s)
             except IOError,msg:
                 print msg
                 return
             page.page(self.shell.inspector.format(file(filename).read()))
 
     def magic_psearch(self, parameter_s=''):
         """Search for object in namespaces by wildcard.
 
         %psearch [options] PATTERN [OBJECT TYPE]
 
         Note: ? can be used as a synonym for %psearch, at the beginning or at
         the end: both a*? and ?a* are equivalent to '%psearch a*'.  Still, the
         rest of the command line must be unchanged (options come first), so
         for example the following forms are equivalent
 
         %psearch -i a* function
         -i a* function?
         ?-i a* function
 
         Arguments:
         
           PATTERN
 
           where PATTERN is a string containing * as a wildcard similar to its
           use in a shell.  The pattern is matched in all namespaces on the
           search path. By default objects starting with a single _ are not
           matched, many IPython generated objects have a single
           underscore. The default is case insensitive matching. Matching is
           also done on the attributes of objects and not only on the objects
           in a module.
 
           [OBJECT TYPE]
 
           Is the name of a python type from the types module. The name is
           given in lowercase without the ending type, ex. StringType is
           written string. By adding a type here only objects matching the
           given type are matched. Using all here makes the pattern match all
           types (this is the default).
 
         Options:
 
           -a: makes the pattern match even objects whose names start with a
           single underscore.  These names are normally ommitted from the
           search.
 
           -i/-c: make the pattern case insensitive/sensitive.  If neither of
           these options is given, the default is read from your ipythonrc
           file.  The option name which sets this value is
           'wildcards_case_sensitive'.  If this option is not specified in your
           ipythonrc file, IPython's internal default is to do a case sensitive
           search.
 
           -e/-s NAMESPACE: exclude/search a given namespace.  The pattern you
           specifiy can be searched in any of the following namespaces:
           'builtin', 'user', 'user_global','internal', 'alias', where
           'builtin' and 'user' are the search defaults.  Note that you should
           not use quotes when specifying namespaces.
 
           'Builtin' contains the python module builtin, 'user' contains all
           user data, 'alias' only contain the shell aliases and no python
           objects, 'internal' contains objects used by IPython.  The
           'user_global' namespace is only used by embedded IPython instances,
           and it contains module-level globals.  You can add namespaces to the
           search with -s or exclude them with -e (these options can be given
           more than once).
     
         Examples:
        
         %psearch a*            -> objects beginning with an a
         %psearch -e builtin a* -> objects NOT in the builtin space starting in a
         %psearch a* function   -> all functions beginning with an a
         %psearch re.e*         -> objects beginning with an e in module re
         %psearch r*.e*         -> objects that start with e in modules starting in r
         %psearch r*.* string   -> all strings in modules beginning with r
 
         Case sensitve search:
        
         %psearch -c a*         list all object beginning with lower case a
 
         Show objects beginning with a single _:
        
         %psearch -a _*         list objects beginning with a single underscore"""
         try:
             parameter_s = parameter_s.encode('ascii')
         except UnicodeEncodeError:
             print 'Python identifiers can only contain ascii characters.'
             return
 
         # default namespaces to be searched
         def_search = ['user','builtin']
 
         # Process options/args
         opts,args = self.parse_options(parameter_s,'cias:e:',list_all=True)
         opt = opts.get
         shell = self.shell
         psearch = shell.inspector.psearch
 
         # select case options
         if opts.has_key('i'):
             ignore_case = True
         elif opts.has_key('c'):
             ignore_case = False
         else:
             ignore_case = not shell.wildcards_case_sensitive
 
         # Build list of namespaces to search from user options
         def_search.extend(opt('s',[]))
         ns_exclude = ns_exclude=opt('e',[])
         ns_search = [nm for nm in def_search if nm not in ns_exclude]
         
         # Call the actual search
         try:
             psearch(args,shell.ns_table,ns_search,
                     show_all=opt('a'),ignore_case=ignore_case)
         except:
             shell.showtraceback()
         
     @testdec.skip_doctest
     def magic_who_ls(self, parameter_s=''):
         """Return a sorted list of all interactive variables.
 
         If arguments are given, only variables of types matching these
         arguments are returned.
 
         Examples
         --------
 
         Define two variables and list them with who_ls::
 
           In [1]: alpha = 123
 
           In [2]: beta = 'test'
 
           In [3]: %who_ls
           Out[3]: ['alpha', 'beta']
 
           In [4]: %who_ls int
           Out[4]: ['alpha']
 
           In [5]: %who_ls str
           Out[5]: ['beta']
         """
 
         user_ns = self.shell.user_ns
         internal_ns = self.shell.internal_ns
         user_ns_hidden = self.shell.user_ns_hidden
         out = [ i for i in user_ns
                 if not i.startswith('_') \
                 and not (i in internal_ns or i in user_ns_hidden) ]
 
         typelist = parameter_s.split()
         if typelist:
             typeset = set(typelist)
             out = [i for i in out if type(user_ns[i]).__name__ in typeset]
 
         out.sort()
         return out
         
     @testdec.skip_doctest
     def magic_who(self, parameter_s=''):
         """Print all interactive variables, with some minimal formatting.
 
         If any arguments are given, only variables whose type matches one of
         these are printed.  For example:
 
           %who function str
 
         will only list functions and strings, excluding all other types of
         variables.  To find the proper type names, simply use type(var) at a
         command line to see how python prints type names.  For example:
 
           In [1]: type('hello')\\
           Out[1]: <type 'str'>
 
         indicates that the type name for strings is 'str'.
 
         %who always excludes executed names loaded through your configuration
         file and things which are internal to IPython.
 
         This is deliberate, as typically you may load many modules and the
         purpose of %who is to show you only what you've manually defined.
 
         Examples
         --------
 
         Define two variables and list them with who::
 
           In [1]: alpha = 123
 
           In [2]: beta = 'test'
 
           In [3]: %who
           alpha   beta
 
           In [4]: %who int
           alpha
 
           In [5]: %who str
           beta
         """
 
         varlist = self.magic_who_ls(parameter_s)
         if not varlist:
             if parameter_s:
                 print 'No variables match your requested type.'
             else:
                 print 'Interactive namespace is empty.'
             return
 
         # if we have variables, move on...
         count = 0
         for i in varlist:
             print i+'\t',
             count += 1
             if count > 8:
                 count = 0
                 print
         print
 
     @testdec.skip_doctest
     def magic_whos(self, parameter_s=''):
         """Like %who, but gives some extra information about each variable.
 
         The same type filtering of %who can be applied here.
 
         For all variables, the type is printed. Additionally it prints:
         
           - For {},[],(): their length.
 
           - For numpy and Numeric arrays, a summary with shape, number of
           elements, typecode and size in memory.
 
           - Everything else: a string representation, snipping their middle if
           too long.
 
         Examples
         --------
 
         Define two variables and list them with whos::
 
           In [1]: alpha = 123
 
           In [2]: beta = 'test'
 
           In [3]: %whos
           Variable   Type        Data/Info
           --------------------------------
           alpha      int         123
           beta       str         test
         """
         
         varnames = self.magic_who_ls(parameter_s)
         if not varnames:
             if parameter_s:
                 print 'No variables match your requested type.'
             else:
                 print 'Interactive namespace is empty.'
             return
 
         # if we have variables, move on...
 
         # for these types, show len() instead of data:
         seq_types = [types.DictType,types.ListType,types.TupleType]
 
         # for numpy/Numeric arrays, display summary info
         try:
             import numpy
         except ImportError:
             ndarray_type = None
         else:
             ndarray_type = numpy.ndarray.__name__
         try:
             import Numeric
         except ImportError:
             array_type = None
         else:
             array_type = Numeric.ArrayType.__name__
 
         # Find all variable names and types so we can figure out column sizes
         def get_vars(i):
             return self.shell.user_ns[i]
         
         # some types are well known and can be shorter
         abbrevs = {'IPython.core.macro.Macro' : 'Macro'}
         def type_name(v):
             tn = type(v).__name__
             return abbrevs.get(tn,tn)
             
         varlist = map(get_vars,varnames)
 
         typelist = []
         for vv in varlist:
             tt = type_name(vv)
 
             if tt=='instance':
                 typelist.append( abbrevs.get(str(vv.__class__),
                                              str(vv.__class__)))
             else:
                 typelist.append(tt)
 
         # column labels and # of spaces as separator
         varlabel = 'Variable'
         typelabel = 'Type'
         datalabel = 'Data/Info'
         colsep = 3
         # variable format strings
         vformat    = "$vname.ljust(varwidth)$vtype.ljust(typewidth)"
         vfmt_short = '$vstr[:25]<...>$vstr[-25:]'
         aformat    = "%s: %s elems, type `%s`, %s bytes"
         # find the size of the columns to format the output nicely
         varwidth = max(max(map(len,varnames)), len(varlabel)) + colsep
         typewidth = max(max(map(len,typelist)), len(typelabel)) + colsep
         # table header
         print varlabel.ljust(varwidth) + typelabel.ljust(typewidth) + \
               ' '+datalabel+'\n' + '-'*(varwidth+typewidth+len(datalabel)+1)
         # and the table itself
         kb = 1024
         Mb = 1048576  # kb**2
         for vname,var,vtype in zip(varnames,varlist,typelist):
             print itpl(vformat),
             if vtype in seq_types:
                 print len(var)
             elif vtype in [array_type,ndarray_type]:
                 vshape = str(var.shape).replace(',','').replace(' ','x')[1:-1]
                 if vtype==ndarray_type:
                     # numpy
                     vsize  = var.size
                     vbytes = vsize*var.itemsize
                     vdtype = var.dtype
                 else:
                     # Numeric
                     vsize  = Numeric.size(var)
                     vbytes = vsize*var.itemsize()
                     vdtype = var.typecode()
                     
                 if vbytes < 100000:
                     print aformat % (vshape,vsize,vdtype,vbytes)
                 else:
                     print aformat % (vshape,vsize,vdtype,vbytes),
                     if vbytes < Mb:
                         print '(%s kb)' % (vbytes/kb,)
                     else:
                         print '(%s Mb)' % (vbytes/Mb,)
             else:
                 try:
                     vstr = str(var)
                 except UnicodeEncodeError:
                     vstr = unicode(var).encode(sys.getdefaultencoding(),
                                                'backslashreplace')
                 vstr = vstr.replace('\n','\\n')
                 if len(vstr) < 50:
                     print vstr
                 else:
                     printpl(vfmt_short)
                 
     def magic_reset(self, parameter_s=''):
         """Resets the namespace by removing all names defined by the user.
 
         Input/Output history are left around in case you need them.
 
         Parameters
         ----------
           -y : force reset without asking for confirmation.
 
         Examples
         --------
         In [6]: a = 1
 
         In [7]: a
         Out[7]: 1
 
         In [8]: 'a' in _ip.user_ns
         Out[8]: True
 
         In [9]: %reset -f
 
         In [10]: 'a' in _ip.user_ns
         Out[10]: False
         """
 
         if parameter_s == '-f':
             ans = True
         else:
             ans = self.shell.ask_yes_no(
                 "Once deleted, variables cannot be recovered. Proceed (y/[n])? ")
         if not ans:
             print 'Nothing done.'
             return
         user_ns = self.shell.user_ns
         for i in self.magic_who_ls():
             del(user_ns[i])
             
         # Also flush the private list of module references kept for script
         # execution protection
         self.shell.clear_main_mod_cache()
 
     def magic_reset_selective(self, parameter_s=''):
         """Resets the namespace by removing names defined by the user.
 
         Input/Output history are left around in case you need them.
 
         %reset_selective [-f] regex
 
         No action is taken if regex is not included
         
         Options
           -f : force reset without asking for confirmation.
 
         Examples
         --------
 
         We first fully reset the namespace so your output looks identical to
         this example for pedagogical reasons; in practice you do not need a
         full reset.
         
         In [1]: %reset -f
 
         Now, with a clean namespace we can make a few variables and use
         %reset_selective to only delete names that match our regexp:
 
         In [2]: a=1; b=2; c=3; b1m=4; b2m=5; b3m=6; b4m=7; b2s=8
 
         In [3]: who_ls
         Out[3]: ['a', 'b', 'b1m', 'b2m', 'b2s', 'b3m', 'b4m', 'c']
 
         In [4]: %reset_selective -f b[2-3]m
 
         In [5]: who_ls
         Out[5]: ['a', 'b', 'b1m', 'b2s', 'b4m', 'c']
 
         In [6]: %reset_selective -f d
 
         In [7]: who_ls
         Out[7]: ['a', 'b', 'b1m', 'b2s', 'b4m', 'c']
 
         In [8]: %reset_selective -f c
 
         In [9]: who_ls
         Out[9]: ['a', 'b', 'b1m', 'b2s', 'b4m']
 
         In [10]: %reset_selective -f b
 
         In [11]: who_ls
         Out[11]: ['a']
         """
         
         opts, regex = self.parse_options(parameter_s,'f')
         
         if opts.has_key('f'):
             ans = True
         else:
             ans = self.shell.ask_yes_no(
                 "Once deleted, variables cannot be recovered. Proceed (y/[n])? ")
         if not ans:
             print 'Nothing done.'
             return
         user_ns = self.shell.user_ns
         if not regex:
             print 'No regex pattern specified. Nothing done.'
             return
         else:
             try:
                 m = re.compile(regex)
             except TypeError:
                 raise TypeError('regex must be a string or compiled pattern')
             for i in self.magic_who_ls():
                 if m.search(i): 
                     del(user_ns[i])        
         
     def magic_logstart(self,parameter_s=''):
         """Start logging anywhere in a session.
 
         %logstart [-o|-r|-t] [log_name [log_mode]]
 
         If no name is given, it defaults to a file named 'ipython_log.py' in your
         current directory, in 'rotate' mode (see below).
 
         '%logstart name' saves to file 'name' in 'backup' mode.  It saves your
         history up to that point and then continues logging.
 
         %logstart takes a second optional parameter: logging mode. This can be one
         of (note that the modes are given unquoted):\\
           append: well, that says it.\\
           backup: rename (if exists) to name~ and start name.\\
           global: single logfile in your home dir, appended to.\\
           over  : overwrite existing log.\\
           rotate: create rotating logs name.1~, name.2~, etc.
 
         Options:
 
           -o: log also IPython's output.  In this mode, all commands which
           generate an Out[NN] prompt are recorded to the logfile, right after
           their corresponding input line.  The output lines are always
           prepended with a '#[Out]# ' marker, so that the log remains valid
           Python code.
 
           Since this marker is always the same, filtering only the output from
           a log is very easy, using for example a simple awk call:
 
             awk -F'#\\[Out\\]# ' '{if($2) {print $2}}' ipython_log.py
 
           -r: log 'raw' input.  Normally, IPython's logs contain the processed
           input, so that user lines are logged in their final form, converted
           into valid Python.  For example, %Exit is logged as
           '_ip.magic("Exit").  If the -r flag is given, all input is logged
           exactly as typed, with no transformations applied.
 
           -t: put timestamps before each input line logged (these are put in
           comments)."""
         
         opts,par = self.parse_options(parameter_s,'ort')
         log_output = 'o' in opts
         log_raw_input = 'r' in opts
         timestamp = 't' in opts
 
         logger = self.shell.logger
 
         # if no args are given, the defaults set in the logger constructor by
         # ipytohn remain valid
         if par:
             try:
                 logfname,logmode = par.split()
             except:
                 logfname = par
                 logmode = 'backup'
         else:
             logfname = logger.logfname
             logmode = logger.logmode
         # put logfname into rc struct as if it had been called on the command
         # line, so it ends up saved in the log header Save it in case we need
         # to restore it...
         old_logfile = self.shell.logfile
         if logfname:
             logfname = os.path.expanduser(logfname)
         self.shell.logfile = logfname
 
         loghead = '# IPython log file\n\n'
         try:
             started  = logger.logstart(logfname,loghead,logmode,
                                        log_output,timestamp,log_raw_input)
         except:
             self.shell.logfile = old_logfile
             warn("Couldn't start log: %s" % sys.exc_info()[1])
         else:
             # log input history up to this point, optionally interleaving
             # output if requested
 
             if timestamp:
                 # disable timestamping for the previous history, since we've
                 # lost those already (no time machine here).
                 logger.timestamp = False
 
             if log_raw_input:
                 input_hist = self.shell.history_manager.input_hist_raw
             else:
                 input_hist = self.shell.history_manager.input_hist_parsed
                 
             if log_output:
                 log_write = logger.log_write
                 output_hist = self.shell.history_manager.output_hist
                 for n in range(1,len(input_hist)-1):
                     log_write(input_hist[n].rstrip())
                     if n in output_hist:
                         log_write(repr(output_hist[n]),'output')
             else:
                 logger.log_write(''.join(input_hist[1:]))
             if timestamp:
                 # re-enable timestamping
                 logger.timestamp = True
                 
             print ('Activating auto-logging. '
                    'Current session state plus future input saved.')
             logger.logstate()
 
     def magic_logstop(self,parameter_s=''):
         """Fully stop logging and close log file.
 
         In order to start logging again, a new %logstart call needs to be made,
         possibly (though not necessarily) with a new filename, mode and other
         options."""
         self.logger.logstop()
 
     def magic_logoff(self,parameter_s=''):
         """Temporarily stop logging.
 
         You must have previously started logging."""
         self.shell.logger.switch_log(0)
         
     def magic_logon(self,parameter_s=''):
         """Restart logging.
 
         This function is for restarting logging which you've temporarily
         stopped with %logoff. For starting logging for the first time, you
         must use the %logstart function, which allows you to specify an
         optional log filename."""
         
         self.shell.logger.switch_log(1)
     
     def magic_logstate(self,parameter_s=''):
         """Print the status of the logging system."""
 
         self.shell.logger.logstate()
         
     def magic_pdb(self, parameter_s=''):
         """Control the automatic calling of the pdb interactive debugger.
 
         Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
         argument it works as a toggle.
 
         When an exception is triggered, IPython can optionally call the
         interactive pdb debugger after the traceback printout. %pdb toggles
         this feature on and off.
 
         The initial state of this feature is set in your ipythonrc
         configuration file (the variable is called 'pdb').
 
         If you want to just activate the debugger AFTER an exception has fired,
         without having to type '%pdb on' and rerunning your code, you can use
         the %debug magic."""
 
         par = parameter_s.strip().lower()
 
         if par:
             try:
                 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
             except KeyError:
                 print ('Incorrect argument. Use on/1, off/0, '
                        'or nothing for a toggle.')
                 return
         else:
             # toggle
             new_pdb = not self.shell.call_pdb
 
         # set on the shell
         self.shell.call_pdb = new_pdb
         print 'Automatic pdb calling has been turned',on_off(new_pdb)
 
     def magic_debug(self, parameter_s=''):
         """Activate the interactive debugger in post-mortem mode.
 
         If an exception has just occurred, this lets you inspect its stack
         frames interactively.  Note that this will always work only on the last
         traceback that occurred, so you must call this quickly after an
         exception that you wish to inspect has fired, because if another one
         occurs, it clobbers the previous one.
 
         If you want IPython to automatically do this on every exception, see
         the %pdb magic for more details.
         """
         self.shell.debugger(force=True)
 
     @testdec.skip_doctest
     def magic_prun(self, parameter_s ='',user_mode=1,
                    opts=None,arg_lst=None,prog_ns=None):
 
         """Run a statement through the python code profiler.
 
         Usage:
           %prun [options] statement
 
         The given statement (which doesn't require quote marks) is run via the
         python profiler in a manner similar to the profile.run() function.
         Namespaces are internally managed to work correctly; profile.run
         cannot be used in IPython because it makes certain assumptions about
         namespaces which do not hold under IPython.
 
         Options:
 
         -l <limit>: you can place restrictions on what or how much of the
         profile gets printed. The limit value can be:
 
           * A string: only information for function names containing this string
           is printed.
 
           * An integer: only these many lines are printed.
 
           * A float (between 0 and 1): this fraction of the report is printed
           (for example, use a limit of 0.4 to see the topmost 40% only).
 
         You can combine several limits with repeated use of the option. For
         example, '-l __init__ -l 5' will print only the topmost 5 lines of
         information about class constructors.
 
         -r: return the pstats.Stats object generated by the profiling. This
         object has all the information about the profile in it, and you can
         later use it for further analysis or in other functions.
 
        -s <key>: sort profile by given key. You can provide more than one key
         by using the option several times: '-s key1 -s key2 -s key3...'. The
         default sorting key is 'time'.
 
         The following is copied verbatim from the profile documentation
         referenced below:
 
         When more than one key is provided, additional keys are used as
         secondary criteria when the there is equality in all keys selected
         before them.
         
         Abbreviations can be used for any key names, as long as the
         abbreviation is unambiguous.  The following are the keys currently
         defined:
 
                 Valid Arg       Meaning
                   "calls"      call count
                   "cumulative" cumulative time
                   "file"       file name
                   "module"     file name
                   "pcalls"     primitive call count
                   "line"       line number
                   "name"       function name
                   "nfl"        name/file/line
                   "stdname"    standard name
                   "time"       internal time
 
         Note that all sorts on statistics are in descending order (placing
         most time consuming items first), where as name, file, and line number
         searches are in ascending order (i.e., alphabetical). The subtle
         distinction between "nfl" and "stdname" is that the standard name is a
         sort of the name as printed, which means that the embedded line
         numbers get compared in an odd way.  For example, lines 3, 20, and 40
         would (if the file names were the same) appear in the string order
         "20" "3" and "40".  In contrast, "nfl" does a numeric compare of the
         line numbers.  In fact, sort_stats("nfl") is the same as
         sort_stats("name", "file", "line").
 
         -T <filename>: save profile results as shown on screen to a text
         file. The profile is still shown on screen.
 
         -D <filename>: save (via dump_stats) profile statistics to given
         filename. This data is in a format understod by the pstats module, and
         is generated by a call to the dump_stats() method of profile
         objects. The profile is still shown on screen.
 
         If you want to run complete programs under the profiler's control, use
         '%run -p [prof_opts] filename.py [args to program]' where prof_opts
         contains profiler specific options as described here.
         
         You can read the complete documentation for the profile module with::
         
           In [1]: import profile; profile.help()
         """
 
         opts_def = Struct(D=[''],l=[],s=['time'],T=[''])
         # protect user quote marks
         parameter_s = parameter_s.replace('"',r'\"').replace("'",r"\'")
         
         if user_mode:  # regular user call
             opts,arg_str = self.parse_options(parameter_s,'D:l:rs:T:',
                                               list_all=1)
             namespace = self.shell.user_ns
         else:  # called to run a program by %run -p
             try:
                 filename = get_py_filename(arg_lst[0])
             except IOError,msg:
                 error(msg)
                 return
 
             arg_str = 'execfile(filename,prog_ns)'
             namespace = locals()
 
         opts.merge(opts_def)
         
         prof = profile.Profile()
         try:
             prof = prof.runctx(arg_str,namespace,namespace)
             sys_exit = ''
         except SystemExit:
             sys_exit = """*** SystemExit exception caught in code being profiled."""
 
         stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
 
         lims = opts.l
         if lims:
             lims = []  # rebuild lims with ints/floats/strings
             for lim in opts.l:
                 try:
                     lims.append(int(lim))
                 except ValueError:
                     try:
                         lims.append(float(lim))
                     except ValueError:
                         lims.append(lim)
                     
         # Trap output.
         stdout_trap = StringIO()
 
         if hasattr(stats,'stream'):
             # In newer versions of python, the stats object has a 'stream'
             # attribute to write into.
             stats.stream = stdout_trap
             stats.print_stats(*lims)
         else:
             # For older versions, we manually redirect stdout during printing
             sys_stdout = sys.stdout
             try:
                 sys.stdout = stdout_trap
                 stats.print_stats(*lims)
             finally:
                 sys.stdout = sys_stdout
                 
         output = stdout_trap.getvalue()
         output = output.rstrip()
 
         page.page(output)
         print sys_exit,
 
         dump_file = opts.D[0]
         text_file = opts.T[0]
         if dump_file:
             prof.dump_stats(dump_file)
             print '\n*** Profile stats marshalled to file',\
                   `dump_file`+'.',sys_exit
         if text_file:
             pfile = file(text_file,'w')
             pfile.write(output)
             pfile.close()
             print '\n*** Profile printout saved to text file',\
                   `text_file`+'.',sys_exit
 
         if opts.has_key('r'):
             return stats
         else:
             return None
 
     @testdec.skip_doctest
     def magic_run(self, parameter_s ='',runner=None,
                   file_finder=get_py_filename):
         """Run the named file inside IPython as a program.
 
         Usage:\\
           %run [-n -i -t [-N<N>] -d [-b<N>] -p [profile options]] file [args]
         
         Parameters after the filename are passed as command-line arguments to
         the program (put in sys.argv). Then, control returns to IPython's
         prompt.
 
         This is similar to running at a system prompt:\\
           $ python file args\\
         but with the advantage of giving you IPython's tracebacks, and of
         loading all variables into your interactive namespace for further use
         (unless -p is used, see below).
 
         The file is executed in a namespace initially consisting only of
         __name__=='__main__' and sys.argv constructed as indicated. It thus
         sees its environment as if it were being run as a stand-alone program
         (except for sharing global objects such as previously imported
         modules). But after execution, the IPython interactive namespace gets
         updated with all variables defined in the program (except for __name__
         and sys.argv). This allows for very convenient loading of code for
         interactive work, while giving each program a 'clean sheet' to run in.
 
         Options:
         
         -n: __name__ is NOT set to '__main__', but to the running file's name
         without extension (as python does under import).  This allows running
         scripts and reloading the definitions in them without calling code
         protected by an ' if __name__ == "__main__" ' clause.
 
         -i: run the file in IPython's namespace instead of an empty one. This
         is useful if you are experimenting with code written in a text editor
         which depends on variables defined interactively.
 
         -e: ignore sys.exit() calls or SystemExit exceptions in the script
         being run.  This is particularly useful if IPython is being used to
         run unittests, which always exit with a sys.exit() call.  In such
         cases you are interested in the output of the test results, not in
         seeing a traceback of the unittest module.
 
         -t: print timing information at the end of the run.  IPython will give
         you an estimated CPU time consumption for your script, which under
         Unix uses the resource module to avoid the wraparound problems of
         time.clock().  Under Unix, an estimate of time spent on system tasks
         is also given (for Windows platforms this is reported as 0.0).
 
         If -t is given, an additional -N<N> option can be given, where <N>
         must be an integer indicating how many times you want the script to
         run.  The final timing report will include total and per run results.
 
         For example (testing the script uniq_stable.py):
 
             In [1]: run -t uniq_stable
 
             IPython CPU timings (estimated):\\
               User  :    0.19597 s.\\
               System:        0.0 s.\\
 
             In [2]: run -t -N5 uniq_stable
 
             IPython CPU timings (estimated):\\
             Total runs performed: 5\\
               Times :      Total       Per run\\
               User  :   0.910862 s,  0.1821724 s.\\
               System:        0.0 s,        0.0 s.
 
         -d: run your program under the control of pdb, the Python debugger.
         This allows you to execute your program step by step, watch variables,
         etc.  Internally, what IPython does is similar to calling:
         
           pdb.run('execfile("YOURFILENAME")')
 
         with a breakpoint set on line 1 of your file.  You can change the line
         number for this automatic breakpoint to be <N> by using the -bN option
         (where N must be an integer).  For example:
 
           %run -d -b40 myscript
 
         will set the first breakpoint at line 40 in myscript.py.  Note that
         the first breakpoint must be set on a line which actually does
         something (not a comment or docstring) for it to stop execution.
 
         When the pdb debugger starts, you will see a (Pdb) prompt.  You must
         first enter 'c' (without qoutes) to start execution up to the first
         breakpoint.
 
         Entering 'help' gives information about the use of the debugger.  You
         can easily see pdb's full documentation with "import pdb;pdb.help()"
         at a prompt.
 
         -p: run program under the control of the Python profiler module (which
         prints a detailed report of execution times, function calls, etc).
 
         You can pass other options after -p which affect the behavior of the
         profiler itself. See the docs for %prun for details.
 
         In this mode, the program's variables do NOT propagate back to the
         IPython interactive namespace (because they remain in the namespace
         where the profiler executes them).
 
         Internally this triggers a call to %prun, see its documentation for
         details on the options available specifically for profiling.
 
         There is one special usage for which the text above doesn't apply:
         if the filename ends with .ipy, the file is run as ipython script,
         just as if the commands were written on IPython prompt.
         """
 
         # get arguments and set sys.argv for program to be run.
         opts,arg_lst = self.parse_options(parameter_s,'nidtN:b:pD:l:rs:T:e',
                                           mode='list',list_all=1)
 
         try:
             filename = file_finder(arg_lst[0])
         except IndexError:
             warn('you must provide at least a filename.')
             print '\n%run:\n',oinspect.getdoc(self.magic_run)
             return
         except IOError,msg:
             error(msg)
             return
 
         if filename.lower().endswith('.ipy'):
             self.shell.safe_execfile_ipy(filename)
             return
         
         # Control the response to exit() calls made by the script being run
         exit_ignore = opts.has_key('e')
         
         # Make sure that the running script gets a proper sys.argv as if it
         # were run from a system shell.
         save_argv = sys.argv # save it for later restoring
         sys.argv = [filename]+ arg_lst[1:]  # put in the proper filename
 
         if opts.has_key('i'):
             # Run in user's interactive namespace
             prog_ns = self.shell.user_ns
             __name__save = self.shell.user_ns['__name__']
             prog_ns['__name__'] = '__main__'
             main_mod = self.shell.new_main_mod(prog_ns)
         else:
             # Run in a fresh, empty namespace
             if opts.has_key('n'):
                 name = os.path.splitext(os.path.basename(filename))[0]
             else:
                 name = '__main__'
 
             main_mod = self.shell.new_main_mod()
             prog_ns = main_mod.__dict__
             prog_ns['__name__'] = name
 
         # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
         # set the __file__ global in the script's namespace
         prog_ns['__file__'] = filename
 
         # pickle fix.  See interactiveshell for an explanation.  But we need to make sure
         # that, if we overwrite __main__, we replace it at the end
         main_mod_name = prog_ns['__name__']
 
         if main_mod_name == '__main__':
             restore_main = sys.modules['__main__']
         else:
             restore_main = False
 
         # This needs to be undone at the end to prevent holding references to
         # every single object ever created.
         sys.modules[main_mod_name] = main_mod
         
         stats = None
         try:
             self.shell.save_history()
 
             if opts.has_key('p'):
                 stats = self.magic_prun('',0,opts,arg_lst,prog_ns)
             else:
                 if opts.has_key('d'):
                     deb = debugger.Pdb(self.shell.colors)
                     # reset Breakpoint state, which is moronically kept
                     # in a class
                     bdb.Breakpoint.next = 1
                     bdb.Breakpoint.bplist = {}
                     bdb.Breakpoint.bpbynumber = [None]
                     # Set an initial breakpoint to stop execution
                     maxtries = 10
                     bp = int(opts.get('b',[1])[0])
                     checkline = deb.checkline(filename,bp)
                     if not checkline:
                         for bp in range(bp+1,bp+maxtries+1):
                             if deb.checkline(filename,bp):
                                 break
                         else:
                             msg = ("\nI failed to find a valid line to set "
                                    "a breakpoint\n"
                                    "after trying up to line: %s.\n"
                                    "Please set a valid breakpoint manually "
                                    "with the -b option." % bp)
                             error(msg)
                             return
                     # if we find a good linenumber, set the breakpoint
                     deb.do_break('%s:%s' % (filename,bp))
                     # Start file run
                     print "NOTE: Enter 'c' at the",
                     print "%s prompt to start your script." % deb.prompt
                     try:
                         deb.run('execfile("%s")' % filename,prog_ns)
                         
                     except:
                         etype, value, tb = sys.exc_info()
                         # Skip three frames in the traceback: the %run one,
                         # one inside bdb.py, and the command-line typed by the
                         # user (run by exec in pdb itself).
                         self.shell.InteractiveTB(etype,value,tb,tb_offset=3)
                 else:
                     if runner is None:
                         runner = self.shell.safe_execfile
                     if opts.has_key('t'):
                         # timed execution
                         try:
                             nruns = int(opts['N'][0])
                             if nruns < 1:
                                 error('Number of runs must be >=1')
                                 return
                         except (KeyError):
                             nruns = 1
                         if nruns == 1:
                             t0 = clock2()
                             runner(filename,prog_ns,prog_ns,
                                    exit_ignore=exit_ignore)
                             t1 = clock2()
                             t_usr = t1[0]-t0[0]
                             t_sys = t1[1]-t0[1]
                             print "\nIPython CPU timings (estimated):"
                             print "  User  : %10s s." % t_usr
                             print "  System: %10s s." % t_sys
                         else:
                             runs = range(nruns)
                             t0 = clock2()
                             for nr in runs:
                                 runner(filename,prog_ns,prog_ns,
                                        exit_ignore=exit_ignore)
                             t1 = clock2()
                             t_usr = t1[0]-t0[0]
                             t_sys = t1[1]-t0[1]
                             print "\nIPython CPU timings (estimated):"
                             print "Total runs performed:",nruns
                             print "  Times : %10s    %10s" % ('Total','Per run')
                             print "  User  : %10s s, %10s s." % (t_usr,t_usr/nruns)
                             print "  System: %10s s, %10s s." % (t_sys,t_sys/nruns)
                             
                     else:
                         # regular execution
                         runner(filename,prog_ns,prog_ns,exit_ignore=exit_ignore)
 
                 if opts.has_key('i'):
                     self.shell.user_ns['__name__'] = __name__save
                 else:
                     # The shell MUST hold a reference to prog_ns so after %run
                     # exits, the python deletion mechanism doesn't zero it out
                     # (leaving dangling references).
                     self.shell.cache_main_mod(prog_ns,filename)
                     # update IPython interactive namespace
 
                     # Some forms of read errors on the file may mean the
                     # __name__ key was never set; using pop we don't have to
                     # worry about a possible KeyError.
                     prog_ns.pop('__name__', None)
 
                     self.shell.user_ns.update(prog_ns)
         finally:
             # It's a bit of a mystery why, but __builtins__ can change from
             # being a module to becoming a dict missing some key data after
             # %run.  As best I can see, this is NOT something IPython is doing
             # at all, and similar problems have been reported before:
             # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
             # Since this seems to be done by the interpreter itself, the best
             # we can do is to at least restore __builtins__ for the user on
             # exit.
             self.shell.user_ns['__builtins__'] = __builtin__
             
             # Ensure key global structures are restored
             sys.argv = save_argv
             if restore_main:
                 sys.modules['__main__'] = restore_main
             else:
                 # Remove from sys.modules the reference to main_mod we'd
                 # added.  Otherwise it will trap references to objects
                 # contained therein.
                 del sys.modules[main_mod_name]
 
             self.shell.reload_history()
                 
         return stats
 
     @testdec.skip_doctest
     def magic_timeit(self, parameter_s =''):
         """Time execution of a Python statement or expression
 
         Usage:\\
           %timeit [-n<N> -r<R> [-t|-c]] statement
 
         Time execution of a Python statement or expression using the timeit
         module.
 
         Options:
         -n<N>: execute the given statement <N> times in a loop. If this value
         is not given, a fitting value is chosen. 
         
         -r<R>: repeat the loop iteration <R> times and take the best result.
         Default: 3
         
         -t: use time.time to measure the time, which is the default on Unix.
         This function measures wall time.
         
         -c: use time.clock to measure the time, which is the default on
         Windows and measures wall time. On Unix, resource.getrusage is used
         instead and returns the CPU user time.
 
         -p<P>: use a precision of <P> digits to display the timing result.
         Default: 3
 
         
         Examples:
 
           In [1]: %timeit pass
           10000000 loops, best of 3: 53.3 ns per loop
 
           In [2]: u = None
 
           In [3]: %timeit u is None
           10000000 loops, best of 3: 184 ns per loop
 
           In [4]: %timeit -r 4 u == None
           1000000 loops, best of 4: 242 ns per loop
 
           In [5]: import time
 
           In [6]: %timeit -n1 time.sleep(2)
           1 loops, best of 3: 2 s per loop
           
 
         The times reported by %timeit will be slightly higher than those
         reported by the timeit.py script when variables are accessed. This is
         due to the fact that %timeit executes the statement in the namespace
         of the shell, compared with timeit.py, which uses a single setup
         statement to import function or create variables. Generally, the bias
         does not matter as long as results from timeit.py are not mixed with
         those from %timeit."""
 
         import timeit
         import math
 
         # XXX: Unfortunately the unicode 'micro' symbol can cause problems in
         # certain terminals.  Until we figure out a robust way of
         # auto-detecting if the terminal can deal with it, use plain 'us' for
         # microseconds.  I am really NOT happy about disabling the proper
         # 'micro' prefix, but crashing is worse... If anyone knows what the
         # right solution for this is, I'm all ears...
         #
         # Note: using
         #
         # s = u'\xb5'
         # s.encode(sys.getdefaultencoding())
         #
         # is not sufficient, as I've seen terminals where that fails but
         # print s
         #
         # succeeds
         #
         # See bug: https://bugs.launchpad.net/ipython/+bug/348466
         
         #units = [u"s", u"ms",u'\xb5',"ns"]
         units = [u"s", u"ms",u'us',"ns"]
         
         scaling = [1, 1e3, 1e6, 1e9]
 
         opts, stmt = self.parse_options(parameter_s,'n:r:tcp:',
                                         posix=False)
         if stmt == "":
             return
         timefunc = timeit.default_timer
         number = int(getattr(opts, "n", 0))
         repeat = int(getattr(opts, "r", timeit.default_repeat))
         precision = int(getattr(opts, "p", 3))
         if hasattr(opts, "t"):
             timefunc = time.time
         if hasattr(opts, "c"):
             timefunc = clock
 
         timer = timeit.Timer(timer=timefunc)
         # this code has tight coupling to the inner workings of timeit.Timer,
         # but is there a better way to achieve that the code stmt has access
         # to the shell namespace?
 
         src = timeit.template % {'stmt': timeit.reindent(stmt, 8),
                                  'setup': "pass"}
         # Track compilation time so it can be reported if too long
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
         
         t0 = clock()
         code = compile(src, "<magic-timeit>", "exec")
         tc = clock()-t0
         
         ns = {}
         exec code in self.shell.user_ns, ns
         timer.inner = ns["inner"]
         
         if number == 0:
             # determine number so that 0.2 <= total time < 2.0
             number = 1
             for i in range(1, 10):
                 if timer.timeit(number) >= 0.2:
                     break
                 number *= 10
         
         best = min(timer.repeat(repeat, number)) / number
 
         if best > 0.0 and best < 1000.0:
             order = min(-int(math.floor(math.log10(best)) // 3), 3)
         elif best >= 1000.0:
             order = 0
         else:
             order = 3
         print u"%d loops, best of %d: %.*g %s per loop" % (number, repeat,
                                                           precision,
                                                           best * scaling[order],
                                                           units[order])
         if tc > tc_min:
             print "Compiler time: %.2f s" % tc
 
     @testdec.skip_doctest
     def magic_time(self,parameter_s = ''):
         """Time execution of a Python statement or expression.
 
         The CPU and wall clock times are printed, and the value of the
         expression (if any) is returned.  Note that under Win32, system time
         is always reported as 0, since it can not be measured.
 
         This function provides very basic timing functionality.  In Python
         2.3, the timeit module offers more control and sophistication, so this
         could be rewritten to use it (patches welcome).
         
         Some examples:
 
           In [1]: time 2**128
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
           Out[1]: 340282366920938463463374607431768211456L
 
           In [2]: n = 1000000
 
           In [3]: time sum(range(n))
           CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
           Wall time: 1.37
           Out[3]: 499999500000L
 
           In [4]: time print 'hello world'
           hello world
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
 
           Note that the time needed by Python to compile the given expression
           will be reported if it is more than 0.1s.  In this example, the
           actual exponentiation is done by Python at compilation time, so while
           the expression can take a noticeable amount of time to compute, that
           time is purely due to the compilation:
 
           In [5]: time 3**9999;
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00 s
 
           In [6]: time 3**999999;
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00 s
           Compiler : 0.78 s
           """
         
         # fail immediately if the given expression can't be compiled
         
         expr = self.shell.prefilter(parameter_s,False)
 
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
         
         try:
             mode = 'eval'
             t0 = clock()
             code = compile(expr,'<timed eval>',mode)
             tc = clock()-t0
         except SyntaxError:
             mode = 'exec'
             t0 = clock()
             code = compile(expr,'<timed exec>',mode)
             tc = clock()-t0
         # skew measurement as little as possible
         glob = self.shell.user_ns
         clk = clock2
         wtime = time.time
         # time execution
         wall_st = wtime()
         if mode=='eval':
             st = clk()
             out = eval(code,glob)
             end = clk()
         else:
             st = clk()
             exec code in glob
             end = clk()
             out = None
         wall_end = wtime()
         # Compute actual times and report
         wall_time = wall_end-wall_st
         cpu_user = end[0]-st[0]
         cpu_sys = end[1]-st[1]
         cpu_tot = cpu_user+cpu_sys
         print "CPU times: user %.2f s, sys: %.2f s, total: %.2f s" % \
               (cpu_user,cpu_sys,cpu_tot)
         print "Wall time: %.2f s" % wall_time
         if tc > tc_min:
             print "Compiler : %.2f s" % tc
         return out
 
     @testdec.skip_doctest
     def magic_macro(self,parameter_s = ''):
         """Define a set of input lines as a macro for future re-execution.
 
         Usage:\\
           %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
         
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed as the
           command line is used instead.
 
         This will define a global variable called `name` which is a string
         made of joining the slices and lines you specify (n1,n2,... numbers
         above) from your input history into a single string. This variable
         acts like an automatic function which re-executes those lines as if
         you had typed them. You just type 'name' at the prompt and the code
         executes.
 
         The notation for indicating number ranges is: n1-n2 means 'use line
         numbers n1,...n2' (the endpoint is included).  That is, '5-7' means
         using the lines numbered 5,6 and 7.
 
         Note: as a 'hidden' feature, you can also use traditional python slice
         notation, where N:M means numbers N through M-1.
 
         For example, if your history contains (%hist prints it):
         
           44: x=1
           45: y=3
           46: z=x+y
           47: print x
           48: a=5
           49: print 'x',x,'y',y
 
         you can create a macro with lines 44 through 47 (included) and line 49
         called my_macro with:
 
           In [55]: %macro my_macro 44-47 49
 
         Now, typing `my_macro` (without quotes) will re-execute all this code
         in one pass.
 
         You don't need to give the line-numbers in order, and any given line
         number can appear multiple times. You can assemble macros with any
         lines from your input history in any order.
 
         The macro is a simple object which holds its value in an attribute,
         but IPython's display system checks for macros and executes them as
         code instead of printing them when you type their name.
 
         You can view a macro's contents by explicitly printing it with:
         
           'print macro_name'.
 
         For one-off cases which DON'T contain magic function calls in them you
         can obtain similar results by explicitly executing slices from your
         input history with:
 
           In [60]: exec In[44:48]+In[49]"""
 
         opts,args = self.parse_options(parameter_s,'r',mode='list')
         if not args:
             macs = [k for k,v in self.shell.user_ns.items() if isinstance(v, Macro)]
             macs.sort()
             return macs
         if len(args) == 1:
             raise UsageError(
                 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
         name,ranges = args[0], args[1:]
         
         #print 'rng',ranges  # dbg
         lines = self.extract_input_slices(ranges,opts.has_key('r'))
         macro = Macro(lines)
         self.shell.define_macro(name, macro)
         print 'Macro `%s` created. To execute, type its name (without quotes).' % name
         print 'Macro contents:'
         print macro,
 
     def magic_save(self,parameter_s = ''):
         """Save a set of lines to a given filename.
 
         Usage:\\
           %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
         
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed as the
           command line is used instead.
 
         This function uses the same syntax as %macro for line extraction, but
         instead of creating a macro it saves the resulting string to the
         filename you specify.
 
         It adds a '.py' extension to the file if you don't do so yourself, and
         it asks for confirmation before overwriting existing files."""
 
         opts,args = self.parse_options(parameter_s,'r',mode='list')
         fname,ranges = args[0], args[1:]
         if not fname.endswith('.py'):
             fname += '.py'
         if os.path.isfile(fname):
             ans = raw_input('File `%s` exists. Overwrite (y/[N])? ' % fname)
             if ans.lower() not in ['y','yes']:
                 print 'Operation cancelled.'
                 return
         cmds = ''.join(self.extract_input_slices(ranges,opts.has_key('r')))
         f = file(fname,'w')
         f.write(cmds)
         f.close()
         print 'The following commands were written to file `%s`:' % fname
         print cmds
 
     def _edit_macro(self,mname,macro):
         """open an editor with the macro data in a file"""
         filename = self.shell.mktempfile(macro.value)
         self.shell.hooks.editor(filename)
 
         # and make a new macro object, to replace the old one
         mfile = open(filename)
         mvalue = mfile.read()
         mfile.close()
         self.shell.user_ns[mname] = Macro(mvalue)
 
     def magic_ed(self,parameter_s=''):
         """Alias to %edit."""
         return self.magic_edit(parameter_s)
 
     @testdec.skip_doctest
     def magic_edit(self,parameter_s='',last_call=['','']):
         """Bring up an editor and execute the resulting code.
 
         Usage:
           %edit [options] [args]
 
         %edit runs IPython's editor hook.  The default version of this hook is
         set to call the __IPYTHON__.rc.editor command.  This is read from your
         environment variable $EDITOR.  If this isn't found, it will default to
         vi under Linux/Unix and to notepad under Windows.  See the end of this
         docstring for how to change the editor hook.
 
         You can also set the value of this editor via the command line option
         '-editor' or in your ipythonrc file. This is useful if you wish to use
         specifically for IPython an editor different from your typical default
         (and for Windows users who typically don't set environment variables).
 
         This command allows you to conveniently edit multi-line code right in
         your IPython session.
         
         If called without arguments, %edit opens up an empty editor with a
         temporary file and will execute the contents of this file when you
         close it (don't forget to save it!).
 
 
         Options:
 
         -n <number>: open the editor at a specified line number.  By default,
         the IPython editor hook uses the unix syntax 'editor +N filename', but
         you can configure this by providing your own modified hook if your
         favorite editor supports line-number specifications with a different
         syntax.
         
         -p: this will call the editor with the same data as the previous time
         it was used, regardless of how long ago (in your current session) it
         was.
 
         -r: use 'raw' input.  This option only applies to input taken from the
         user's history.  By default, the 'processed' history is used, so that
         magics are loaded in their transformed version to valid Python.  If
         this option is given, the raw input as typed as the command line is
         used instead.  When you exit the editor, it will be executed by
         IPython's own processor.
         
         -x: do not execute the edited code immediately upon exit. This is
         mainly useful if you are editing programs which need to be called with
         command line arguments, which you can then do using %run.
 
 
         Arguments:
 
         If arguments are given, the following possibilites exist:
 
         - The arguments are numbers or pairs of colon-separated numbers (like
         1 4:8 9). These are interpreted as lines of previous input to be
         loaded into the editor. The syntax is the same of the %macro command.
 
         - If the argument doesn't start with a number, it is evaluated as a
         variable and its contents loaded into the editor. You can thus edit
         any string which contains python code (including the result of
         previous edits).
 
         - If the argument is the name of an object (other than a string),
         IPython will try to locate the file where it was defined and open the
         editor at the point where it is defined. You can use `%edit function`
         to load an editor exactly at the point where 'function' is defined,
         edit it and have the file be executed automatically.
 
         If the object is a macro (see %macro for details), this opens up your
         specified editor with a temporary file containing the macro's data.
         Upon exit, the macro is reloaded with the contents of the file.
 
         Note: opening at an exact line is only supported under Unix, and some
         editors (like kedit and gedit up to Gnome 2.8) do not understand the
         '+NUMBER' parameter necessary for this feature. Good editors like
         (X)Emacs, vi, jed, pico and joe all do.
 
         - If the argument is not found as a variable, IPython will look for a
         file with that name (adding .py if necessary) and load it into the
         editor. It will execute its contents with execfile() when you exit,
         loading any code in the file into your interactive namespace.
 
         After executing your code, %edit will return as output the code you
         typed in the editor (except when it was an existing file). This way
         you can reload the code in further invocations of %edit as a variable,
         via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
         the output.
 
         Note that %edit is also available through the alias %ed.
 
         This is an example of creating a simple function inside the editor and
         then modifying it. First, start up the editor:
 
         In [1]: ed
         Editing... done. Executing edited code...
         Out[1]: 'def foo():n    print "foo() was defined in an editing session"n'
 
         We can then call the function foo():
         
         In [2]: foo()
         foo() was defined in an editing session
 
         Now we edit foo.  IPython automatically loads the editor with the
         (temporary) file where foo() was previously defined:
         
         In [3]: ed foo
         Editing... done. Executing edited code...
 
         And if we call foo() again we get the modified version:
         
         In [4]: foo()
         foo() has now been changed!
 
         Here is an example of how to edit a code snippet successive
         times. First we call the editor:
 
         In [5]: ed
         Editing... done. Executing edited code...
         hello
         Out[5]: "print 'hello'n"
 
         Now we call it again with the previous output (stored in _):
 
         In [6]: ed _
         Editing... done. Executing edited code...
         hello world
         Out[6]: "print 'hello world'n"
 
         Now we call it with the output #8 (stored in _8, also as Out[8]):
 
         In [7]: ed _8
         Editing... done. Executing edited code...
         hello again
         Out[7]: "print 'hello again'n"
 
 
         Changing the default editor hook:
 
         If you wish to write your own editor hook, you can put it in a
         configuration file which you load at startup time.  The default hook
         is defined in the IPython.core.hooks module, and you can use that as a
         starting example for further modifications.  That file also has
         general instructions on how to set a new hook for use once you've
         defined it."""
         
         # FIXME: This function has become a convoluted mess.  It needs a
         # ground-up rewrite with clean, simple logic.
 
         def make_filename(arg):
             "Make a filename from the given args"
             try:
                 filename = get_py_filename(arg)
             except IOError:
                 if args.endswith('.py'):
                     filename = arg
                 else:
                     filename = None
             return filename
 
         # custom exceptions
         class DataIsObject(Exception): pass
 
         opts,args = self.parse_options(parameter_s,'prxn:')
         # Set a few locals from the options for convenience:
         opts_p = opts.has_key('p')
         opts_r = opts.has_key('r')
         
         # Default line number value
         lineno = opts.get('n',None)
 
         if opts_p:
             args = '_%s' % last_call[0]
             if not self.shell.user_ns.has_key(args):
                 args = last_call[1]
             
         # use last_call to remember the state of the previous call, but don't
         # let it be clobbered by successive '-p' calls.
         try:
             last_call[0] = self.shell.displayhook.prompt_count
             if not opts_p:
                 last_call[1] = parameter_s
         except:
             pass
 
         # by default this is done with temp files, except when the given
         # arg is a filename
         use_temp = 1
 
         if re.match(r'\d',args):
             # Mode where user specifies ranges of lines, like in %macro.
             # This means that you can't edit files whose names begin with
             # numbers this way. Tough.
             ranges = args.split()
             data = ''.join(self.extract_input_slices(ranges,opts_r))
         elif args.endswith('.py'):
             filename = make_filename(args)
             data = ''
             use_temp = 0
         elif args:
             try:
                 # Load the parameter given as a variable. If not a string,
                 # process it as an object instead (below)
 
                 #print '*** args',args,'type',type(args)  # dbg
                 data = eval(args,self.shell.user_ns)
                 if not type(data) in StringTypes:
                     raise DataIsObject
 
             except (NameError,SyntaxError):
                 # given argument is not a variable, try as a filename
                 filename = make_filename(args)
                 if filename is None:
                     warn("Argument given (%s) can't be found as a variable "
                          "or as a filename." % args)
                     return
 
                 data = ''
                 use_temp = 0
             except DataIsObject:
 
                 # macros have a special edit function
                 if isinstance(data,Macro):
                     self._edit_macro(args,data)
                     return
                                 
                 # For objects, try to edit the file where they are defined
                 try:
                     filename = inspect.getabsfile(data)
                     if 'fakemodule' in filename.lower() and inspect.isclass(data):                     
                         # class created by %edit? Try to find source
                         # by looking for method definitions instead, the
                         # __module__ in those classes is FakeModule.
                         attrs = [getattr(data, aname) for aname in dir(data)]
                         for attr in attrs:
                             if not inspect.ismethod(attr):
                                 continue
                             filename = inspect.getabsfile(attr)
                             if filename and 'fakemodule' not in filename.lower():
                                 # change the attribute to be the edit target instead
                                 data = attr 
                                 break
                     
                     datafile = 1
                 except TypeError:
                     filename = make_filename(args)
                     datafile = 1
                     warn('Could not find file where `%s` is defined.\n'
                          'Opening a file named `%s`' % (args,filename))
                 # Now, make sure we can actually read the source (if it was in
                 # a temp file it's gone by now).
                 if datafile:
                     try:
                         if lineno is None:
                             lineno = inspect.getsourcelines(data)[1]
                     except IOError:
                         filename = make_filename(args)
                         if filename is None:
                             warn('The file `%s` where `%s` was defined cannot '
                                  'be read.' % (filename,data))
                             return
                 use_temp = 0
         else:
             data = ''
 
         if use_temp:
             filename = self.shell.mktempfile(data)
             print 'IPython will make a temporary file named:',filename
 
         # do actual editing here
         print 'Editing...',
         sys.stdout.flush()
         try:
             # Quote filenames that may have spaces in them
             if ' ' in filename:
                 filename = "%s" % filename
             self.shell.hooks.editor(filename,lineno)
         except TryNext:
             warn('Could not open editor')
             return
         
         # XXX TODO: should this be generalized for all string vars?
         # For now, this is special-cased to blocks created by cpaste
         if args.strip() == 'pasted_block':
             self.shell.user_ns['pasted_block'] = file_read(filename)
         
         if opts.has_key('x'):  # -x prevents actual execution
             print
         else:
             print 'done. Executing edited code...'
             if opts_r:
                 self.shell.run_cell(file_read(filename))
             else:
                 self.shell.safe_execfile(filename,self.shell.user_ns,
                                          self.shell.user_ns)
         
                                                      
         if use_temp:
             try:
                 return open(filename).read()
             except IOError,msg:
                 if msg.filename == filename:
                     warn('File not found. Did you forget to save?')
                     return
                 else:
                     self.shell.showtraceback()
 
     def magic_xmode(self,parameter_s = ''):
         """Switch modes for the exception handlers.
 
         Valid modes: Plain, Context and Verbose.
 
         If called without arguments, acts as a toggle."""
 
         def xmode_switch_err(name):
             warn('Error changing %s exception modes.\n%s' %
                  (name,sys.exc_info()[1]))
 
         shell = self.shell
         new_mode = parameter_s.strip().capitalize()
         try:
             shell.InteractiveTB.set_mode(mode=new_mode)
             print 'Exception reporting mode:',shell.InteractiveTB.mode
         except:
             xmode_switch_err('user')
 
     def magic_colors(self,parameter_s = ''):
         """Switch color scheme for prompts, info system and exception handlers.
 
         Currently implemented schemes: NoColor, Linux, LightBG.
 
         Color scheme names are not case-sensitive.
         
         Examples
         --------
         To get a plain black and white terminal::
         
           %colors nocolor
         """
 
         def color_switch_err(name):
             warn('Error changing %s color schemes.\n%s' %
                  (name,sys.exc_info()[1]))
             
         
         new_scheme = parameter_s.strip()
         if not new_scheme:
             raise UsageError(
                 "%colors: you must specify a color scheme. See '%colors?'")
             return
         # local shortcut
         shell = self.shell
 
         import IPython.utils.rlineimpl as readline
 
         if not readline.have_readline and sys.platform == "win32":
             msg = """\
 Proper color support under MS Windows requires the pyreadline library.
 You can find it at:
 http://ipython.scipy.org/moin/PyReadline/Intro
 Gary's readline needs the ctypes module, from:
 http://starship.python.net/crew/theller/ctypes
 (Note that ctypes is already part of Python versions 2.5 and newer).
 
 Defaulting color scheme to 'NoColor'"""
             new_scheme = 'NoColor'
             warn(msg)
         
         # readline option is 0
         if not shell.has_readline:
             new_scheme = 'NoColor'
             
         # Set prompt colors
         try:
             shell.displayhook.set_colors(new_scheme)
         except:
             color_switch_err('prompt')
         else:
             shell.colors = \
                        shell.displayhook.color_table.active_scheme_name
         # Set exception colors
         try:
             shell.InteractiveTB.set_colors(scheme = new_scheme)
             shell.SyntaxTB.set_colors(scheme = new_scheme)
         except:
             color_switch_err('exception')
 
         # Set info (for 'object?') colors
         if shell.color_info:
             try:
                 shell.inspector.set_active_scheme(new_scheme)
             except:
                 color_switch_err('object inspector')
         else:
             shell.inspector.set_active_scheme('NoColor')
                 
     def magic_pprint(self, parameter_s=''):
         """Toggle pretty printing on/off."""
         ptformatter = self.shell.display_formatter.formatters['text/plain']
         ptformatter.pprint = bool(1 - ptformatter.pprint)
         print 'Pretty printing has been turned', \
               ['OFF','ON'][ptformatter.pprint]
                 
     def magic_Exit(self, parameter_s=''):
         """Exit IPython."""
 
         self.shell.ask_exit()
 
     # Add aliases as magics so all common forms work: exit, quit, Exit, Quit.
     magic_exit = magic_quit = magic_Quit = magic_Exit
 
     #......................................................................
     # Functions to implement unix shell-type things
 
     @testdec.skip_doctest
     def magic_alias(self, parameter_s = ''):
         """Define an alias for a system command.
 
         '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
 
         Then, typing 'alias_name params' will execute the system command 'cmd
         params' (from your underlying operating system).
 
         Aliases have lower precedence than magic functions and Python normal
         variables, so if 'foo' is both a Python variable and an alias, the
         alias can not be executed until 'del foo' removes the Python variable.
 
         You can use the %l specifier in an alias definition to represent the
         whole line when the alias is called.  For example:
 
           In [2]: alias bracket echo "Input in brackets: <%l>"
           In [3]: bracket hello world
           Input in brackets: <hello world>
 
         You can also define aliases with parameters using %s specifiers (one
         per parameter):
         
           In [1]: alias parts echo first %s second %s
           In [2]: %parts A B
           first A second B
           In [3]: %parts A
           Incorrect number of arguments: 2 expected.
           parts is an alias to: 'echo first %s second %s'
 
         Note that %l and %s are mutually exclusive.  You can only use one or
         the other in your aliases.
 
         Aliases expand Python variables just like system calls using ! or !! 
         do: all expressions prefixed with '$' get expanded.  For details of
         the semantic rules, see PEP-215:
         http://www.python.org/peps/pep-0215.html.  This is the library used by
         IPython for variable expansion.  If you want to access a true shell
         variable, an extra $ is necessary to prevent its expansion by IPython:
 
         In [6]: alias show echo
         In [7]: PATH='A Python string'
         In [8]: show $PATH
         A Python string
         In [9]: show $$PATH
         /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
 
         You can use the alias facility to acess all of $PATH.  See the %rehash
         and %rehashx functions, which automatically create aliases for the
         contents of your $PATH.
 
         If called with no parameters, %alias prints the current alias table."""
 
         par = parameter_s.strip()
         if not par:
             stored = self.db.get('stored_aliases', {} )
             aliases = sorted(self.shell.alias_manager.aliases)
             # for k, v in stored:
             #     atab.append(k, v[0])
 
             print "Total number of aliases:", len(aliases)
             sys.stdout.flush()
             return aliases
         
         # Now try to define a new one
         try:
             alias,cmd = par.split(None, 1)
         except:
             print oinspect.getdoc(self.magic_alias)
         else:
             self.shell.alias_manager.soft_define_alias(alias, cmd)
     # end magic_alias
 
     def magic_unalias(self, parameter_s = ''):
         """Remove an alias"""
 
         aname = parameter_s.strip()
         self.shell.alias_manager.undefine_alias(aname)
         stored = self.db.get('stored_aliases', {} )
         if aname in stored:
             print "Removing %stored alias",aname
             del stored[aname]
             self.db['stored_aliases'] = stored
 
     def magic_rehashx(self, parameter_s = ''):
         """Update the alias table with all executable files in $PATH.
 
         This version explicitly checks that every entry in $PATH is a file
         with execute access (os.X_OK), so it is much slower than %rehash.
 
         Under Windows, it checks executability as a match agains a
         '|'-separated string of extensions, stored in the IPython config
         variable win_exec_ext.  This defaults to 'exe|com|bat'.
         
         This function also resets the root module cache of module completer,
         used on slow filesystems.
         """
         from IPython.core.alias import InvalidAliasError
 
         # for the benefit of module completer in ipy_completers.py
         del self.db['rootmodules']
         
         path = [os.path.abspath(os.path.expanduser(p)) for p in 
             os.environ.get('PATH','').split(os.pathsep)]
         path = filter(os.path.isdir,path)
 
         syscmdlist = []
         # Now define isexec in a cross platform manner.
         if os.name == 'posix':
             isexec = lambda fname:os.path.isfile(fname) and \
                      os.access(fname,os.X_OK)
         else:
             try:
                 winext = os.environ['pathext'].replace(';','|').replace('.','')
             except KeyError:
                 winext = 'exe|com|bat|py'
             if 'py' not in winext:
                 winext += '|py'
             execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
             isexec = lambda fname:os.path.isfile(fname) and execre.match(fname)
         savedir = os.getcwd()
 
         # Now walk the paths looking for executables to alias.
         try:
             # write the whole loop for posix/Windows so we don't have an if in
             # the innermost part
             if os.name == 'posix':
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):
                         if isexec(ff):
                             try:
                                 # Removes dots from the name since ipython
                                 # will assume names with dots to be python.
                                 self.shell.alias_manager.define_alias(
                                     ff.replace('.',''), ff)
                             except InvalidAliasError:
                                 pass
                             else:
                                 syscmdlist.append(ff)
             else:
                 no_alias = self.shell.alias_manager.no_alias
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):
                         base, ext = os.path.splitext(ff)
                         if isexec(ff) and base.lower() not in no_alias:
                             if ext.lower() == '.exe':
                                 ff = base
                                 try:
                                     # Removes dots from the name since ipython
                                     # will assume names with dots to be python.
                                     self.shell.alias_manager.define_alias(
                                         base.lower().replace('.',''), ff)
                                 except InvalidAliasError:
                                     pass
                                 syscmdlist.append(ff)
             db = self.db
             db['syscmdlist'] = syscmdlist
         finally:
             os.chdir(savedir)
     
     @testdec.skip_doctest    
     def magic_pwd(self, parameter_s = ''):
         """Return the current working directory path.
         
         Examples
         --------
         ::
         
           In [9]: pwd
           Out[9]: '/home/tsuser/sprint/ipython'
         """
         return os.getcwd()
     
     @testdec.skip_doctest
     def magic_cd(self, parameter_s=''):
         """Change the current working directory.
 
         This command automatically maintains an internal list of directories
         you visit during your IPython session, in the variable _dh. The
         command %dhist shows this history nicely formatted. You can also
         do 'cd -<tab>' to see directory history conveniently.
 
         Usage:
 
           cd 'dir': changes to directory 'dir'.
 
           cd -: changes to the last visited directory.
 
           cd -<n>: changes to the n-th directory in the directory history.
 
           cd --foo: change to directory that matches 'foo' in history
             
           cd -b <bookmark_name>: jump to a bookmark set by %bookmark
              (note: cd <bookmark_name> is enough if there is no
               directory <bookmark_name>, but a bookmark with the name exists.)
               'cd -b <tab>' allows you to tab-complete bookmark names. 
 
         Options:
 
         -q: quiet.  Do not print the working directory after the cd command is
         executed.  By default IPython's cd command does print this directory,
         since the default prompts do not display path information.
         
         Note that !cd doesn't work for this purpose because the shell where
         !command runs is immediately discarded after executing 'command'.
         
         Examples
         --------
         ::
         
           In [10]: cd parent/child
           /home/tsuser/parent/child
         """
 
         parameter_s = parameter_s.strip()
         #bkms = self.shell.persist.get("bookmarks",{})
 
         oldcwd = os.getcwd()
         numcd = re.match(r'(-)(\d+)$',parameter_s)
         # jump in directory history by number
         if numcd:
             nn = int(numcd.group(2))
             try:
                 ps = self.shell.user_ns['_dh'][nn]
             except IndexError:
                 print 'The requested directory does not exist in history.'
                 return
             else:
                 opts = {}
         elif parameter_s.startswith('--'):
             ps = None
             fallback = None
             pat = parameter_s[2:]
             dh = self.shell.user_ns['_dh']
             # first search only by basename (last component)
             for ent in reversed(dh):
                 if pat in os.path.basename(ent) and os.path.isdir(ent):
                     ps = ent
                     break
             
                 if fallback is None and pat in ent and os.path.isdir(ent):
                     fallback = ent
                 
             # if we have no last part match, pick the first full path match
             if ps is None:
                 ps = fallback
             
             if ps is None:
                 print "No matching entry in directory history"
                 return
             else:
                 opts = {}
                 
             
         else:
             #turn all non-space-escaping backslashes to slashes, 
             # for c:\windows\directory\names\
             parameter_s = re.sub(r'\\(?! )','/', parameter_s)            
             opts,ps = self.parse_options(parameter_s,'qb',mode='string')
         # jump to previous
         if ps == '-':
             try:
                 ps = self.shell.user_ns['_dh'][-2]
             except IndexError:
                 raise UsageError('%cd -: No previous directory to change to.')
         # jump to bookmark if needed
         else:
             if not os.path.isdir(ps) or opts.has_key('b'):
                 bkms = self.db.get('bookmarks', {})
             
                 if bkms.has_key(ps):
                     target = bkms[ps]
                     print '(bookmark:%s) -> %s' % (ps,target)
                     ps = target
                 else:
                     if opts.has_key('b'):
                         raise UsageError("Bookmark '%s' not found.  "
                               "Use '%%bookmark -l' to see your bookmarks." % ps)
             
         # at this point ps should point to the target dir
         if ps:
             try:                
                 os.chdir(os.path.expanduser(ps))
                 if hasattr(self.shell, 'term_title') and self.shell.term_title:
                     set_term_title('IPython: ' + abbrev_cwd())
             except OSError:
                 print sys.exc_info()[1]
             else:
                 cwd = os.getcwd()
                 dhist = self.shell.user_ns['_dh']
                 if oldcwd != cwd:
                     dhist.append(cwd)
                     self.db['dhist'] = compress_dhist(dhist)[-100:]
                 
         else:
             os.chdir(self.shell.home_dir)
             if hasattr(self.shell, 'term_title') and self.shell.term_title:
                 set_term_title('IPython: ' + '~')
             cwd = os.getcwd()
             dhist = self.shell.user_ns['_dh']
             
             if oldcwd != cwd:
                 dhist.append(cwd)
                 self.db['dhist'] = compress_dhist(dhist)[-100:]
         if not 'q' in opts and self.shell.user_ns['_dh']:
             print self.shell.user_ns['_dh'][-1]
 
 
     def magic_env(self, parameter_s=''):
         """List environment variables."""
         
         return os.environ.data
 
     def magic_pushd(self, parameter_s=''):
         """Place the current dir on stack and change directory.
         
         Usage:\\
           %pushd ['dirname']
         """
         
         dir_s = self.shell.dir_stack
         tgt = os.path.expanduser(parameter_s)
         cwd = os.getcwd().replace(self.home_dir,'~')
         if tgt:
             self.magic_cd(parameter_s)
         dir_s.insert(0,cwd)
         return self.magic_dirs()
 
     def magic_popd(self, parameter_s=''):
         """Change to directory popped off the top of the stack.
         """
         if not self.shell.dir_stack:
             raise UsageError("%popd on empty stack")
         top = self.shell.dir_stack.pop(0)
         self.magic_cd(top)
         print "popd ->",top
 
     def magic_dirs(self, parameter_s=''):
         """Return the current directory stack."""
 
         return self.shell.dir_stack
 
     def magic_dhist(self, parameter_s=''):
         """Print your history of visited directories.
 
         %dhist       -> print full history\\
         %dhist n     -> print last n entries only\\
         %dhist n1 n2 -> print entries between n1 and n2 (n1 not included)\\
 
         This history is automatically maintained by the %cd command, and
         always available as the global list variable _dh. You can use %cd -<n>
         to go to directory number <n>.
         
         Note that most of time, you should view directory history by entering
         cd -<TAB>.
         
         """
 
         dh = self.shell.user_ns['_dh']
         if parameter_s:
             try:
                 args = map(int,parameter_s.split())
             except:
                 self.arg_err(Magic.magic_dhist)
                 return
             if len(args) == 1:
                 ini,fin = max(len(dh)-(args[0]),0),len(dh)
             elif len(args) == 2:
                 ini,fin = args
             else:
                 self.arg_err(Magic.magic_dhist)
                 return
         else:
             ini,fin = 0,len(dh)
         nlprint(dh,
                 header = 'Directory history (kept in _dh)',
                 start=ini,stop=fin)
 
     @testdec.skip_doctest
     def magic_sc(self, parameter_s=''):
         """Shell capture - execute a shell command and capture its output.
 
         DEPRECATED. Suboptimal, retained for backwards compatibility.
         
         You should use the form 'var = !command' instead. Example:
          
          "%sc -l myfiles = ls ~" should now be written as
             
          "myfiles = !ls ~"
          
         myfiles.s, myfiles.l and myfiles.n still apply as documented
         below.
 
         --
         %sc [options] varname=command
 
         IPython will run the given command using commands.getoutput(), and
         will then update the user's interactive namespace with a variable
         called varname, containing the value of the call.  Your command can
         contain shell wildcards, pipes, etc.
 
         The '=' sign in the syntax is mandatory, and the variable name you
         supply must follow Python's standard conventions for valid names.
         
         (A special format without variable name exists for internal use)
 
         Options:
 
           -l: list output.  Split the output on newlines into a list before
           assigning it to the given variable.  By default the output is stored
           as a single string.
 
           -v: verbose.  Print the contents of the variable.
 
         In most cases you should not need to split as a list, because the
         returned value is a special type of string which can automatically
         provide its contents either as a list (split on newlines) or as a
         space-separated string.  These are convenient, respectively, either
         for sequential processing or to be passed to a shell command.
 
         For example:
 
         # all-random
         
             # Capture into variable a
             In [1]: sc a=ls *py
 
             # a is a string with embedded newlines
             In [2]: a
             Out[2]: 'setup.py\\nwin32_manual_post_install.py'
 
             # which can be seen as a list:
             In [3]: a.l
             Out[3]: ['setup.py', 'win32_manual_post_install.py']
 
             # or as a whitespace-separated string:
             In [4]: a.s
             Out[4]: 'setup.py win32_manual_post_install.py'
 
             # a.s is useful to pass as a single command line:
             In [5]: !wc -l $a.s
               146 setup.py
               130 win32_manual_post_install.py
               276 total
 
             # while the list form is useful to loop over:
             In [6]: for f in a.l:
               ...:      !wc -l $f
               ...:
             146 setup.py
             130 win32_manual_post_install.py
 
         Similiarly, the lists returned by the -l option are also special, in
         the sense that you can equally invoke the .s attribute on them to
         automatically get a whitespace-separated string from their contents:
 
             In [7]: sc -l b=ls *py
 
             In [8]: b
             Out[8]: ['setup.py', 'win32_manual_post_install.py']
 
             In [9]: b.s
             Out[9]: 'setup.py win32_manual_post_install.py'
 
         In summary, both the lists and strings used for ouptut capture have
         the following special attributes:
 
             .l (or .list) : value as list.
             .n (or .nlstr): value as newline-separated string.
             .s (or .spstr): value as space-separated string.
         """
 
         opts,args = self.parse_options(parameter_s,'lv')
         # Try to get a variable name and command to run
         try:
             # the variable name must be obtained from the parse_options
             # output, which uses shlex.split to strip options out.
             var,_ = args.split('=',1)
             var = var.strip()
             # But the the command has to be extracted from the original input
             # parameter_s, not on what parse_options returns, to avoid the
             # quote stripping which shlex.split performs on it.
             _,cmd = parameter_s.split('=',1)
         except ValueError:
             var,cmd = '',''
         # If all looks ok, proceed
         split = 'l' in opts
         out = self.shell.getoutput(cmd, split=split)
         if opts.has_key('v'):
             print '%s ==\n%s' % (var,pformat(out))
         if var:
             self.shell.user_ns.update({var:out})
         else:
             return out
 
     def magic_sx(self, parameter_s=''):
         """Shell execute - run a shell command and capture its output.
 
         %sx command
 
         IPython will run the given command using commands.getoutput(), and
         return the result formatted as a list (split on '\\n').  Since the
         output is _returned_, it will be stored in ipython's regular output
         cache Out[N] and in the '_N' automatic variables.
 
         Notes:
 
         1) If an input line begins with '!!', then %sx is automatically
         invoked.  That is, while:
           !ls
         causes ipython to simply issue system('ls'), typing
           !!ls
         is a shorthand equivalent to:
           %sx ls
         
         2) %sx differs from %sc in that %sx automatically splits into a list,
         like '%sc -l'.  The reason for this is to make it as easy as possible
         to process line-oriented shell output via further python commands.
         %sc is meant to provide much finer control, but requires more
         typing.
 
         3) Just like %sc -l, this is a list with special attributes:
 
           .l (or .list) : value as list.
           .n (or .nlstr): value as newline-separated string.
           .s (or .spstr): value as whitespace-separated string.
 
         This is very useful when trying to use such lists as arguments to
         system commands."""
 
         if parameter_s:
             return self.shell.getoutput(parameter_s)
 
     def magic_r(self, parameter_s=''):
         """Repeat previous input.
 
         Note: Consider using the more powerfull %rep instead!
         
         If given an argument, repeats the previous command which starts with
         the same string, otherwise it just repeats the previous input.
 
         Shell escaped commands (with ! as first character) are not recognized
         by this system, only pure python code and magic commands.
         """
 
         start = parameter_s.strip()
         esc_magic = ESC_MAGIC
         # Identify magic commands even if automagic is on (which means
         # the in-memory version is different from that typed by the user).
         if self.shell.automagic:
             start_magic = esc_magic+start
         else:
             start_magic = start
         # Look through the input history in reverse
         for n in range(len(self.shell.history_manager.input_hist_parsed)-2,0,-1):
             input = self.shell.history_manager.input_hist_parsed[n]
             # skip plain 'r' lines so we don't recurse to infinity
             if input != '_ip.magic("r")\n' and \
                    (input.startswith(start) or input.startswith(start_magic)):
                 #print 'match',`input`  # dbg
                 print 'Executing:',input,
                 self.shell.run_cell(input)
                 return
         print 'No previous input matching `%s` found.' % start
 
     
     def magic_bookmark(self, parameter_s=''):
         """Manage IPython's bookmark system.
 
         %bookmark <name>       - set bookmark to current dir
         %bookmark <name> <dir> - set bookmark to <dir>
         %bookmark -l           - list all bookmarks
         %bookmark -d <name>    - remove bookmark
         %bookmark -r           - remove all bookmarks
 
         You can later on access a bookmarked folder with:
           %cd -b <name>
         or simply '%cd <name>' if there is no directory called <name> AND
         there is such a bookmark defined.
 
         Your bookmarks persist through IPython sessions, but they are
         associated with each profile."""
 
         opts,args = self.parse_options(parameter_s,'drl',mode='list')
         if len(args) > 2:
             raise UsageError("%bookmark: too many arguments")
 
         bkms = self.db.get('bookmarks',{})
             
         if opts.has_key('d'):
             try:
                 todel = args[0]
             except IndexError:
                 raise UsageError(
                     "%bookmark -d: must provide a bookmark to delete")
             else:
                 try:
                     del bkms[todel]
                 except KeyError:
                     raise UsageError(
                         "%%bookmark -d: Can't delete bookmark '%s'" % todel)
 
         elif opts.has_key('r'):
             bkms = {}
         elif opts.has_key('l'):
             bks = bkms.keys()
             bks.sort()
             if bks:
                 size = max(map(len,bks))
             else:
                 size = 0
             fmt = '%-'+str(size)+'s -> %s'
             print 'Current bookmarks:'
             for bk in bks:
                 print fmt % (bk,bkms[bk])
         else:
             if not args:
                 raise UsageError("%bookmark: You must specify the bookmark name")
             elif len(args)==1:
                 bkms[args[0]] = os.getcwd()
             elif len(args)==2:
                 bkms[args[0]] = args[1]
         self.db['bookmarks'] = bkms
 
     def magic_pycat(self, parameter_s=''):
         """Show a syntax-highlighted file through a pager.
 
         This magic is similar to the cat utility, but it will assume the file
         to be Python source and will show it with syntax highlighting. """
         
         try:
             filename = get_py_filename(parameter_s)
             cont = file_read(filename)
         except IOError:
             try:
                 cont = eval(parameter_s,self.user_ns)
             except NameError:
                 cont = None
         if cont is None:
             print "Error: no such file or variable"
             return
 
         page.page(self.shell.pycolorize(cont))
 
     def _rerun_pasted(self):
         """ Rerun a previously pasted command.
         """
         b = self.user_ns.get('pasted_block', None)
         if b is None:
             raise UsageError('No previous pasted block available')
         print "Re-executing '%s...' (%d chars)"% (b.split('\n',1)[0], len(b))
         exec b in self.user_ns
 
     def _get_pasted_lines(self, sentinel):
         """ Yield pasted lines until the user enters the given sentinel value.
         """
         from IPython.core import interactiveshell
         print "Pasting code; enter '%s' alone on the line to stop." % sentinel
         while True:
             l = interactiveshell.raw_input_original(':')
             if l == sentinel:
                 return
             else:
                 yield l
 
     def _strip_pasted_lines_for_code(self, raw_lines):
         """ Strip non-code parts of a sequence of lines to return a block of
         code.
         """
         # Regular expressions that declare text we strip from the input:
         strip_re =  [r'^\s*In \[\d+\]:', # IPython input prompt
                      r'^\s*(\s?>)+', # Python input prompt
                      r'^\s*\.{3,}', # Continuation prompts
                      r'^\++',
                      ]
 
         strip_from_start = map(re.compile,strip_re)
         
         lines = []
         for l in raw_lines:
             for pat in strip_from_start: 
                 l = pat.sub('',l)
             lines.append(l)
                          
         block = "\n".join(lines) + '\n'
         #print "block:\n",block
         return block
 
     def _execute_block(self, block, par):
         """ Execute a block, or store it in a variable, per the user's request.
         """
         if not par:
             b = textwrap.dedent(block)
             self.user_ns['pasted_block'] = b
             exec b in self.user_ns
         else:
             self.user_ns[par] = SList(block.splitlines())
             print "Block assigned to '%s'" % par
 
     def magic_quickref(self,arg):
         """ Show a quick reference sheet """
         import IPython.core.usage
         qr = IPython.core.usage.quick_reference + self.magic_magic('-brief')
         
         page.page(qr)
 
     def magic_doctest_mode(self,parameter_s=''):
         """Toggle doctest mode on and off.
 
         This mode is intended to make IPython behave as much as possible like a
         plain Python shell, from the perspective of how its prompts, exceptions
         and output look.  This makes it easy to copy and paste parts of a
         session into doctests.  It does so by:
 
         - Changing the prompts to the classic ``>>>`` ones.
         - Changing the exception reporting mode to 'Plain'.
         - Disabling pretty-printing of output.
 
         Note that IPython also supports the pasting of code snippets that have
         leading '>>>' and '...' prompts in them.  This means that you can paste
         doctests from files or docstrings (even if they have leading
         whitespace), and the code will execute correctly.  You can then use
         '%history -t' to see the translated history; this will give you the
         input after removal of all the leading prompts and whitespace, which
         can be pasted back into an editor.
 
         With these features, you can switch into this mode easily whenever you
         need to do testing and changes to doctests, without having to leave
         your existing IPython session.
         """
 
         from IPython.utils.ipstruct import Struct
 
         # Shorthands
         shell = self.shell
         oc = shell.displayhook
         meta = shell.meta
         disp_formatter = self.shell.display_formatter
         ptformatter = disp_formatter.formatters['text/plain']
         # dstore is a data store kept in the instance metadata bag to track any
         # changes we make, so we can undo them later.
         dstore = meta.setdefault('doctest_mode',Struct())
         save_dstore = dstore.setdefault
 
         # save a few values we'll need to recover later
         mode = save_dstore('mode',False)
         save_dstore('rc_pprint',ptformatter.pprint)
         save_dstore('xmode',shell.InteractiveTB.mode)
         save_dstore('rc_separate_out',shell.separate_out)
         save_dstore('rc_separate_out2',shell.separate_out2)
         save_dstore('rc_prompts_pad_left',shell.prompts_pad_left)
         save_dstore('rc_separate_in',shell.separate_in)
         save_dstore('rc_plain_text_only',disp_formatter.plain_text_only)
 
         if mode == False:
             # turn on
             oc.prompt1.p_template = '>>> '
             oc.prompt2.p_template = '... '
             oc.prompt_out.p_template = ''
 
             # Prompt separators like plain python
             oc.input_sep = oc.prompt1.sep = ''
             oc.output_sep = ''
             oc.output_sep2 = ''
 
             oc.prompt1.pad_left = oc.prompt2.pad_left = \
                                   oc.prompt_out.pad_left = False
 
             ptformatter.pprint = False
             disp_formatter.plain_text_only = True
             
             shell.magic_xmode('Plain')
         else:
             # turn off
             oc.prompt1.p_template = shell.prompt_in1
             oc.prompt2.p_template = shell.prompt_in2
             oc.prompt_out.p_template = shell.prompt_out
 
             oc.input_sep = oc.prompt1.sep = dstore.rc_separate_in
 
             oc.output_sep = dstore.rc_separate_out
             oc.output_sep2 = dstore.rc_separate_out2
 
             oc.prompt1.pad_left = oc.prompt2.pad_left = \
                          oc.prompt_out.pad_left = dstore.rc_prompts_pad_left
 
             ptformatter.pprint = dstore.rc_pprint
             disp_formatter.plain_text_only = dstore.rc_plain_text_only
 
             shell.magic_xmode(dstore.xmode)
 
         # Store new mode and inform
         dstore.mode = bool(1-int(mode))
         mode_label = ['OFF','ON'][dstore.mode]
         print 'Doctest mode is:', mode_label
 
     def magic_gui(self, parameter_s=''):
         """Enable or disable IPython GUI event loop integration.
 
         %gui [GUINAME]
 
         This magic replaces IPython's threaded shells that were activated
         using the (pylab/wthread/etc.) command line flags.  GUI toolkits
         can now be enabled, disabled and swtiched at runtime and keyboard
         interrupts should work without any problems.  The following toolkits
         are supported:  wxPython, PyQt4, PyGTK, and Tk::
 
             %gui wx      # enable wxPython event loop integration
             %gui qt4|qt  # enable PyQt4 event loop integration
             %gui gtk     # enable PyGTK event loop integration
             %gui tk      # enable Tk event loop integration
             %gui         # disable all event loop integration
 
         WARNING:  after any of these has been called you can simply create
         an application object, but DO NOT start the event loop yourself, as
         we have already handled that.
         """
         from IPython.lib.inputhook import enable_gui
         opts, arg = self.parse_options(parameter_s, '')
         if arg=='': arg = None
         return enable_gui(arg)
 
     def magic_load_ext(self, module_str):
         """Load an IPython extension by its module name."""
         return self.extension_manager.load_extension(module_str)
 
     def magic_unload_ext(self, module_str):
         """Unload an IPython extension by its module name."""
         self.extension_manager.unload_extension(module_str)
 
     def magic_reload_ext(self, module_str):
         """Reload an IPython extension by its module name."""
         self.extension_manager.reload_extension(module_str)
 
     @testdec.skip_doctest
     def magic_install_profiles(self, s):
         """Install the default IPython profiles into the .ipython dir.
 
         If the default profiles have already been installed, they will not
         be overwritten. You can force overwriting them by using the ``-o``
         option::
 
             In [1]: %install_profiles -o
         """
         if '-o' in s:
             overwrite = True
         else:
             overwrite = False
         from IPython.config import profile
         profile_dir = os.path.split(profile.__file__)[0]
         ipython_dir = self.ipython_dir
         files = os.listdir(profile_dir)
 
         to_install = []
         for f in files:
             if f.startswith('ipython_config'):
                 src = os.path.join(profile_dir, f)
                 dst = os.path.join(ipython_dir, f)
                 if (not os.path.isfile(dst)) or overwrite:
                     to_install.append((f, src, dst))
         if len(to_install)>0:
             print "Installing profiles to: ", ipython_dir
             for (f, src, dst) in to_install:
                 shutil.copy(src, dst)
                 print "    %s" % f
 
     def magic_install_default_config(self, s):
         """Install IPython's default config file into the .ipython dir.
 
         If the default config file (:file:`ipython_config.py`) is already
         installed, it will not be overwritten. You can force overwriting
         by using the ``-o`` option::
 
             In [1]: %install_default_config
         """
         if '-o' in s:
             overwrite = True
         else:
             overwrite = False
         from IPython.config import default
         config_dir = os.path.split(default.__file__)[0]
         ipython_dir = self.ipython_dir
         default_config_file_name = 'ipython_config.py'
         src = os.path.join(config_dir, default_config_file_name)
         dst = os.path.join(ipython_dir, default_config_file_name)
         if (not os.path.isfile(dst)) or overwrite:
             shutil.copy(src, dst)
             print "Installing default config file: %s" % dst
 
     # Pylab support: simple wrappers that activate pylab, load gui input
     # handling and modify slightly %run
 
     @testdec.skip_doctest
     def _pylab_magic_run(self, parameter_s=''):
         Magic.magic_run(self, parameter_s,
                         runner=mpl_runner(self.shell.safe_execfile))
 
     _pylab_magic_run.__doc__ = magic_run.__doc__
 
     @testdec.skip_doctest
     def magic_pylab(self, s):
         """Load numpy and matplotlib to work interactively.
 
         %pylab [GUINAME]
 
         This function lets you activate pylab (matplotlib, numpy and
         interactive support) at any point during an IPython session.
 
         It will import at the top level numpy as np, pyplot as plt, matplotlib,
         pylab and mlab, as well as all names from numpy and pylab.
 
         Parameters
         ----------
         guiname : optional
           One of the valid arguments to the %gui magic ('qt', 'wx', 'gtk' or
           'tk').  If given, the corresponding Matplotlib backend is used,
           otherwise matplotlib's default (which you can override in your
           matplotlib config file) is used.
 
         Examples
         --------
         In this case, where the MPL default is TkAgg:
         In [2]: %pylab
 
         Welcome to pylab, a matplotlib-based Python environment.
         Backend in use: TkAgg
         For more information, type 'help(pylab)'.
 
         But you can explicitly request a different backend:
         In [3]: %pylab qt
 
         Welcome to pylab, a matplotlib-based Python environment.
         Backend in use: Qt4Agg
         For more information, type 'help(pylab)'.
         """
         self.shell.enable_pylab(s)
 
     def magic_tb(self, s):
         """Print the last traceback with the currently active exception mode.
 
         See %xmode for changing exception reporting modes."""
         self.shell.showtraceback()
     
     @testdec.skip_doctest
     def magic_precision(self, s=''):
         """Set floating point precision for pretty printing.
         
         Can set either integer precision or a format string.
         
         If numpy has been imported and precision is an int,
         numpy display precision will also be set, via ``numpy.set_printoptions``.
         
         If no argument is given, defaults will be restored.
         
         Examples
         --------
         ::
         
             In [1]: from math import pi
         
             In [2]: %precision 3
             Out[2]: '%.3f'
         
             In [3]: pi
             Out[3]: 3.142
         
             In [4]: %precision %i
             Out[4]: '%i'
         
             In [5]: pi
             Out[5]: 3
         
             In [6]: %precision %e
             Out[6]: '%e'
         
             In [7]: pi**10
             Out[7]: 9.364805e+04
         
             In [8]: %precision
             Out[8]: '%r'
         
             In [9]: pi**10
             Out[9]: 93648.047476082982
         
         """
         
         ptformatter = self.shell.display_formatter.formatters['text/plain']
         ptformatter.float_precision = s
         return ptformatter.float_format
 
 # end Magic

IPython/core/usage.py

@@ -1,507 +1,508 b''
 # -*- coding: utf-8 -*-
 """Usage information for the main IPython applications.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2010  The IPython Development Team
 #  Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 import sys
 from IPython.core import release
 
 cl_usage = """\
 ipython [options] [files]
 
 IPython: an enhanced interactive Python shell.
 
     A Python shell with automatic history (input and output), dynamic object
     introspection, easier configuration, command completion, access to the
     system shell and more.  IPython can also be embedded in running programs.
 
     If invoked with no options, it executes all the files listed in sequence
     and exits, use -i to enter interactive mode after running the files.  Files
     ending in .py will be treated as normal Python, but files ending in .ipy
     can contain special IPython syntax (magic commands, shell expansions, etc.)
 
     Please note that some of the configuration options are not available at the
     command line, simply because they are not practical here. Look into your
     ipython_config.py configuration file for details on those.
 
-    This file typically installed in the $HOME/.ipython directory.  For Windows
-    users, $HOME resolves to C:\\Documents and Settings\\YourUserName in most
-    instances.
+    This file is typically installed in the IPYTHON_DIR directory. For Linux
+    users, this will be $HOME/.config/ipython, and for other users it will be
+    $HOME/.ipython.  For Windows users, $HOME resolves to C:\\Documents and
+    Settings\\YourUserName in most instances.
 
     In IPython's documentation, we will refer to this directory as IPYTHON_DIR,
     you can change its default location by setting any path you want in this
     environment variable.
 
     For more information, see the manual available in HTML and PDF in your
     installation, or online at http://ipython.scipy.org.
 """
 
 interactive_usage = """
 IPython -- An enhanced Interactive Python
 =========================================
 
 IPython offers a combination of convenient shell features, special commands
 and a history mechanism for both input (command history) and output (results
 caching, similar to Mathematica). It is intended to be a fully compatible
 replacement for the standard Python interpreter, while offering vastly
 improved functionality and flexibility.
 
 At your system command line, type 'ipython -help' to see the command line
 options available. This document only describes interactive features.
 
 Warning: IPython relies on the existence of a global variable called __IP which
 controls the shell itself. If you redefine __IP to anything, bizarre behavior
 will quickly occur.
 
 MAIN FEATURES
 
 * Access to the standard Python help. As of Python 2.1, a help system is
   available with access to object docstrings and the Python manuals. Simply
   type 'help' (no quotes) to access it.
 
 * Magic commands: type %magic for information on the magic subsystem.
 
 * System command aliases, via the %alias command or the ipythonrc config file.
 
 * Dynamic object information:
 
   Typing ?word or word? prints detailed information about an object.  If
   certain strings in the object are too long (docstrings, code, etc.) they get
   snipped in the center for brevity.
 
   Typing ??word or word?? gives access to the full information without
   snipping long strings. Long strings are sent to the screen through the less
   pager if longer than the screen, printed otherwise.
 
   The ?/?? system gives access to the full source code for any object (if
   available), shows function prototypes and other useful information.
 
   If you just want to see an object's docstring, type '%pdoc object' (without
   quotes, and without % if you have automagic on).
 
   Both %pdoc and ?/?? give you access to documentation even on things which are
   not explicitely defined. Try for example typing {}.get? or after import os,
   type os.path.abspath??. The magic functions %pdef, %source and %file operate
   similarly.
 
 * Completion in the local namespace, by typing TAB at the prompt.
 
   At any time, hitting tab will complete any available python commands or
   variable names, and show you a list of the possible completions if there's
   no unambiguous one. It will also complete filenames in the current directory.
 
   This feature requires the readline and rlcomplete modules, so it won't work
   if your Python lacks readline support (such as under Windows).
 
 * Search previous command history in two ways (also requires readline):
 
   - Start typing, and then use Ctrl-p (previous,up) and Ctrl-n (next,down) to
   search through only the history items that match what you've typed so
   far. If you use Ctrl-p/Ctrl-n at a blank prompt, they just behave like
   normal arrow keys.
 
   - Hit Ctrl-r: opens a search prompt. Begin typing and the system searches
   your history for lines that match what you've typed so far, completing as
   much as it can.
 
 * Persistent command history across sessions (readline required).
 
 * Logging of input with the ability to save and restore a working session.
   
 * System escape with !. Typing !ls will run 'ls' in the current directory.
 
 * The reload command does a 'deep' reload of a module: changes made to the
   module since you imported will actually be available without having to exit.
 
 * Verbose and colored exception traceback printouts. See the magic xmode and
   xcolor functions for details (just type %magic).
 
 * Input caching system:
 
   IPython offers numbered prompts (In/Out) with input and output caching. All
   input is saved and can be retrieved as variables (besides the usual arrow
   key recall).
 
   The following GLOBAL variables always exist (so don't overwrite them!):
   _i: stores previous input.
   _ii: next previous.
   _iii: next-next previous.
   _ih : a list of all input _ih[n] is the input from line n.
 
   Additionally, global variables named _i<n> are dynamically created (<n>
   being the prompt counter), such that _i<n> == _ih[<n>]
 
   For example, what you typed at prompt 14 is available as _i14 and _ih[14].
 
   You can create macros which contain multiple input lines from this history,
   for later re-execution, with the %macro function.
 
   The history function %hist allows you to see any part of your input history
   by printing a range of the _i variables. Note that inputs which contain
   magic functions (%) appear in the history with a prepended comment. This is
   because they aren't really valid Python code, so you can't exec them.
 
 * Output caching system:
 
   For output that is returned from actions, a system similar to the input
   cache exists but using _ instead of _i. Only actions that produce a result
   (NOT assignments, for example) are cached. If you are familiar with
   Mathematica, IPython's _ variables behave exactly like Mathematica's %
   variables.
 
   The following GLOBAL variables always exist (so don't overwrite them!):
   _ (one underscore): previous output.
   __ (two underscores): next previous.
   ___ (three underscores): next-next previous.
 
   Global variables named _<n> are dynamically created (<n> being the prompt
   counter), such that the result of output <n> is always available as _<n>.
 
   Finally, a global dictionary named _oh exists with entries for all lines
   which generated output.
 
 * Directory history:
 
   Your history of visited directories is kept in the global list _dh, and the
   magic %cd command can be used to go to any entry in that list.
 
 * Auto-parentheses and auto-quotes (adapted from Nathan Gray's LazyPython)
 
     1. Auto-parentheses
         Callable objects (i.e. functions, methods, etc) can be invoked like
         this (notice the commas between the arguments):
             >>> callable_ob arg1, arg2, arg3
         and the input will be translated to this:
             --> callable_ob(arg1, arg2, arg3)
         You can force auto-parentheses by using '/' as the first character
         of a line.  For example:
             >>> /globals             # becomes 'globals()'
         Note that the '/' MUST be the first character on the line!  This
         won't work:
             >>> print /globals    # syntax error
             
         In most cases the automatic algorithm should work, so you should
         rarely need to explicitly invoke /. One notable exception is if you
         are trying to call a function with a list of tuples as arguments (the
         parenthesis will confuse IPython):
             In [1]: zip (1,2,3),(4,5,6)  # won't work
         but this will work:
             In [2]: /zip (1,2,3),(4,5,6)
             ------> zip ((1,2,3),(4,5,6))
             Out[2]= [(1, 4), (2, 5), (3, 6)]        
 
         IPython tells you that it has altered your command line by
         displaying the new command line preceded by -->.  e.g.:
             In [18]: callable list
             -------> callable (list) 
 
     2. Auto-Quoting
         You can force auto-quoting of a function's arguments by using ',' as
         the first character of a line.  For example:
             >>> ,my_function /home/me   # becomes my_function("/home/me")
 
         If you use ';' instead, the whole argument is quoted as a single
         string (while ',' splits on whitespace):
             >>> ,my_function a b c   # becomes my_function("a","b","c")
             >>> ;my_function a b c   # becomes my_function("a b c")
 
         Note that the ',' MUST be the first character on the line!  This
         won't work:
             >>> x = ,my_function /home/me    # syntax error
 """
 
 interactive_usage_min =  """\
 An enhanced console for Python.
 Some of its features are:
 - Readline support if the readline library is present.
 - Tab completion in the local namespace.
 - Logging of input, see command-line options.
 - System shell escape via ! , eg !ls.
 - Magic commands, starting with a % (like %ls, %pwd, %cd, etc.)
 - Keeps track of locally defined variables via %who, %whos.
 - Show object information with a ? eg ?x or x? (use ?? for more info).
 """
 
 quick_reference = r"""
 IPython -- An enhanced Interactive Python - Quick Reference Card
 ================================================================
 
 obj?, obj??      : Get help, or more help for object (also works as
                    ?obj, ??obj).
 ?foo.*abc*       : List names in 'foo' containing 'abc' in them.
 %magic           : Information about IPython's 'magic' % functions.
 
 Magic functions are prefixed by %, and typically take their arguments without
 parentheses, quotes or even commas for convenience.
  
 Example magic function calls:
 
 %alias d ls -F   : 'd' is now an alias for 'ls -F'
 alias d ls -F    : Works if 'alias' not a python name
 alist = %alias   : Get list of aliases to 'alist'
 cd /usr/share    : Obvious. cd -<tab> to choose from visited dirs.
 %cd??            : See help AND source for magic %cd
 
 System commands:
     
 !cp a.txt b/     : System command escape, calls os.system()
 cp a.txt b/      : after %rehashx, most system commands work without !
 cp ${f}.txt $bar : Variable expansion in magics and system commands
 files = !ls /usr : Capture sytem command output
 files.s, files.l, files.n: "a b c", ['a','b','c'], 'a\nb\nc'
 
 History:
 
 _i, _ii, _iii    : Previous, next previous, next next previous input
 _i4, _ih[2:5]    : Input history line 4, lines 2-4
 exec _i81        : Execute input history line #81 again
 %rep 81          : Edit input history line #81        
 _, __, ___       : previous, next previous, next next previous output
 _dh              : Directory history
 _oh              : Output history
 %hist            : Command history. '%hist -g foo' search history for 'foo'
 
 Autocall:
 
 f 1,2            : f(1,2)
 /f 1,2           : f(1,2) (forced autoparen)
 ,f 1 2           : f("1","2")
 ;f 1 2           : f("1 2")
 
 Remember: TAB completion works in many contexts, not just file names
 or python names.
 
 The following magic functions are currently available:
 
 """
 
 gui_reference = """\
 ===============================
  The graphical IPython console
 ===============================
 
 This console is designed to emulate the look, feel and workflow of a terminal
 environment, while adding a number of enhancements that are simply not possible
 in a real terminal, such as inline syntax highlighting, true multiline editing,
 inline graphics and much more.
 
 This quick reference document contains the basic information you'll need to
 know to make the most efficient use of it.  For the various command line
 options available at startup, type ``--help`` at the command line.
 
 
 Multiline editing
 =================
 
 The graphical console is capable of true multiline editing, but it also tries
 to behave intuitively like a terminal when possible.  If you are used to
 IPyhton's old terminal behavior, you should find the transition painless, and
 once you learn a few basic keybindings it will be a much more efficient
 environment.
 
 For single expressions or indented blocks, the console behaves almost like the
 terminal IPython: single expressions are immediately evaluated, and indented
 blocks are evaluated once a single blank line is entered::
 
     In [1]: print "Hello IPython!"  # Enter was pressed at the end of the line
     Hello IPython!
     
     In [2]: for i in range(10):
        ...: 	print i,
        ...: 	
     0 1 2 3 4 5 6 7 8 9
 
 If you want to enter more than one expression in a single input block
 (something not possible in the terminal), you can use ``Control-Enter`` at the
 end of your first line instead of ``Enter``.  At that point the console goes
 into 'cell mode' and even if your inputs are not indented, it will continue
 accepting arbitrarily many lines until either you enter an extra blank line or
 you hit ``Shift-Enter`` (the key binding that forces execution).  When a
 multiline cell is entered, IPython analyzes it and executes its code producing
 an ``Out[n]`` prompt only for the last expression in it, while the rest of the
 cell is executed as if it was a script.  An example should clarify this::
 
     In [3]: x=1  # Hit C-Enter here
        ...: y=2  # from now on, regular Enter is sufficient
        ...: z=3
        ...: x**2  # This does *not* produce an Out[] value
        ...: x+y+z  # Only the last expression does
        ...: 
     Out[3]: 6
 
 The behavior where an extra blank line forces execution is only active if you
 are actually typing at the keyboard each line, and is meant to make it mimic
 the IPython terminal behavior.  If you paste a long chunk of input (for example
 a long script copied form an editor or web browser), it can contain arbitrarily
 many intermediate blank lines and they won't cause any problems.  As always,
 you can then make it execute by appending a blank line *at the end* or hitting
 ``Shift-Enter`` anywhere within the cell.
 
 With the up arrow key, you can retrieve previous blocks of input that contain
 multiple lines.  You can move inside of a multiline cell like you would in any
 text editor.  When you want it executed, the simplest thing to do is to hit the
 force execution key, ``Shift-Enter`` (though you can also navigate to the end
 and append a blank line by using ``Enter`` twice).
 
 If you've edited a multiline cell and accidentally navigate out of it with the
 up or down arrow keys, IPython will clear the cell and replace it with the
 contents of the one above or below that you navigated to.  If this was an
 accident and you want to retrieve the cell you were editing, use the Undo
 keybinding, ``Control-z``.
 
 
 Key bindings
 ============
 
 The IPython console supports most of the basic Emacs line-oriented keybindings,
 in addition to some of its own.
 
 The keybinding prefixes mean:
 
 - ``C``: Control
 - ``S``: Shift
 - ``M``: Meta (typically the Alt key)
 
 The keybindings themselves are:
 
 - ``Enter``: insert new line (may cause execution, see above).
 - ``C-Enter``: force new line, *never* causes execution.
 - ``S-Enter``: *force* execution regardless of where cursor is, no newline added.
 - ``C-c``: copy highlighted text to clipboard (prompts are automatically stripped).
 - ``C-S-c``: copy highlighted text to clipboard (prompts are not stripped).
 - ``C-v``: paste text from clipboard.
 - ``C-z``: undo (retrieves lost text if you move out of a cell with the arrows).
 - ``C-S-z``: redo.
 - ``C-o``: move to 'other' area, between pager and terminal.
 - ``C-l``: clear terminal.
 - ``C-a``: go to beginning of line.
 - ``C-e``: go to end of line.
 - ``C-k``: kill from cursor to the end of the line.
 - ``C-y``: yank (paste)
 - ``C-p``: previous line (like up arrow)
 - ``C-n``: next line (like down arrow)
 - ``C-f``: forward (like right arrow)
 - ``C-b``: back (like left arrow)
 - ``C-d``: delete next character.
 - ``M-<``: move to the beginning of the input region.
 - ``M->``: move to the end of the input region.
 - ``M-d``: delete next word.
 - ``M-Backspace``: delete previous word.
 - ``C-.``: force a kernel restart (a confirmation dialog appears).
 - ``C-+``: increase font size.
 - ``C--``: decrease font size.
 
 The IPython pager
 =================
 
 IPython will show long blocks of text from many sources using a builtin pager.
 You can control where this pager appears with the ``--paging`` command-line
 flag:
 
 - default: it is overlaid on top of the main terminal.  You must quit the pager
   to get back to the terminal (similar to how a pager such as ``less`` or
   ``more`` works).
 
 - vertical: the console is made double-tall, and the pager appears on the
   bottom area when needed.  You can view its contents while using the terminal.
 
 - horizontal: the console is made double-wide, and the pager appears on the
   right area when needed.  You can view its contents while using the terminal.
 
 If you use the vertical or horizontal paging modes, you can navigate between
 terminal and pager as follows:
 
 - Tab key: goes from pager to terminal (but not the other way around).
 - Control-o: goes from one to another always.
 - Mouse: click on either.
 
 In all cases, the ``q`` or ``Escape`` keys quit the pager (when used with the
 focus on the pager area).
 
 
 Running subprocesses
 ====================
 
 The graphical IPython console uses the ``pexpect`` module to run subprocesses
 when you type ``!command``.  This has a number of advantages (true asynchronous
 output from subprocesses as well as very robust termination of rogue
 subprocesses with ``Control-C``), as well as some limitations.  The main
 limitation is that you can *not* interact back with the subprocess, so anything
 that invokes a pager or expects you to type input into it will block and hang
 (you can kill it with ``Control-C``).
 
 We have provided as magics ``%less`` to page files (aliased to ``%more``),
 ``%clear`` to clear the terminal, and ``%man`` on Linux/OSX.  These cover the
 most common commands you'd want to call in your subshell and that would cause
 problems if invoked via ``!cmd``, but you need to be aware of this limitation.
 
 Display
 =======
 
 The IPython console can now display objects in a variety of formats, including
 HTML, PNG and SVG. This is accomplished using the display functions in
 ``IPython.core.display``::
 
     In [4]: from IPython.core.display import display, display_html
 
     In [5]: from IPython.core.display import display_png, display_svg
 
 Python objects can simply be passed to these functions and the appropriate
 representations will be displayed in the console as long as the objects know
 how to compute those representations. The easiest way of teaching objects how
 to format themselves in various representations is to define special methods
 such as: ``__html``, ``__svg__`` and ``__png__``. IPython's display formatters
 can also be given custom formatter functions for various types::
 
     In [6]: ip = get_ipython()
 
     In [7]: html_formatter = ip.display_formatter.formatters['text/html']
 
     In [8]: html_formatter.for_type(Foo, foo_to_html)
 
 For further details, see ``IPython.core.formatters``.
 
 Inline matplotlib graphics
 ==========================
 
 The IPython console is capable of displaying matplotlib figures inline, in SVG
 format.  If started with the ``--pylab inline`` flag, then all figures are
 rendered inline automatically.  If started with ``--pylab`` or ``--pylab <your
 backend>``, then a GUI backend will be used, but IPython's ``display()`` and
 ``getfigs()`` functions can be used to view plots inline::
 
     In [9]: display(*getfigs())    # display all figures inline
 
     In[10]: display(*getfigs(1,2)) # display figures 1 and 2 inline
 """
 
 
 quick_guide = """\
 ?         -> Introduction and overview of IPython's features.
 %quickref -> Quick reference.
 help      -> Python's own help system.
 object?   -> Details about 'object', use 'object??' for extra details.
 """
 
 gui_note = """\
 %guiref   -> A brief reference about the graphical user interface.
 """
 
 default_banner_parts = [
     'Python %s\n' % (sys.version.split('\n')[0],),
     'Type "copyright", "credits" or "license" for more information.\n\n',
     'IPython %s -- An enhanced Interactive Python.\n' % (release.version,),
     quick_guide
 ]
 
 default_gui_banner_parts = default_banner_parts + [gui_note]
 
 default_banner = ''.join(default_banner_parts)
 
 default_gui_banner = ''.join(default_gui_banner_parts)

IPython/kernel/controllerservice.py

@@ -1,374 +1,374 b''
 # encoding: utf-8
 # -*- test-case-name: IPython.kernel.test.test_controllerservice -*-
 
 """A Twisted Service for the IPython Controller.
 
 The IPython Controller:
 
 * Listens for Engines to connect and then manages access to those engines.
 * Listens for clients and passes commands from client to the Engines.
 * Exposes an asynchronous interfaces to the Engines which themselves can block.
 * Acts as a gateway to the Engines.
 
 The design of the controller is somewhat abstract to allow flexibility in how 
 the controller is presented to clients.  This idea is that there is a basic
 ControllerService class that allows engines to connect to it.  But, this 
 basic class has no client interfaces.  To expose client interfaces developers
 provide an adapter that makes the ControllerService look like something.  For 
 example, one client interface might support task farming and another might
 support interactive usage.  The important thing is that by using interfaces
 and adapters, a single controller can be accessed from multiple interfaces.
 Furthermore, by adapting various client interfaces to various network
 protocols, each client interface can be exposed to multiple network protocols.
 See multiengine.py for an example of how to adapt the ControllerService
 to a client interface.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import os
 
 from twisted.application import service
 from twisted.python import log
 from zope.interface import Interface, implements, Attribute
 
 from IPython.kernel.engineservice import \
     IEngineCore, \
     IEngineSerialized, \
     IEngineQueued
     
 from IPython.utils.path import get_ipython_dir
 from IPython.kernel import codeutil
 
 #-------------------------------------------------------------------------------
 # Interfaces for the Controller
 #-------------------------------------------------------------------------------
 
 class IControllerCore(Interface):
     """Basic methods any controller must have.
     
     This is basically the aspect of the controller relevant to the 
     engines and does not assume anything about how the engines will
     be presented to a client.
     """
         
     engines = Attribute("A dict of engine ids and engine instances.")
         
     def register_engine(remoteEngine, id=None, ip=None, port=None, 
         pid=None):
         """Register new remote engine.
         
         The controller can use the ip, port, pid of the engine to do useful things
         like kill the engines.
         
         :Parameters:
             remoteEngine
                 An implementer of IEngineCore, IEngineSerialized and IEngineQueued.
             id : int
                 Requested id.
             ip : str
                 IP address the engine is running on.
             port : int
                 Port the engine is on.
             pid : int
                 pid of the running engine.
         
         :Returns: A dict of {'id':id} and possibly other key, value pairs.
         """
     
     def unregister_engine(id):
         """Handle a disconnecting engine.
         
         :Parameters:
             id
                 The integer engine id of the engine to unregister.
         """
         
     def on_register_engine_do(f, includeID, *args, **kwargs):
         """Call ``f(*args, **kwargs)`` when an engine is registered.
         
         :Parameters:
             includeID : int
                 If True the first argument to f will be the id of the engine.
         """
             
     def on_unregister_engine_do(f, includeID, *args, **kwargs):
         """Call ``f(*args, **kwargs)`` when an engine is unregistered.
         
         :Parameters:
             includeID : int
                 If True the first argument to f will be the id of the engine.
         """
     
     def on_register_engine_do_not(f):
         """Stop calling f on engine registration"""
     
     def on_unregister_engine_do_not(f):
         """Stop calling f on engine unregistration"""
         
     def on_n_engines_registered_do(n, f, *arg, **kwargs):
         """Call f(*args, **kwargs) the first time the nth engine registers."""
                     
 class IControllerBase(IControllerCore):
     """The basic controller interface."""
     pass 
 
 
 #-------------------------------------------------------------------------------
 # Implementation of the ControllerService
 #-------------------------------------------------------------------------------
 
 class ControllerService(object, service.Service):
     """A basic Controller represented as a Twisted Service.
     
     This class doesn't implement any client notification mechanism.  That
     is up to adapted subclasses.
     """
     
     # I also pick up the IService interface by inheritance from service.Service
     implements(IControllerBase)
     name = 'ControllerService'
     
     def __init__(self, maxEngines=511, saveIDs=False):
         self.saveIDs = saveIDs
         self.engines = {}
         self.availableIDs = range(maxEngines,-1,-1)   # [511,...,0]
         self._onRegister = []
         self._onUnregister = []
         self._onNRegistered = []
     
     #---------------------------------------------------------------------------
     # Methods used to save the engine info to a log file
     #---------------------------------------------------------------------------
     
     def _buildEngineInfoString(self, id, ip, port, pid):
         if id is None:
             id = -99
         if ip is None:
             ip = "-99"
         if port is None:
             port = -99
         if pid is None:
             pid = -99
         return "Engine Info: %d %s %d %d" % (id, ip , port, pid)
         
     def _logEngineInfo(self, id, ip, port, pid):
         log.msg(self._buildEngineInfoString(id,ip,port,pid))
     
     def _getEngineInfoLogFile(self):
         # Store all logs inside the ipython directory
         ipdir = get_ipython_dir()
         pjoin = os.path.join
         logdir_base = pjoin(ipdir,'log')
         if not os.path.isdir(logdir_base):
             os.makedirs(logdir_base)
         logfile = os.path.join(logdir_base,'ipcontroller-%s-engine-info.log' % os.getpid())
         return logfile
     
     def _logEngineInfoToFile(self, id, ip, port, pid):
         """Log info about an engine to a log file.
         
         When an engine registers with a ControllerService, the ControllerService
         saves information about the engine to a log file.  That information
         can be useful for various purposes, such as killing hung engines, etc.
         
         This method takes the assigned id, ip/port and pid of the engine
         and saves it to a file of the form:
         
-        ~/.ipython/log/ipcontroller-###-engine-info.log
+        IPYTHON_DIR/log/ipcontroller-###-engine-info.log
         
         where ### is the pid of the controller.
         
         Each line of this file has the form:
         
         Engine Info: ip ip port pid
         
         If any of the entries are not known, they are replaced by -99.
         """
         
         fname = self._getEngineInfoLogFile()
         f = open(fname, 'a')
         s = self._buildEngineInfoString(id,ip,port,pid)
         f.write(s + '\n')
         f.close()
     
     #---------------------------------------------------------------------------
     # IControllerCore methods
     #---------------------------------------------------------------------------
         
     def register_engine(self, remoteEngine, id=None,
         ip=None, port=None, pid=None):
         """Register new engine connection"""
         
         # What happens if these assertions fail?
         assert IEngineCore.providedBy(remoteEngine), \
             "engine passed to register_engine doesn't provide IEngineCore"
         assert IEngineSerialized.providedBy(remoteEngine), \
             "engine passed to register_engine doesn't provide IEngineSerialized"
         assert IEngineQueued.providedBy(remoteEngine), \
             "engine passed to register_engine doesn't provide IEngineQueued"
         assert isinstance(id, int) or id is None, \
             "id to register_engine must be an integer or None"
         assert isinstance(ip, str) or ip is None, \
             "ip to register_engine must be a string or None"
         assert isinstance(port, int) or port is None, \
             "port to register_engine must be an integer or None"
         assert isinstance(pid, int) or pid is None, \
             "pid to register_engine must be an integer or None"
             
         desiredID = id
         if desiredID in self.engines.keys():
             desiredID = None
             
         if desiredID in self.availableIDs:
             getID = desiredID
             self.availableIDs.remove(desiredID)
         else:
             getID = self.availableIDs.pop()
         remoteEngine.id = getID
         remoteEngine.service = self
         self.engines[getID] = remoteEngine
 
         # Log the Engine Information for monitoring purposes
         self._logEngineInfoToFile(getID, ip, port, pid)
 
         msg = "registered engine with id: %i" %getID
         log.msg(msg)
         
         for i in range(len(self._onRegister)):
             (f,args,kwargs,ifid) = self._onRegister[i]
             try:
                 if ifid:
                     f(getID, *args, **kwargs)
                 else:
                     f(*args, **kwargs)
             except:
                 self._onRegister.pop(i)
         
         # Call functions when the nth engine is registered and them remove them
         for i, (n, f, args, kwargs) in enumerate(self._onNRegistered):
             if len(self.engines.keys()) == n:
                 try:
                     try:
                         f(*args, **kwargs)
                     except:
                         log.msg("Function %r failed when the %ith engine registered" % (f, n))
                 finally:
                     self._onNRegistered.pop(i)
         
         return {'id':getID}
     
     def unregister_engine(self, id):
         """Unregister engine by id."""
         
         assert isinstance(id, int) or id is None, \
             "id to unregister_engine must be an integer or None"
         
         msg = "unregistered engine with id: %i" %id
         log.msg(msg)
         try:
             del self.engines[id]
         except KeyError:
             log.msg("engine with id %i was not registered" % id)
         else:
             if not self.saveIDs:
                 self.availableIDs.append(id)
                 # Sort to assign lower ids first
                 self.availableIDs.sort(reverse=True) 
             else:
                 log.msg("preserving id %i" %id)
             
             for i in range(len(self._onUnregister)):
                 (f,args,kwargs,ifid) = self._onUnregister[i]
                 try:
                     if ifid:
                         f(id, *args, **kwargs)
                     else:
                         f(*args, **kwargs)
                 except:
                     self._onUnregister.pop(i)
     
     def on_register_engine_do(self, f, includeID, *args, **kwargs):
         assert callable(f), "f must be callable"
         self._onRegister.append((f,args,kwargs,includeID))
 
     def on_unregister_engine_do(self, f, includeID, *args, **kwargs):
         assert callable(f), "f must be callable"
         self._onUnregister.append((f,args,kwargs,includeID))
     
     def on_register_engine_do_not(self, f):
         for i in range(len(self._onRegister)):
             g = self._onRegister[i][0]
             if f == g:
                 self._onRegister.pop(i)
                 return
     
     def on_unregister_engine_do_not(self, f):
         for i in range(len(self._onUnregister)):
             g = self._onUnregister[i][0]
             if f == g:
                 self._onUnregister.pop(i)
                 return
 
     def on_n_engines_registered_do(self, n, f, *args, **kwargs):
         if len(self.engines.keys()) >= n:
             f(*args, **kwargs)
         else:
             self._onNRegistered.append((n,f,args,kwargs))
             
 
 #-------------------------------------------------------------------------------
 # Base class for adapting controller to different client APIs
 #-------------------------------------------------------------------------------
 
 class ControllerAdapterBase(object):
     """All Controller adapters should inherit from this class.
     
     This class provides a wrapped version of the IControllerBase interface that
     can be used to easily create new custom controllers.  Subclasses of this
     will provide a full implementation of IControllerBase.
     
     This class doesn't implement any client notification mechanism.  That
     is up to subclasses.
     """
     
     implements(IControllerBase)
     
     def __init__(self, controller):
         self.controller = controller
         # Needed for IControllerCore
         self.engines = self.controller.engines
         
     def register_engine(self, remoteEngine, id=None,
         ip=None, port=None, pid=None):
         return self.controller.register_engine(remoteEngine, 
             id, ip, port, pid)
     
     def unregister_engine(self, id):
         return self.controller.unregister_engine(id)
 
     def on_register_engine_do(self, f, includeID, *args, **kwargs):
         return self.controller.on_register_engine_do(f, includeID, *args, **kwargs)
 
     def on_unregister_engine_do(self, f, includeID, *args, **kwargs):
         return self.controller.on_unregister_engine_do(f, includeID, *args, **kwargs)        
     
     def on_register_engine_do_not(self, f):
         return self.controller.on_register_engine_do_not(f)
     
     def on_unregister_engine_do_not(self, f):
         return self.controller.on_unregister_engine_do_not(f)        
 
     def on_n_engines_registered_do(self, n, f, *args, **kwargs):
         return self.controller.on_n_engines_registered_do(n, f, *args, **kwargs)

IPython/kernel/ipcontrollerapp.py

@@ -1,271 +1,271 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The IPython controller application.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
 
 import copy
 import sys
 
 from twisted.application import service
 from twisted.internet import reactor
 from twisted.python import log
 
 from IPython.config.loader import Config
 from IPython.kernel import controllerservice
 from IPython.kernel.clusterdir import (
     ApplicationWithClusterDir,
     ClusterDirConfigLoader
 )
 from IPython.kernel.fcutil import FCServiceFactory, FURLError
 from IPython.utils.traitlets import Instance, Unicode
 
 
 #-----------------------------------------------------------------------------
 # Module level variables
 #-----------------------------------------------------------------------------
 
 
 #: The default config file name for this application
 default_config_file_name = u'ipcontroller_config.py'
 
 
 _description = """Start the IPython controller for parallel computing.
 
 The IPython controller provides a gateway between the IPython engines and
 clients. The controller needs to be started before the engines and can be
 configured using command line options or using a cluster directory. Cluster
 directories contain config, log and security files and are usually located in
-your .ipython directory and named as "cluster_<profile>". See the --profile
+your ipython directory and named as "cluster_<profile>". See the --profile
 and --cluster-dir options for details.
 """
 
 #-----------------------------------------------------------------------------
 # Default interfaces
 #-----------------------------------------------------------------------------
 
 # The default client interfaces for FCClientServiceFactory.interfaces
 default_client_interfaces = Config()
 default_client_interfaces.Task.interface_chain = [
     'IPython.kernel.task.ITaskController',
     'IPython.kernel.taskfc.IFCTaskController'
 ]
 
 default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
 
 default_client_interfaces.MultiEngine.interface_chain = [
     'IPython.kernel.multiengine.IMultiEngine',
     'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
 ]
 
 default_client_interfaces.MultiEngine.furl_file = u'ipcontroller-mec.furl'
 
 # Make this a dict we can pass to Config.__init__ for the default
 default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))
 
 
 
 # The default engine interfaces for FCEngineServiceFactory.interfaces
 default_engine_interfaces = Config()
 default_engine_interfaces.Default.interface_chain = [
     'IPython.kernel.enginefc.IFCControllerBase'
 ]
 
 default_engine_interfaces.Default.furl_file = u'ipcontroller-engine.furl'
 
 # Make this a dict we can pass to Config.__init__ for the default
 default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
 
 
 #-----------------------------------------------------------------------------
 # Service factories
 #-----------------------------------------------------------------------------
 
 
 class FCClientServiceFactory(FCServiceFactory):
     """A Foolscap implementation of the client services."""
 
     cert_file = Unicode(u'ipcontroller-client.pem', config=True)
     interfaces = Instance(klass=Config, kw=default_client_interfaces,
                           allow_none=False, config=True)
 
 
 class FCEngineServiceFactory(FCServiceFactory):
     """A Foolscap implementation of the engine services."""
 
     cert_file = Unicode(u'ipcontroller-engine.pem', config=True)
     interfaces = Instance(klass=dict, kw=default_engine_interfaces,
                           allow_none=False, config=True)
 
 
 #-----------------------------------------------------------------------------
 # Command line options
 #-----------------------------------------------------------------------------
 
 
 class IPControllerAppConfigLoader(ClusterDirConfigLoader):
 
     def _add_arguments(self):
         super(IPControllerAppConfigLoader, self)._add_arguments()
         paa = self.parser.add_argument
         # Client config
         paa('--client-ip',
             type=str, dest='FCClientServiceFactory.ip', 
             help='The IP address or hostname the controller will listen on for '
             'client connections.',
             metavar='FCClientServiceFactory.ip')
         paa('--client-port',
             type=int, dest='FCClientServiceFactory.port', 
             help='The port the controller will listen on for client connections. '
             'The default is to use 0, which will autoselect an open port.',
             metavar='FCClientServiceFactory.port')
         paa('--client-location',), dict(
             type=str, dest='FCClientServiceFactory.location',
             help='The hostname or IP that clients should connect to. This does '
             'not control which interface the controller listens on. Instead, this '
             'determines the hostname/IP that is listed in the FURL, which is how '
             'clients know where to connect. Useful if the controller is listening '
             'on multiple interfaces.',
             metavar='FCClientServiceFactory.location')
         # Engine config
         paa('--engine-ip',
             type=str, dest='FCEngineServiceFactory.ip', 
             help='The IP address or hostname the controller will listen on for '
             'engine connections.',
             metavar='FCEngineServiceFactory.ip')
         paa('--engine-port',
             type=int, dest='FCEngineServiceFactory.port',
             help='The port the controller will listen on for engine connections. '
             'The default is to use 0, which will autoselect an open port.',
             metavar='FCEngineServiceFactory.port')
         paa('--engine-location',
             type=str, dest='FCEngineServiceFactory.location',
             help='The hostname or IP that engines should connect to. This does '
             'not control which interface the controller listens on. Instead, this '
             'determines the hostname/IP that is listed in the FURL, which is how '
             'engines know where to connect. Useful if the controller is listening '
             'on multiple interfaces.',
             metavar='FCEngineServiceFactory.location')
         # Global config
         paa('--log-to-file',
             action='store_true', dest='Global.log_to_file',
             help='Log to a file in the log directory (default is stdout)')
         paa('-r','--reuse-furls', 
             action='store_true', dest='Global.reuse_furls',
             help='Try to reuse all FURL files. If this is not set all FURL files '
             'are deleted before the controller starts. This must be set if '
             'specific ports are specified by --engine-port or --client-port.')
         paa('--no-secure',
             action='store_false', dest='Global.secure',
             help='Turn off SSL encryption for all connections.')
         paa('--secure',
             action='store_true', dest='Global.secure',
             help='Turn off SSL encryption for all connections.')
 
 
 #-----------------------------------------------------------------------------
 # The main application
 #-----------------------------------------------------------------------------
 
 
 class IPControllerApp(ApplicationWithClusterDir):
 
     name = u'ipcontroller'
     description = _description
     command_line_loader = IPControllerAppConfigLoader
     default_config_file_name = default_config_file_name
     auto_create_cluster_dir = True
 
     def create_default_config(self):
         super(IPControllerApp, self).create_default_config()
         # Don't set defaults for Global.secure or Global.reuse_furls
         # as those are set in a component.
         self.default_config.Global.import_statements = []
         self.default_config.Global.clean_logs = True
 
     def pre_construct(self):
         super(IPControllerApp, self).pre_construct()
         c = self.master_config
         # The defaults for these are set in FCClientServiceFactory and
         # FCEngineServiceFactory, so we only set them here if the global
         # options have be set to override the class level defaults.
         if hasattr(c.Global, 'reuse_furls'):
             c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls
             c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls
             del c.Global.reuse_furls
         if hasattr(c.Global, 'secure'):
             c.FCClientServiceFactory.secure = c.Global.secure
             c.FCEngineServiceFactory.secure = c.Global.secure
             del c.Global.secure
 
     def construct(self):
         # This is the working dir by now.
         sys.path.insert(0, '')
 
         self.start_logging()
         self.import_statements()
 
         # Create the service hierarchy
         self.main_service = service.MultiService()
         # The controller service
         controller_service = controllerservice.ControllerService()
         controller_service.setServiceParent(self.main_service)
         # The client tub and all its refereceables
         try:
             csfactory = FCClientServiceFactory(config=self.master_config, adaptee=controller_service)
         except FURLError, e:
             log.err(e)
             self.exit(0)
         client_service = csfactory.create()
         client_service.setServiceParent(self.main_service)
         # The engine tub
         try:
             esfactory = FCEngineServiceFactory(config=self.master_config, adaptee=controller_service)
         except FURLError, e:
             log.err(e)
             self.exit(0)
         engine_service = esfactory.create()
         engine_service.setServiceParent(self.main_service)
 
     def import_statements(self):
         statements = self.master_config.Global.import_statements
         for s in statements:
             try:
                 log.msg("Executing statement: '%s'" % s)
                 exec s in globals(), locals()
             except:
                 log.msg("Error running statement: %s" % s)
 
     def start_app(self):
         # Start the controller service.
         self.main_service.startService()
         # Write the .pid file overwriting old ones. This allow multiple
         # controllers to clober each other. But Windows is not cleaning
         # these up properly.
         self.write_pid_file(overwrite=True)
         # Add a trigger to delete the .pid file upon shutting down.
         reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
         reactor.run()
 
 
 def launch_new_instance():
     """Create and run the IPython controller"""
     app = IPControllerApp()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()

IPython/kernel/ipengineapp.py

@@ -1,242 +1,242 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The IPython controller application
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 
 from twisted.application import service
 from twisted.internet import reactor
 from twisted.python import log
 
 from IPython.kernel.clusterdir import (
     ApplicationWithClusterDir,
     ClusterDirConfigLoader
 )
 from IPython.kernel.engineconnector import EngineConnector
 from IPython.kernel.engineservice import EngineService
 from IPython.kernel.fcutil import Tub
 from IPython.utils.importstring import import_item
 
 #-----------------------------------------------------------------------------
 # Module level variables
 #-----------------------------------------------------------------------------
 
 #: The default config file name for this application
 default_config_file_name = u'ipengine_config.py'
 
 
 mpi4py_init = """from mpi4py import MPI as mpi
 mpi.size = mpi.COMM_WORLD.Get_size()
 mpi.rank = mpi.COMM_WORLD.Get_rank()
 """
 
 
 pytrilinos_init = """from PyTrilinos import Epetra
 class SimpleStruct:
 pass
 mpi = SimpleStruct()
 mpi.rank = 0
 mpi.size = 0
 """
 
 
 _description = """Start an IPython engine for parallel computing.\n\n
 
 IPython engines run in parallel and perform computations on behalf of a client
 and controller. A controller needs to be started before the engines. The
 engine can be configured using command line options or using a cluster
 directory. Cluster directories contain config, log and security files and are
-usually located in your .ipython directory and named as "cluster_<profile>".
+usually located in your ipython directory and named as "cluster_<profile>".
 See the --profile and --cluster-dir options for details.
 """
 
 #-----------------------------------------------------------------------------
 # Command line options
 #-----------------------------------------------------------------------------
 
 
 class IPEngineAppConfigLoader(ClusterDirConfigLoader):
 
     def _add_arguments(self):
         super(IPEngineAppConfigLoader, self)._add_arguments()
         paa = self.parser.add_argument
         # Controller config
         paa('--furl-file',
             type=unicode, dest='Global.furl_file',
             help='The full location of the file containing the FURL of the '
             'controller. If this is not given, the FURL file must be in the '
             'security directory of the cluster directory.  This location is '
             'resolved using the --profile and --app-dir options.',
             metavar='Global.furl_file')
         # MPI
         paa('--mpi',
             type=str, dest='MPI.use',
             help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).',
             metavar='MPI.use')
         # Global config
         paa('--log-to-file',
             action='store_true', dest='Global.log_to_file',
             help='Log to a file in the log directory (default is stdout)')
 
 
 #-----------------------------------------------------------------------------
 # Main application
 #-----------------------------------------------------------------------------
 
 
 class IPEngineApp(ApplicationWithClusterDir):
 
     name = u'ipengine'
     description = _description
     command_line_loader = IPEngineAppConfigLoader
     default_config_file_name = default_config_file_name
     auto_create_cluster_dir = True
 
     def create_default_config(self):
         super(IPEngineApp, self).create_default_config()
 
         # The engine should not clean logs as we don't want to remove the
         # active log files of other running engines.
         self.default_config.Global.clean_logs = False
 
         # Global config attributes
         self.default_config.Global.exec_lines = []
         self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
 
         # Configuration related to the controller
         # This must match the filename (path not included) that the controller
         # used for the FURL file.
         self.default_config.Global.furl_file_name = u'ipcontroller-engine.furl'
         # If given, this is the actual location of the controller's FURL file.
         # If not, this is computed using the profile, app_dir and furl_file_name
         self.default_config.Global.furl_file = u''
 
         # The max number of connection attemps and the initial delay between
         # those attemps.
         self.default_config.Global.connect_delay = 0.1
         self.default_config.Global.connect_max_tries = 15
 
         # MPI related config attributes
         self.default_config.MPI.use = ''
         self.default_config.MPI.mpi4py = mpi4py_init
         self.default_config.MPI.pytrilinos = pytrilinos_init
 
     def post_load_command_line_config(self):
         pass
 
     def pre_construct(self):
         super(IPEngineApp, self).pre_construct()
         self.find_cont_furl_file()
 
     def find_cont_furl_file(self):
         """Set the furl file.
 
         Here we don't try to actually see if it exists for is valid as that
         is hadled by the connection logic.
         """
         config = self.master_config
         # Find the actual controller FURL file
         if not config.Global.furl_file:
             try_this = os.path.join(
                 config.Global.cluster_dir, 
                 config.Global.security_dir,
                 config.Global.furl_file_name
             )
             config.Global.furl_file = try_this
 
     def construct(self):
         # This is the working dir by now.
         sys.path.insert(0, '')
 
         self.start_mpi()
         self.start_logging()
 
         # Create the underlying shell class and EngineService
         shell_class = import_item(self.master_config.Global.shell_class)
         self.engine_service = EngineService(shell_class, mpi=mpi)
 
         self.exec_lines()
 
         # Create the service hierarchy
         self.main_service = service.MultiService()
         self.engine_service.setServiceParent(self.main_service)
         self.tub_service = Tub()
         self.tub_service.setServiceParent(self.main_service)
         # This needs to be called before the connection is initiated
         self.main_service.startService()
 
         # This initiates the connection to the controller and calls
         # register_engine to tell the controller we are ready to do work
         self.engine_connector = EngineConnector(self.tub_service)
 
         log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
 
         reactor.callWhenRunning(self.call_connect)
 
     def call_connect(self):
         d = self.engine_connector.connect_to_controller(
             self.engine_service, 
             self.master_config.Global.furl_file,
             self.master_config.Global.connect_delay,
             self.master_config.Global.connect_max_tries
         )
 
         def handle_error(f):
             log.msg('Error connecting to controller. This usually means that '
             'i) the controller was not started, ii) a firewall was blocking '
             'the engine from connecting to the controller or iii) the engine '
             ' was not pointed at the right FURL file:')
             log.msg(f.getErrorMessage())
             reactor.callLater(0.1, reactor.stop)
 
         d.addErrback(handle_error)
 
     def start_mpi(self):
         global mpi
         mpikey = self.master_config.MPI.use
         mpi_import_statement = self.master_config.MPI.get(mpikey, None)
         if mpi_import_statement is not None:
             try:
                 self.log.info("Initializing MPI:")
                 self.log.info(mpi_import_statement)
                 exec mpi_import_statement in globals()
             except:
                 mpi = None
         else:
             mpi = None
 
     def exec_lines(self):
         for line in self.master_config.Global.exec_lines:
             try:
                 log.msg("Executing statement: '%s'" % line)
                 self.engine_service.execute(line)
             except:
                 log.msg("Error executing statement: %s" % line)
 
     def start_app(self):
         reactor.run()
 
 
 def launch_new_instance():
     """Create and run the IPython controller"""
     app = IPEngineApp()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()
 

IPython/utils/path.py

@@ -1,345 +1,383 b''
 # encoding: utf-8
 """
 Utilities for path handling.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 
 import IPython
 from IPython.utils.process import system
 from IPython.utils.importstring import import_item
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 
 def _get_long_path_name(path):
     """Dummy no-op."""
     return path
 
 
 if sys.platform == 'win32':
     def _get_long_path_name(path):
         """Get a long path name (expand ~) on Windows using ctypes.
 
         Examples
         --------
 
         >>> get_long_path_name('c:\\docume~1')
         u'c:\\\\Documents and Settings'
 
         """
         try:
             import ctypes
         except ImportError:
             raise ImportError('you need to have ctypes installed for this to work')
         _GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
         _GetLongPathName.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p,
             ctypes.c_uint ]
 
         buf = ctypes.create_unicode_buffer(260)
         rv = _GetLongPathName(path, buf, 260)
         if rv == 0 or rv > 260:
             return path
         else:
             return buf.value
 
 
 def get_long_path_name(path):
     """Expand a path into its long form.
 
     On Windows this expands any ~ in the paths. On other platforms, it is
     a null operation.
     """
     return _get_long_path_name(path)
 
 
 def get_py_filename(name):
     """Return a valid python filename in the current directory.
 
     If the given name is not a file, it adds '.py' and searches again.
     Raises IOError with an informative message if the file isn't found."""
 
     name = os.path.expanduser(name)
     if not os.path.isfile(name) and not name.endswith('.py'):
         name += '.py'
     if os.path.isfile(name):
         return name
     else:
         raise IOError,'File `%s` not found.' % name
 
 
 def filefind(filename, path_dirs=None):
     """Find a file by looking through a sequence of paths.
 
     This iterates through a sequence of paths looking for a file and returns
     the full, absolute path of the first occurence of the file.  If no set of
     path dirs is given, the filename is tested as is, after running through
     :func:`expandvars` and :func:`expanduser`.  Thus a simple call::
 
         filefind('myfile.txt')
 
     will find the file in the current working dir, but::
 
         filefind('~/myfile.txt')
 
     Will find the file in the users home directory.  This function does not
     automatically try any paths, such as the cwd or the user's home directory.
     
     Parameters
     ----------
     filename : str
         The filename to look for.
     path_dirs : str, None or sequence of str
         The sequence of paths to look for the file in.  If None, the filename
         need to be absolute or be in the cwd.  If a string, the string is
         put into a sequence and the searched.  If a sequence, walk through
         each element and join with ``filename``, calling :func:`expandvars`
         and :func:`expanduser` before testing for existence.
         
     Returns
     -------
     Raises :exc:`IOError` or returns absolute path to file.
     """
     
     # If paths are quoted, abspath gets confused, strip them...
     filename = filename.strip('"').strip("'")
     # If the input is an absolute path, just check it exists
     if os.path.isabs(filename) and os.path.isfile(filename):
         return filename
         
     if path_dirs is None:
         path_dirs = ("",)
     elif isinstance(path_dirs, basestring):
         path_dirs = (path_dirs,)
         
     for path in path_dirs:
         if path == '.': path = os.getcwd()
         testname = expand_path(os.path.join(path, filename))
         if os.path.isfile(testname):
             return os.path.abspath(testname)
         
     raise IOError("File %r does not exist in any of the search paths: %r" % 
                   (filename, path_dirs) )
 
 
 class HomeDirError(Exception):
     pass
 
 
 def get_home_dir():
     """Return the closest possible equivalent to a 'home' directory.
 
     * On POSIX, we try $HOME.
     * On Windows we try:
       - %HOMESHARE%
       - %HOMEDRIVE\%HOMEPATH%
       - %USERPROFILE%
       - Registry hack for My Documents
       - %HOME%: rare, but some people with unix-like setups may have defined it
     * On Dos C:\
  
     Currently only Posix and NT are implemented, a HomeDirError exception is
     raised for all other OSes.
     """
 
     isdir = os.path.isdir
     env = os.environ
 
     # first, check py2exe distribution root directory for _ipython.
     # This overrides all. Normally does not exist.
 
     if hasattr(sys, "frozen"): #Is frozen by py2exe
         if '\\library.zip\\' in IPython.__file__.lower():#libraries compressed to zip-file
             root, rest = IPython.__file__.lower().split('library.zip')
         else: 
             root=os.path.join(os.path.split(IPython.__file__)[0],"../../")
         root=os.path.abspath(root).rstrip('\\')
         if isdir(os.path.join(root, '_ipython')):
             os.environ["IPYKITROOT"] = root
         return root.decode(sys.getfilesystemencoding())
 
     if os.name == 'posix':
         # Linux, Unix, AIX, OS X
         try:
             homedir = env['HOME']
         except KeyError:
             raise HomeDirError('Undefined $HOME, IPython cannot proceed.')
         else:
             return homedir.decode(sys.getfilesystemencoding())
     elif os.name == 'nt':
         # Now for win9x, XP, Vista, 7?
         # For some strange reason all of these return 'nt' for os.name.
         # First look for a network home directory. This will return the UNC
         # path (\\server\\Users\%username%) not the mapped path (Z:\). This
         # is needed when running IPython on cluster where all paths have to 
         # be UNC.
         try:
             homedir = env['HOMESHARE']
         except KeyError:
             pass
         else:
             if isdir(homedir):
                 return homedir.decode(sys.getfilesystemencoding())
 
         # Now look for a local home directory
         try:
             homedir = os.path.join(env['HOMEDRIVE'],env['HOMEPATH'])
         except KeyError:
             pass
         else:
             if isdir(homedir):
                 return homedir.decode(sys.getfilesystemencoding())
 
         # Now the users profile directory
         try:
             homedir = os.path.join(env['USERPROFILE'])
         except KeyError:
             pass
         else:
             if isdir(homedir):
                 return homedir.decode(sys.getfilesystemencoding())
 
         # Use the registry to get the 'My Documents' folder.
         try:
             import _winreg as wreg
             key = wreg.OpenKey(
                 wreg.HKEY_CURRENT_USER,
                 "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
             )
             homedir = wreg.QueryValueEx(key,'Personal')[0]
             key.Close()
         except:
             pass
         else:
             if isdir(homedir):
                 return homedir.decode(sys.getfilesystemencoding())
 
         # A user with a lot of unix tools in win32 may have defined $HOME.
         # Try this as a last ditch option.
         try:
             homedir = env['HOME']
         except KeyError:
             pass
         else:
             if isdir(homedir):
                 return homedir.decode(sys.getfilesystemencoding())
 
         # If all else fails, raise HomeDirError
         raise HomeDirError('No valid home directory could be found')
     elif os.name == 'dos':
         # Desperate, may do absurd things in classic MacOS. May work under DOS.
         return 'C:\\'.decode(sys.getfilesystemencoding())
     else:
         raise HomeDirError('No valid home directory could be found for your OS')
 
+def get_xdg_dir():
+    """Return the XDG_CONFIG_HOME, if it is defined and exists, else None.
+    
+    This is only for posix (Linux,Unix,OS X, etc) systems.
+    """
+
+    isdir = os.path.isdir
+    env = os.environ
+    
+    if os.name == 'posix':
+        # Linux, Unix, AIX, OS X
+        # use ~/.config if not set OR empty
+        xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config')
+        if xdg and isdir(xdg):
+            return xdg.decode(sys.getfilesystemencoding())
+    
+    return None
+    
 
 def get_ipython_dir():
     """Get the IPython directory for this platform and user.
     
     This uses the logic in `get_home_dir` to find the home directory
     and the adds .ipython to the end of the path.
     """
+    
+    env = os.environ
+    pjoin = os.path.join
+    exists = os.path.exists
+    
     ipdir_def = '.ipython'
+    xdg_def = 'ipython'
+    
     home_dir = get_home_dir()
+    xdg_dir = get_xdg_dir()
     # import pdb; pdb.set_trace()  # dbg
-    ipdir = os.environ.get(
-        'IPYTHON_DIR', os.environ.get(
-            'IPYTHONDIR', os.path.join(home_dir, ipdir_def)
-        )
-    )
+    ipdir = env.get('IPYTHON_DIR', env.get('IPYTHONDIR', None))
+    if ipdir is None:
+        # not set explicitly, use XDG_CONFIG_HOME or HOME
+        home_ipdir = pjoin(home_dir, ipdir_def)
+        if xdg_dir:
+            # use XDG, as long as the user isn't already
+            # using $HOME/.ipython and *not* XDG/ipython
+            
+            xdg_ipdir = pjoin(xdg_dir, xdg_def)
+        
+            if exists(xdg_ipdir) or not exists(home_ipdir):
+                ipdir = xdg_ipdir
+        
+        if ipdir is None:
+            # not using XDG
+            ipdir = home_ipdir
+    
     return ipdir.decode(sys.getfilesystemencoding())
 
 
 def get_ipython_package_dir():
     """Get the base directory where IPython itself is installed."""
     ipdir = os.path.dirname(IPython.__file__)
     return ipdir.decode(sys.getfilesystemencoding())
 
 
 def get_ipython_module_path(module_str):
     """Find the path to an IPython module in this version of IPython.
 
     This will always find the version of the module that is in this importable
     IPython package. This will always return the path to the ``.py``
     version of the module.
     """
     if module_str == 'IPython':
         return os.path.join(get_ipython_package_dir(), '__init__.py')
     mod = import_item(module_str)
     the_path = mod.__file__.replace('.pyc', '.py')
     the_path = the_path.replace('.pyo', '.py')
     return the_path.decode(sys.getfilesystemencoding())
 
 
 def expand_path(s):
     """Expand $VARS and ~names in a string, like a shell
 
     :Examples:
     
        In [2]: os.environ['FOO']='test'
 
        In [3]: expand_path('variable FOO is $FOO')
        Out[3]: 'variable FOO is test'
     """
     # This is a pretty subtle hack. When expand user is given a UNC path
     # on Windows (\\server\share$\%username%), os.path.expandvars, removes
     # the $ to get (\\server\share\%username%). I think it considered $
     # alone an empty var. But, we need the $ to remains there (it indicates
     # a hidden share).
     if os.name=='nt':
         s = s.replace('$\\', 'IPYTHON_TEMP')
     s = os.path.expandvars(os.path.expanduser(s))
     if os.name=='nt':
         s = s.replace('IPYTHON_TEMP', '$\\')
     return s
 
 
 def target_outdated(target,deps):
     """Determine whether a target is out of date.
 
     target_outdated(target,deps) -> 1/0
 
     deps: list of filenames which MUST exist.
     target: single filename which may or may not exist.
 
     If target doesn't exist or is older than any file listed in deps, return
     true, otherwise return false.
     """
     try:
         target_time = os.path.getmtime(target)
     except os.error:
         return 1
     for dep in deps:
         dep_time = os.path.getmtime(dep)
         if dep_time > target_time:
             #print "For target",target,"Dep failed:",dep # dbg
             #print "times (dep,tar):",dep_time,target_time # dbg
             return 1
     return 0
 
 
 def target_update(target,deps,cmd):
     """Update a target with a given command given a list of dependencies.
 
     target_update(target,deps,cmd) -> runs cmd if target is outdated.
 
     This is just a wrapper around target_outdated() which calls the given
     command if target is outdated."""
 
     if target_outdated(target,deps):
         system(cmd)
 

IPython/utils/tests/test_path.py

@@ -1,272 +1,358 b''
 # encoding: utf-8
 """Tests for IPython.utils.path.py"""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import shutil
 import sys
 import tempfile
 
 from os.path import join, abspath, split
 
 import nose.tools as nt
 
 from nose import with_setup
 
 import IPython
 from IPython.testing import decorators as dec
 from IPython.testing.decorators import skip_if_not_win32, skip_win32
 from IPython.utils import path
 
 # Platform-dependent imports
 try:
     import _winreg as wreg
 except ImportError:
     #Fake _winreg module on none windows platforms
     import new
     sys.modules["_winreg"] = new.module("_winreg")
     import _winreg as wreg
     #Add entries that needs to be stubbed by the testing code
     (wreg.OpenKey, wreg.QueryValueEx,) = (None, None)
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 env = os.environ
 TEST_FILE_PATH = split(abspath(__file__))[0]
 TMP_TEST_DIR = tempfile.mkdtemp()
 HOME_TEST_DIR = join(TMP_TEST_DIR, "home_test_dir")
+XDG_TEST_DIR = join(HOME_TEST_DIR, "xdg_test_dir")
 IP_TEST_DIR = join(HOME_TEST_DIR,'.ipython')
 #
 # Setup/teardown functions/decorators
 #
 
 def setup():
     """Setup testenvironment for the module:
     
             - Adds dummy home dir tree
     """
     # Do not mask exceptions here.  In particular, catching WindowsError is a
     # problem because that exception is only defined on Windows...
     os.makedirs(IP_TEST_DIR)
+    os.makedirs(os.path.join(XDG_TEST_DIR, 'ipython'))
 
 
 def teardown():
     """Teardown testenvironment for the module:
     
             - Remove dummy home dir tree
     """
     # Note: we remove the parent test dir, which is the root of all test
     # subdirs we may have created.  Use shutil instead of os.removedirs, so
     # that non-empty directories are all recursively removed.
     shutil.rmtree(TMP_TEST_DIR)
 
 
 def setup_environment():
     """Setup testenvironment for some functions that are tested 
     in this module. In particular this functions stores attributes
     and other things that we need to stub in some test functions.
     This needs to be done on a function level and not module level because 
     each testfunction needs a pristine environment.
     """
     global oldstuff, platformstuff
     oldstuff = (env.copy(), os.name, path.get_home_dir, IPython.__file__)
 
     if os.name == 'nt':
         platformstuff = (wreg.OpenKey, wreg.QueryValueEx,)
 
 
 def teardown_environment():
     """Restore things that were remebered by the setup_environment function
     """
     (oldenv, os.name, get_home_dir, IPython.__file__,) = oldstuff
         
     for key in env.keys():
         if key not in oldenv:
             del env[key]
     env.update(oldenv)
     if hasattr(sys, 'frozen'):
         del sys.frozen
     if os.name == 'nt':
         (wreg.OpenKey, wreg.QueryValueEx,) = platformstuff
 
 # Build decorator that uses the setup_environment/setup_environment
 with_environment = with_setup(setup_environment, teardown_environment)
 
 
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_1():
     """Testcase for py2exe logic, un-compressed lib
     """
     sys.frozen = True
     
     #fake filename for IPython.__init__
     IPython.__file__ = abspath(join(HOME_TEST_DIR, "Lib/IPython/__init__.py"))
     
     home_dir = path.get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
 
 
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_2():
     """Testcase for py2exe logic, compressed lib
     """
     sys.frozen = True
     #fake filename for IPython.__init__
     IPython.__file__ = abspath(join(HOME_TEST_DIR, "Library.zip/IPython/__init__.py")).lower()
     
     home_dir = path.get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR).lower())
 
 
 @with_environment
 @skip_win32
 def test_get_home_dir_3():
     """Testcase $HOME is set, then use its value as home directory."""
     env["HOME"] = HOME_TEST_DIR
     home_dir = path.get_home_dir()
     nt.assert_equal(home_dir, env["HOME"])
 
 
 @with_environment
 def test_get_home_dir_4():
     """Testcase $HOME is not set, os=='posix'. 
     This should fail with HomeDirError"""
     
     os.name = 'posix'
     if 'HOME' in env: del env['HOME']
     nt.assert_raises(path.HomeDirError, path.get_home_dir)
 
 
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_5():
     """Using HOMEDRIVE + HOMEPATH, os=='nt'.
 
     HOMESHARE is missing.
     """
 
     os.name = 'nt'
     env.pop('HOMESHARE', None)
     env['HOMEDRIVE'], env['HOMEPATH'] = os.path.splitdrive(HOME_TEST_DIR)
     home_dir = path.get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
 
 
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_6():
     """Using USERPROFILE, os=='nt'.
 
     HOMESHARE, HOMEDRIVE, HOMEPATH are missing.
     """
 
     os.name = 'nt'
     env.pop('HOMESHARE', None)
     env.pop('HOMEDRIVE', None)
     env.pop('HOMEPATH', None)
     env["USERPROFILE"] = abspath(HOME_TEST_DIR)
     home_dir = path.get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
 
 
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_7():
     """Using HOMESHARE, os=='nt'."""
 
     os.name = 'nt'
     env["HOMESHARE"] = abspath(HOME_TEST_DIR)
     home_dir = path.get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
     
 # Should we stub wreg fully so we can run the test on all platforms?
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_8():
     """Using registry hack for 'My Documents', os=='nt'
     
     HOMESHARE, HOMEDRIVE, HOMEPATH, USERPROFILE and others are missing.
     """
     os.name = 'nt'
     # Remove from stub environment all keys that may be set
     for key in ['HOME', 'HOMESHARE', 'HOMEDRIVE', 'HOMEPATH', 'USERPROFILE']:
         env.pop(key, None)
 
     #Stub windows registry functions
     def OpenKey(x, y):
         class key:
             def Close(self):
                 pass
         return key()
     def QueryValueEx(x, y):
         return [abspath(HOME_TEST_DIR)]
 
     wreg.OpenKey = OpenKey
     wreg.QueryValueEx = QueryValueEx
 
     home_dir = path.get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
 
 
 @with_environment
 def test_get_ipython_dir_1():
     """test_get_ipython_dir_1, Testcase to see if we can call get_ipython_dir without Exceptions."""
     env['IPYTHON_DIR'] = "someplace/.ipython"
     ipdir = path.get_ipython_dir()
     nt.assert_equal(ipdir, "someplace/.ipython")
 
 
 @with_environment
 def test_get_ipython_dir_2():
     """test_get_ipython_dir_2, Testcase to see if we can call get_ipython_dir without Exceptions."""
     path.get_home_dir = lambda : "someplace"
     os.name = "posix"
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
+    env.pop('XDG_CONFIG_HOME', None)
     ipdir = path.get_ipython_dir()
     nt.assert_equal(ipdir, os.path.join("someplace", ".ipython"))
 
+@with_environment
+def test_get_ipython_dir_3():
+    """test_get_ipython_dir_3, use XDG if defined, and .ipython doesn't exist."""
+    path.get_home_dir = lambda : "someplace"
+    os.name = "posix"
+    env.pop('IPYTHON_DIR', None)
+    env.pop('IPYTHONDIR', None)
+    env['XDG_CONFIG_HOME'] = XDG_TEST_DIR
+    ipdir = path.get_ipython_dir()
+    nt.assert_equal(ipdir, os.path.join(XDG_TEST_DIR, "ipython"))
+
+@with_environment
+def test_get_ipython_dir_4():
+    """test_get_ipython_dir_4, use XDG if both exist."""
+    path.get_home_dir = lambda : HOME_TEST_DIR
+    os.name = "posix"
+    env.pop('IPYTHON_DIR', None)
+    env.pop('IPYTHONDIR', None)
+    env['XDG_CONFIG_HOME'] = XDG_TEST_DIR
+    xdg_ipdir = os.path.join(XDG_TEST_DIR, "ipython")
+    ipdir = path.get_ipython_dir()
+    nt.assert_equal(ipdir, xdg_ipdir)
+
+@with_environment
+def test_get_ipython_dir_5():
+    """test_get_ipython_dir_5, use .ipython if exists and XDG defined, but doesn't exist."""
+    os.name = "posix"
+    env.pop('IPYTHON_DIR', None)
+    env.pop('IPYTHONDIR', None)
+    env['XDG_CONFIG_HOME'] = XDG_TEST_DIR
+    os.rmdir(os.path.join(XDG_TEST_DIR, 'ipython'))
+    ipdir = path.get_ipython_dir()
+    nt.assert_equal(ipdir, IP_TEST_DIR)
+
+@with_environment
+def test_get_ipython_dir_6():
+    """test_get_ipython_dir_6, use XDG if defined and neither exist."""
+    path.get_home_dir = lambda : 'somehome'
+    path.get_xdg_dir = lambda : 'somexdg'
+    os.name = "posix"
+    env.pop('IPYTHON_DIR', None)
+    env.pop('IPYTHONDIR', None)
+    xdg_ipdir = os.path.join("somexdg", "ipython")
+    ipdir = path.get_ipython_dir()
+    nt.assert_equal(ipdir, xdg_ipdir)
+
+@with_environment
+def test_get_xdg_dir_1():
+    """test_get_xdg_dir_1, check xdg_dir"""
+    reload(path)
+    path.get_home_dir = lambda : 'somewhere'
+    os.name = "posix"
+    env.pop('IPYTHON_DIR', None)
+    env.pop('IPYTHONDIR', None)
+    env.pop('XDG_CONFIG_HOME', None)
+    
+    nt.assert_equal(path.get_xdg_dir(), os.path.join('somewhere', '.config'))
+
+
+@with_environment
+def test_get_xdg_dir_1():
+    """test_get_xdg_dir_1, check nonexistant xdg_dir"""
+    reload(path)
+    path.get_home_dir = lambda : HOME_TEST_DIR
+    os.name = "posix"
+    env.pop('IPYTHON_DIR', None)
+    env.pop('IPYTHONDIR', None)
+    env.pop('XDG_CONFIG_HOME', None)
+    nt.assert_equal(path.get_xdg_dir(), None)
+
+@with_environment
+def test_get_xdg_dir_2():
+    """test_get_xdg_dir_2, check xdg_dir default to ~/.config"""
+    reload(path)
+    path.get_home_dir = lambda : HOME_TEST_DIR
+    os.name = "posix"
+    env.pop('IPYTHON_DIR', None)
+    env.pop('IPYTHONDIR', None)
+    env.pop('XDG_CONFIG_HOME', None)
+    cfgdir=os.path.join(path.get_home_dir(), '.config')
+    os.makedirs(cfgdir)
+    
+    nt.assert_equal(path.get_xdg_dir(), cfgdir)
 
 def test_filefind():
     """Various tests for filefind"""
     f = tempfile.NamedTemporaryFile()
     # print 'fname:',f.name
     alt_dirs = path.get_ipython_dir()
     t = path.filefind(f.name, alt_dirs)
     # print 'found:',t
 
 
 def test_get_ipython_package_dir():
     ipdir = path.get_ipython_package_dir()
     nt.assert_true(os.path.isdir(ipdir))
 
 
 def test_get_ipython_module_path():
     ipapp_path = path.get_ipython_module_path('IPython.frontend.terminal.ipapp')
     nt.assert_true(os.path.isfile(ipapp_path))
 
 
 @dec.skip_if_not_win32
 def test_get_long_path_name_win32():
     p = path.get_long_path_name('c:\\docume~1')
     nt.assert_equals(p,u'c:\\Documents and Settings') 
 
 
 @dec.skip_win32
 def test_get_long_path_name():
     p = path.get_long_path_name('/usr/local')
     nt.assert_equals(p,'/usr/local')
 

docs/source/config/old.txt

@@ -1,230 +1,231 b''
 .. _initial config:
 
 =============================================================
 Outdated configuration information that might still be useful
 =============================================================
 
 .. warning::
 
     All of the information in this file is outdated. Until the new
     configuration system is better documented, this material is being kept.
 
 This section will help you set various things in your environment for
 your IPython sessions to be as efficient as possible. All of IPython's
 configuration information, along with several example files, is stored
-in a directory named by default $HOME/.ipython. You can change this by
+in a directory named by default $HOME/.config/ipython if $HOME/.config 
+exists (Linux), or $HOME/.ipython as a secondary default. You can change this by
 defining the environment variable IPYTHONDIR, or at runtime with the
 command line option -ipythondir.
 
 If all goes well, the first time you run IPython it should automatically create
 a user copy of the config directory for you, based on its builtin defaults. You
 can look at the files it creates to learn more about configuring the
 system. The main file you will modify to configure IPython's behavior is called
 ipythonrc (with a .ini extension under Windows), included for reference
 :ref:`here <ipythonrc>`. This file is very commented and has many variables you
 can change to suit your taste, you can find more details :ref:`here
 <customization>`. Here we discuss the basic things you will want to make sure
 things are working properly from the beginning.
 
 Color
 =====
 
 The default IPython configuration has most bells and whistles turned on
 (they're pretty safe). But there's one that may cause problems on some
 systems: the use of color on screen for displaying information. This is
 very useful, since IPython can show prompts and exception tracebacks
 with various colors, display syntax-highlighted source code, and in
 general make it easier to visually parse information.
 
 The following terminals seem to handle the color sequences fine:
 
     * Linux main text console, KDE Konsole, Gnome Terminal, E-term,
       rxvt, xterm.
     * CDE terminal (tested under Solaris). This one boldfaces light colors.
     * (X)Emacs buffers. See the emacs_ section for more details on
       using IPython with (X)Emacs.
     * A Windows (XP/2k) command prompt with pyreadline_.
     * A Windows (XP/2k) CygWin shell. Although some users have reported
       problems; it is not clear whether there is an issue for everyone
       or only under specific configurations. If you have full color
       support under cygwin, please post to the IPython mailing list so
       this issue can be resolved for all users.
 
 .. _pyreadline: https://code.launchpad.net/pyreadline
       
 These have shown problems:
 
     * Windows command prompt in WinXP/2k logged into a Linux machine via
       telnet or ssh.
     * Windows native command prompt in WinXP/2k, without Gary Bishop's
       extensions. Once Gary's readline library is installed, the normal
       WinXP/2k command prompt works perfectly.
 
 Currently the following color schemes are available:
 
     * NoColor: uses no color escapes at all (all escapes are empty '' ''
       strings). This 'scheme' is thus fully safe to use in any terminal.
     * Linux: works well in Linux console type environments: dark
       background with light fonts. It uses bright colors for
       information, so it is difficult to read if you have a light
       colored background.
     * LightBG: the basic colors are similar to those in the Linux scheme
       but darker. It is easy to read in terminals with light backgrounds.
 
 IPython uses colors for two main groups of things: prompts and
 tracebacks which are directly printed to the terminal, and the object
 introspection system which passes large sets of data through a pager.
 
 Input/Output prompts and exception tracebacks
 =============================================
 
 You can test whether the colored prompts and tracebacks work on your
 system interactively by typing '%colors Linux' at the prompt (use
 '%colors LightBG' if your terminal has a light background). If the input
 prompt shows garbage like::
 
     [0;32mIn [[1;32m1[0;32m]: [0;00m
 
 instead of (in color) something like::
 
     In [1]:
 
 this means that your terminal doesn't properly handle color escape
 sequences. You can go to a 'no color' mode by typing '%colors NoColor'.
 
 You can try using a different terminal emulator program (Emacs users,
 see below). To permanently set your color preferences, edit the file
-$HOME/.ipython/ipythonrc and set the colors option to the desired value.
+$IPYTHON_DIR/ipythonrc and set the colors option to the desired value.
 
 
 Object details (types, docstrings, source code, etc.)
 =====================================================
 
 IPython has a set of special functions for studying the objects you are working
 with, discussed in detail :ref:`here <dynamic_object_info>`. But this system
 relies on passing information which is longer than your screen through a data
 pager, such as the common Unix less and more programs. In order to be able to
 see this information in color, your pager needs to be properly configured. I
 strongly recommend using less instead of more, as it seems that more simply can
 not understand colored text correctly.
 
 In order to configure less as your default pager, do the following:
 
    1. Set the environment PAGER variable to less.
    2. Set the environment LESS variable to -r (plus any other options
       you always want to pass to less by default). This tells less to
       properly interpret control sequences, which is how color
       information is given to your terminal.
 
 For the bash shell, add to your ~/.bashrc file the lines::
 
     export PAGER=less
     export LESS=-r
 
 For the csh or tcsh shells, add to your ~/.cshrc file the lines::
 
     setenv PAGER less
     setenv LESS -r
     
 There is similar syntax for other Unix shells, look at your system
 documentation for details.
 
 If you are on a system which lacks proper data pagers (such as Windows),
 IPython will use a very limited builtin pager.
 
 .. _Prompts:
 
 Fine-tuning your prompt
 =======================
 
 IPython's prompts can be customized using a syntax similar to that of
 the bash shell. Many of bash's escapes are supported, as well as a few
 additional ones. We list them below::
 
     \#
         the prompt/history count number. This escape is automatically
         wrapped in the coloring codes for the currently active color scheme. 
     \N
         the 'naked' prompt/history count number: this is just the number
         itself, without any coloring applied to it. This lets you produce
         numbered prompts with your own colors. 
     \D
         the prompt/history count, with the actual digits replaced by dots.
         Used mainly in continuation prompts (prompt_in2) 
     \w
         the current working directory 
     \W
         the basename of current working directory 
     \Xn
         where $n=0\ldots5.$ The current working directory, with $HOME
         replaced by ~, and filtered out to contain only $n$ path elements 
     \Yn
         Similar to \Xn, but with the $n+1$ element included if it is ~ (this
         is similar to the behavior of the %cn escapes in tcsh) 
     \u
         the username of the current user 
     \$
         if the effective UID is 0, a #, otherwise a $ 
     \h
         the hostname up to the first '.' 
     \H
         the hostname 
     \n
         a newline 
     \r
         a carriage return 
     \v
         IPython version string 
 
 In addition to these, ANSI color escapes can be insterted into the
 prompts, as \C_ColorName. The list of valid color names is: Black, Blue,
 Brown, Cyan, DarkGray, Green, LightBlue, LightCyan, LightGray,
 LightGreen, LightPurple, LightRed, NoColor, Normal, Purple, Red, White,
 Yellow.
 
 Finally, IPython supports the evaluation of arbitrary expressions in
 your prompt string. The prompt strings are evaluated through the syntax
 of PEP 215, but basically you can use $x.y to expand the value of x.y,
 and for more complicated expressions you can use braces: ${foo()+x} will
 call function foo and add to it the value of x, before putting the
 result into your prompt. For example, using
 prompt_in1 '${commands.getoutput("uptime")}\nIn [\#]: ' 
 will print the result of the uptime command on each prompt (assuming the
 commands module has been imported in your ipythonrc file).
 
 
       Prompt examples
 
 The following options in an ipythonrc file will give you IPython's
 default prompts::
 
     prompt_in1 'In [\#]:' 
     prompt_in2 '   .\D.:' 
     prompt_out 'Out[\#]:'
 
 which look like this::
 
     In [1]: 1+2 
     Out[1]: 3
 
     In [2]: for i in (1,2,3): 
        ...:    print i, 
        ...: 
     1 2 3
 
 These will give you a very colorful prompt with path information::
 
     #prompt_in1 '\C_Red\u\C_Blue[\C_Cyan\Y1\C_Blue]\C_LightGreen\#>' 
     prompt_in2 ' ..\D>' 
     prompt_out '<\#>'
 
 which look like this::
 
     fperez[~/ipython]1> 1+2 
                     <1> 3 
     fperez[~/ipython]2> for i in (1,2,3): 
                    ...>     print i, 
                    ...> 
     1 2 3
 
 

docs/source/config/overview.txt

@@ -1,329 +1,342 b''
 .. _config_overview:
 
 ============================================
 Overview of the IPython configuration system
 ============================================
 
 This section describes the IPython configuration system. Starting with version
 0.11, IPython has a completely new configuration system that is quite
 different from the older :file:`ipythonrc` or :file:`ipy_user_conf.py`
 approaches. The new configuration system was designed from scratch to address
 the particular configuration needs of IPython. While there are many
 other excellent configuration systems out there, we found that none of them
 met our requirements.
 
 .. warning::
 
     If you are upgrading to version 0.11 of IPython, you will need to migrate
     your old :file:`ipythonrc` or :file:`ipy_user_conf.py` configuration files
     to the new system.  Read on for information on how to do this.
 
 The discussion that follows is focused on teaching user's how to configure
 IPython to their liking.  Developer's who want to know more about how they
 can enable their objects to take advantage of the configuration system
 should consult our :ref:`developer guide <developer_guide>`
 
 The main concepts
 =================
 
 There are a number of abstractions that the IPython configuration system uses.
 Each of these abstractions is represented by a Python class.
 
 Configuration object: :class:`~IPython.config.loader.Config`
     A configuration object is a simple dictionary-like class that holds
     configuration attributes and sub-configuration objects. These classes
     support dotted attribute style access (``Foo.bar``) in addition to the
     regular dictionary style access (``Foo['bar']``). Configuration objects
     are smart. They know how to merge themselves with other configuration
     objects and they automatically create sub-configuration objects.
 
 Application: :class:`~IPython.core.application.Application`
     An application is a process that does a specific job. The most obvious
     application is the :command:`ipython` command line program. Each
     application reads a *single* configuration file and command line options
     and then produces a master configuration object for the application. This
     configuration object is then passed to the configurable objects that the
     application creates. These configurable objects implement the actual logic
     of the application and know how to configure themselves given the
     configuration object.
 
 Component: :class:`~IPython.config.configurable.Configurable`
     A configurable is a regular Python class that serves as a base class for
     all main classes in an application. The
     :class:`~IPython.config.configurable.Configurable` base class is
     lightweight and only does one things.
 
     This :class:`~IPython.config.configurable.Configurable` is a subclass
     of :class:`~IPython.utils.traitlets.HasTraits` that knows how to configure
     itself. Class level traits with the metadata ``config=True`` become
     values that can be configured from the command line and configuration
     files.
     
     Developers create :class:`~IPython.config.configurable.Configurable`
     subclasses that implement all of the logic in the application. Each of
     these subclasses has its own configuration information that controls how
     instances are created.
 
 Having described these main concepts, we can now state the main idea in our
 configuration system: *"configuration" allows the default values of class
 attributes to be controlled on a class by class basis*. Thus all instances of
 a given class are configured in the same way. Furthermore, if two instances
 need to be configured differently, they need to be instances of two different
 classes. While this model may seem a bit restrictive, we have found that it
 expresses most things that need to be configured extremely well. However, it
 is possible to create two instances of the same class that have different
 trait values. This is done by overriding the configuration.
 
 Now, we show what our configuration objects and files look like.
 
 Configuration objects and files
 ===============================
 
 A configuration file is simply a pure Python file that sets the attributes
 of a global, pre-created configuration object.  This configuration object is a 
 :class:`~IPython.config.loader.Config` instance.  While in a configuration
 file, to get a reference to this object, simply call the :func:`get_config`
 function.  We inject this function into the global namespace that the 
 configuration file is executed in.
 
 Here is an example of a super simple configuration file that does nothing::
 
     c = get_config()
 
 Once you get a reference to the configuration object, you simply set
 attributes on it.  All you have to know is:
 
 * The name of each attribute.
 * The type of each attribute.
 
 The answers to these two questions are provided by the various
 :class:`~IPython.config.configurable.Configurable` subclasses that an
 application uses. Let's look at how this would work for a simple component
 subclass::
 
     # Sample component that can be configured.
     from IPython.config.configurable import Configurable
     from IPython.utils.traitlets import Int, Float, Str, Bool
     
     class MyClass(Configurable):
         name = Str('defaultname', config=True)
         ranking = Int(0, config=True)
         value = Float(99.0)
         # The rest of the class implementation would go here..
 
 In this example, we see that :class:`MyClass` has three attributes, two
 of whom (``name``, ``ranking``) can be configured.  All of the attributes
 are given types and default values.  If a :class:`MyClass` is instantiated,
 but not configured, these default values will be used.  But let's see how
 to configure this class in a configuration file::
 
     # Sample config file
     c = get_config()
     
     c.MyClass.name = 'coolname'
     c.MyClass.ranking = 10
 
 After this configuration file is loaded, the values set in it will override
 the class defaults anytime a :class:`MyClass` is created.  Furthermore,
 these attributes will be type checked and validated anytime they are set.
 This type checking is handled by the :mod:`IPython.utils.traitlets` module,
 which provides the :class:`Str`, :class:`Int` and :class:`Float` types.  In 
 addition to these traitlets, the :mod:`IPython.utils.traitlets` provides
 traitlets for a number of other types.
 
 .. note::
 
     Underneath the hood, the :class:`Configurable` base class is a subclass of
     :class:`IPython.utils.traitlets.HasTraits`. The
     :mod:`IPython.utils.traitlets` module is a lightweight version of
     :mod:`enthought.traits`. Our implementation is a pure Python subset
     (mostly API compatible) of :mod:`enthought.traits` that does not have any
     of the automatic GUI generation capabilities. Our plan is to achieve 100%
     API compatibility to enable the actual :mod:`enthought.traits` to
     eventually be used instead. Currently, we cannot use
     :mod:`enthought.traits` as we are committed to the core of IPython being
     pure Python.
 
 It should be very clear at this point what the naming convention is for 
 configuration attributes::
 
     c.ClassName.attribute_name = attribute_value
 
 Here, ``ClassName`` is the name of the class whose configuration attribute you
 want to set, ``attribute_name`` is the name of the attribute you want to set
 and ``attribute_value`` the the value you want it to have. The ``ClassName``
 attribute of ``c`` is not the actual class, but instead is another
 :class:`~IPython.config.loader.Config` instance.
 
 .. note::
 
     The careful reader may wonder how the ``ClassName`` (``MyClass`` in
     the above example) attribute of the configuration object ``c`` gets
     created. These attributes are created on the fly by the
     :class:`~IPython.config.loader.Config` instance, using a simple naming
     convention. Any attribute of a :class:`~IPython.config.loader.Config`
     instance whose name begins with an uppercase character is assumed to be a
     sub-configuration and a new empty :class:`~IPython.config.loader.Config`
     instance is dynamically created for that attribute. This allows deeply
     hierarchical information created easily (``c.Foo.Bar.value``) on the fly.
 
 Configuration files inheritance
 ===============================
 
 Let's say you want to have different configuration files for various purposes.
 Our configuration system makes it easy for one configuration file to inherit
 the information in another configuration file. The :func:`load_subconfig`
 command can be used in a configuration file for this purpose. Here is a simple
 example that loads all of the values from the file :file:`base_config.py`::
 
     # base_config.py
     c = get_config()
     c.MyClass.name = 'coolname'
     c.MyClass.ranking = 100
 
 into the configuration file :file:`main_config.py`::
 
     # main_config.py
     c = get_config()
     
     # Load everything from base_config.py
     load_subconfig('base_config.py')
     
     # Now override one of the values
     c.MyClass.name = 'bettername'
 
 In a situation like this the :func:`load_subconfig` makes sure that the
 search path for sub-configuration files is inherited from that of the parent.
 Thus, you can typically put the two in the same directory and everything will
 just work.
 
 Class based configuration inheritance
 =====================================
 
 There is another aspect of configuration where inheritance comes into play.
 Sometimes, your classes will have an inheritance hierarchy that you want
 to be reflected in the configuration system.  Here is a simple example::
 
     from IPython.config.configurable import Configurable
     from IPython.utils.traitlets import Int, Float, Str, Bool
     
     class Foo(Configurable):
         name = Str('fooname', config=True)
         value = Float(100.0, config=True)
     
     class Bar(Foo):
         name = Str('barname', config=True)
         othervalue = Int(0, config=True)
 
 Now, we can create a configuration file to configure instances of :class:`Foo`
 and :class:`Bar`::
 
     # config file
     c = get_config()
     
     c.Foo.name = 'bestname'
     c.Bar.othervalue = 10
 
 This class hierarchy and configuration file accomplishes the following:
 
 * The default value for :attr:`Foo.name` and :attr:`Bar.name` will be
   'bestname'.  Because :class:`Bar` is a :class:`Foo` subclass it also
   picks up the configuration information for :class:`Foo`.
 * The default value for :attr:`Foo.value` and :attr:`Bar.value` will be
   ``100.0``, which is the value specified as the class default.
 * The default value for :attr:`Bar.othervalue` will be 10 as set in the
   configuration file.  Because :class:`Foo` is the parent of :class:`Bar`
   it doesn't know anything about the :attr:`othervalue` attribute.
 
+
+.. _ipython_dir:
+
 Configuration file location
 ===========================
 
 So where should you put your configuration files?  By default, all IPython
 applications look in the so called "IPython directory".  The location of
 this directory is determined by the following algorithm:
 
 * If the ``--ipython-dir`` command line flag is given, its value is used.
 
 * If not, the value returned by :func:`IPython.utils.path.get_ipython_dir`
   is used. This function will first look at the :envvar:`IPYTHON_DIR`
-  environment variable and then default to the directory
-  :file:`$HOME/.ipython`.
+  environment variable and then default to a platform-specific default.
+
+On posix systems (Linux, Unix, etc.), IPython respects the ``$XDG_CONFIG_HOME``
+part of the `XDG Base Directory`_ specification. If ``$XDG_CONFIG_HOME`` is
+defined and exists ( ``XDG_CONFIG_HOME`` has a default interpretation of
+:file:`$HOME/.config`), then IPython's config directory will be located in
+:file:`$XDG_CONFIG_HOME/ipython`.  If users still have an IPython directory
+in :file:`$HOME/.ipython`, then that will be used. in preference to the 
+system default.
 
 For most users, the default value will simply be something like 
-:file:`$HOME/.ipython`.
+:file:`$HOME/.config/ipython` on Linux, or :file:`$HOME/.ipython`
+elsewhere.
 
 Once the location of the IPython directory has been determined, you need to
 know what filename to use for the configuration file. The basic idea is that
 each application has its own default configuration filename. The default named
 used by the :command:`ipython` command line program is
 :file:`ipython_config.py`.  This value can be overriden by the ``-config_file``
 command line flag.  A sample :file:`ipython_config.py` file can be found
 in :mod:`IPython.config.default.ipython_config.py`.  Simple copy it to your
 IPython directory to begin using it.
 
 .. _Profiles:
 
 Profiles
 ========
 
 A profile is simply a configuration file that follows a simple naming
 convention and can be loaded using a simplified syntax.  The idea is
 that users often want to maintain a set of configuration files for different
 purposes:  one for doing numerical computing with NumPy and SciPy and
 another for doing symbolic computing with SymPy.  Profiles make it easy
 to keep a separate configuration file for each of these purposes.
 
 Let's start by showing how a profile is used:
 
 .. code-block:: bash
 
     $ ipython -p sympy
 
 This tells the :command:`ipython` command line program to get its
 configuration from the "sympy" profile. The search path for profiles is the
 same as that of regular configuration files. The only difference is that
 profiles are named in a special way. In the case above, the "sympy" profile
 would need to have the name :file:`ipython_config_sympy.py`.
 
 The general pattern is this: simply add ``_profilename`` to the end of the
 normal configuration file name. Then load the profile by adding ``-p
 profilename`` to your command line options.
 
 IPython ships with some sample profiles in :mod:`IPython.config.profile`.
 Simply copy these to your IPython directory to begin using them.
 
 Design requirements
 ===================
 
 Here are the main requirements we wanted our configuration system to have:
 
 * Support for hierarchical configuration information.
 
 * Full integration with command line option parsers.  Often, you want to read
   a configuration file, but then override some of the values with command line
   options.  Our configuration system automates this process and allows each
   command line option to be linked to a particular attribute in the 
   configuration hierarchy that it will override.
 
 * Configuration files that are themselves valid Python code. This accomplishes
   many things. First, it becomes possible to put logic in your configuration
   files that sets attributes based on your operating system, network setup,
   Python version, etc. Second, Python has a super simple syntax for accessing
   hierarchical data structures, namely regular attribute access
   (``Foo.Bar.Bam.name``). Third, using Python makes it easy for users to
   import configuration attributes from one configuration file to another.
   Forth, even though Python is dynamically typed, it does have types that can
   be checked at runtime. Thus, a ``1`` in a config file is the integer '1',
   while a ``'1'`` is a string.
 
 * A fully automated method for getting the configuration information to the
   classes that need it at runtime. Writing code that walks a configuration
   hierarchy to extract a particular attribute is painful. When you have
   complex configuration information with hundreds of attributes, this makes
   you want to cry.
 
 * Type checking and validation that doesn't require the entire configuration
   hierarchy to be specified statically before runtime. Python is a very
   dynamic language and you don't always know everything that needs to be
   configured when a program starts.
 
+
+.. _`XDG Base Directory`: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html

docs/source/development/doc_guide.txt

@@ -1,167 +1,167 b''
 .. _documenting-ipython:
 
 =====================
  Documenting IPython
 =====================
 
 When contributing code to IPython, you should strive for clarity and
 consistency, without falling prey to a style straitjacket.  Basically,
 'document everything, try to be consistent, do what makes sense.'
 
 By and large we follow existing Python practices in major projects like Python
 itself or NumPy, this document provides some additional detail for IPython.
 
 
 Standalone documentation
 ========================
 
 All standalone documentation should be written in plain text (``.txt``) files
 using reStructuredText [reStructuredText]_ for markup and formatting. All such
 documentation should be placed in the directory :file:`docs/source` of the
 IPython source tree. Or, when appropriate, a suitably named subdirectory
 should be used. The documentation in this location will serve as the main
 source for IPython documentation.
 
 The actual HTML and PDF docs are built using the Sphinx [Sphinx]_
 documentation generation tool. Once you have Sphinx installed, you can build
 the html docs yourself by doing:
 
 .. code-block:: bash
 
     $ cd ipython-mybranch/docs
     $ make html
 
 Our usage of Sphinx follows that of matplotlib [Matplotlib]_ closely. We are
 using a number of Sphinx tools and extensions written by the matplotlib team
 and will mostly follow their conventions, which are nicely spelled out in
 their documentation guide [MatplotlibDocGuide]_. What follows is thus a
 abridged version of the matplotlib documentation guide, taken with permission
 from the matplotlib team.
 
 If you are reading this in a web browser, you can click on the "Show Source"
 link to see the original reStricturedText for the following examples.
 
 A bit of Python code::
 
     for i in range(10):
         print i,
     print "A big number:",2**34
 
 An interactive Python session::
 
     >>> from IPython.utils.path import get_ipython_dir
     >>> get_ipython_dir()
-    '/home/fperez/.ipython'
+    '/home/fperez/.config/ipython'
 
 An IPython session:
 
 .. code-block:: ipython
 
   In [7]: import IPython
 
   In [8]: print "This IPython is version:",IPython.__version__
   This IPython is version: 0.9.1
 
   In [9]: 2+4
   Out[9]: 6
 
 
 A bit of shell code:
 
 .. code-block:: bash
 
     cd /tmp
     echo "My home directory is: $HOME"
     ls
 
 Docstring format
 ================
 
 Good docstrings are very important.  Unfortunately, Python itself only provides
 a rather loose standard for docstrings [PEP257]_, and there is no universally
 accepted convention for all the different parts of a complete docstring.
 However, the NumPy project has established a very reasonable standard, and has
 developed some tools to support the smooth inclusion of such docstrings in
 Sphinx-generated manuals.  Rather than inventing yet another pseudo-standard,
 IPython will be henceforth documented using the NumPy conventions; we carry
 copies of some of the NumPy support tools to remain self-contained, but share
 back upstream with NumPy any improvements or fixes we may make to the tools.
 
 The NumPy documentation guidelines [NumPyDocGuide]_ contain detailed
 information on this standard, and for a quick overview, the NumPy example
 docstring [NumPyExampleDocstring]_ is a useful read.
 
 
 For user-facing APIs, we try to be fairly strict about following the above
 standards (even though they mean more verbose and detailed docstrings).
 Wherever you can reasonably expect people to do introspection with::
 
   In [1]: some_function?
 
 the docstring should follow the NumPy style and be fairly detailed.
 
 For purely internal methods that are only likely to be read by others extending
 IPython itself we are a bit more relaxed, especially for small/short methods
 and functions whose intent is reasonably obvious.  We still expect docstrings
 to be written, but they can be simpler.  For very short functions with a
 single-line docstring you can use something like::
 
     def add(a, b):
        """The sum of two numbers.
        """
        code
 
 and for longer multiline strings::
 
     def add(a, b):
        """The sum of two numbers.
 
        Here is the rest of the docs.
        """
        code
 
 
 Here are two additional PEPs of interest regarding documentation of code.
 While both of these were rejected, the ideas therein form much of the basis of
 docutils (the machinery to process reStructuredText):
 
 * `Docstring Processing System Framework <http://www.python.org/peps/pep-0256.html>`_
 * `Docutils Design Specification <http://www.python.org/peps/pep-0258.html>`_
 
 .. note::
 
    In the past IPython used epydoc so currently many docstrings still use
    epydoc conventions.  We will update them as we go, but all new code should
    be documented using the NumPy standard.
    
 Building and uploading
 ======================
 The built docs are stored in a separate repository. Through some github magic,
 they're automatically exposed as a website. It works like this:
 
 * You will need to have sphinx and latex installed. In Ubuntu, install
   ``texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended``.
   Install the latest version of sphinx from PyPI (``pip install sphinx``).
 * Ensure that the development version of IPython is the first in your system
   path. You can either use a virtualenv, or modify your PYTHONPATH.
 * Switch into the docs directory, and run ``make gh-pages``. This will build
   your updated docs as html and pdf, then automatically check out the latest
   version of the docs repository, copy the built docs into it, and commit your
   changes.
 * Open the built docs in a web browser, and check that they're as expected.
 * (When building the docs for a new tagged release, you will have to add its link to
   index.rst, then run ``python build_index.py`` to update index.html. Commit the 
   change.)
 * Upload the docs with ``git push``. This only works if you have write access to
   the docs repository.
 * If you are building a version that is not the current dev branch, nor a tagged release,
   then you must run gh-pages.py directly with ``python gh-pages.py <version>``, and *not*
   with ``make gh-pages``.
 
 .. [reStructuredText] reStructuredText.  http://docutils.sourceforge.net/rst.html
 .. [Sphinx] Sphinx. http://sphinx.pocoo.org/
 .. [MatplotlibDocGuide] http://matplotlib.sourceforge.net/devel/documenting_mpl.html
 .. [PEP257] PEP 257.  http://www.python.org/peps/pep-0257.html
 .. [NumPyDocGuide] NumPy documentation guide.  http://projects.scipy.org/numpy/wiki/CodingStyleGuidelines
 .. [NumPyExampleDocstring] NumPy example docstring. http://projects.scipy.org/numpy/browser/trunk/doc/EXAMPLE_DOCSTRING.txt
 

docs/source/interactive/reference.txt

@@ -1,1577 +1,1578 b''
 =================
 IPython reference
 =================
 
 .. warning::
 
     As of the 0.11 version of IPython, some of the features and APIs
     described in this section have been deprecated or are broken.  Our plan
     is to continue to support these features, but they need to be updated
     to take advantage of recent API changes.  Furthermore, this section
     of the documentation need to be updated to reflect all of these changes.
 
 .. _command_line_options:
 
 Command-line usage
 ==================
 
 You start IPython with the command::
 
     $ ipython [options] files
 
 If invoked with no options, it executes all the files listed in sequence
 and drops you into the interpreter while still acknowledging any options
 you may have set in your ipythonrc file. This behavior is different from
 standard Python, which when called as python -i will only execute one
 file and ignore your configuration setup.
 
 Please note that some of the configuration options are not available at
 the command line, simply because they are not practical here. Look into
-your ipythonrc configuration file for details on those. This file
-typically installed in the $HOME/.ipython directory. For Windows users,
-$HOME resolves to C:\\Documents and Settings\\YourUserName in most
-instances. In the rest of this text, we will refer to this directory as
-IPYTHON_DIR.
+your ipythonrc configuration file for details on those. This file is typically 
+installed in the IPYTHON_DIR directory. For Linux
+users, this will be $HOME/.config/ipython, and for other users it will be
+$HOME/.ipython.  For Windows users, $HOME resolves to C:\\Documents and
+Settings\\YourUserName in most instances.
+
 
 
 
 Special Threading Options
 -------------------------
 
 Previously IPython had command line options for controlling GUI event loop
 integration (-gthread, -qthread, -q4thread, -wthread, -pylab). As of IPython
 version 0.11, these have been deprecated. Please see the new ``%gui``
 magic command or :ref:`this section <gui_support>` for details on the new
 interface.
 
 Regular Options
 ---------------
 
 After the above threading options have been given, regular options can
 follow in any order. All options can be abbreviated to their shortest
 non-ambiguous form and are case-sensitive. One or two dashes can be
 used. Some options have an alternate short form, indicated after a ``|``.
 
 Most options can also be set from your ipythonrc configuration file. See
 the provided example for more details on what the options do. Options
 given at the command line override the values set in the ipythonrc file.
 
 All options with a [no] prepended can be specified in negated form
 (-nooption instead of -option) to turn the feature off.
 
     -help  print a help message and exit.
 
     -pylab
     Deprecated.  See :ref:`Matplotlib support <matplotlib_support>`
     for more details.
 
     -autocall <val>
 	Make IPython automatically call any callable object even if you
 	didn't type explicit parentheses. For example, 'str 43' becomes
 	'str(43)' automatically. The value can be '0' to disable the feature,
 	'1' for smart autocall, where it is not applied if there are no more
 	arguments on the line, and '2' for full autocall, where all callable
 	objects are automatically called (even if no arguments are
 	present). The default is '1'.
 
     -[no]autoindent    
 	Turn automatic indentation on/off.
 
     -[no]automagic 
 	make magic commands automatic (without needing their first character
 	to be %). Type %magic at the IPython prompt for more information.
 
     -[no]autoedit_syntax 
 	When a syntax error occurs after editing a file, automatically
 	open the file to the trouble causing line for convenient
 	fixing.
 
     -[no]banner		Print the initial information banner (default on).
 
     -c <command>
 	execute the given command string. This is similar to the -c
 	option in the normal Python interpreter.
 
     -cache_size, cs <n> 
 	size of the output cache (maximum number of entries to hold in
 	memory). The default is 1000, you can change it permanently in your
 	config file. Setting it to 0 completely disables the caching system,
 	and the minimum value accepted is 20 (if you provide a value less than
 	20, it is reset to 0 and a warning is issued) This limit is defined
 	because otherwise you'll spend more time re-flushing a too small cache
 	than working.
 
     -classic, cl 
 	Gives IPython a similar feel to the classic Python
 	prompt.
 
     -colors <scheme> 
 	Color scheme for prompts and exception reporting. Currently
 	implemented: NoColor, Linux and LightBG.
 
     -[no]color_info 
 	IPython can display information about objects via a set of functions,
 	and optionally can use colors for this, syntax highlighting source
 	code and various other elements.  However, because this information is
 	passed through a pager (like 'less') and many pagers get confused with
 	color codes, this option is off by default. You can test it and turn
 	it on permanently in your ipythonrc file if it works for you. As a
 	reference, the 'less' pager supplied with Mandrake 8.2 works ok, but
 	that in RedHat 7.2 doesn't.
 
 	Test it and turn it on permanently if it works with your
 	system. The magic function %color_info allows you to toggle this
 	interactively for testing.
 
     -[no]debug 
 	Show information about the loading process. Very useful to pin down
 	problems with your configuration files or to get details about
 	session restores.
 
     -[no]deep_reload: 
 	IPython can use the deep_reload module which reloads changes in
 	modules recursively (it replaces the reload() function, so you don't
 	need to change anything to use it).  deep_reload() forces a full
 	reload of modules whose code may have changed, which the default
 	reload() function does not.
 
 	When deep_reload is off, IPython will use the normal reload(),
 	but deep_reload will still be available as dreload(). This
 	feature is off by default [which means that you have both
 	normal reload() and dreload()].
 
     -editor <name>
 	Which editor to use with the %edit command. By default,
 	IPython will honor your EDITOR environment variable (if not
 	set, vi is the Unix default and notepad the Windows one).
 	Since this editor is invoked on the fly by IPython and is
 	meant for editing small code snippets, you may want to use a
 	small, lightweight editor here (in case your default EDITOR is
 	something like Emacs).
 
     -ipythondir <name> 
 	name of your IPython configuration directory IPYTHON_DIR. This
 	can also be specified through the environment variable
 	IPYTHON_DIR.
 
     -log, l 
 	generate a log file of all input. The file is named
 	ipython_log.py in your current directory (which prevents logs
 	from multiple IPython sessions from trampling each other). You
 	can use this to later restore a session by loading your
 	logfile as a file to be executed with option -logplay (see
 	below).
 
     -logfile, lf <name>   specify the name of your logfile.
 
     -logplay, lp <name> 
 
       you can replay a previous log. For restoring a session as close as
       possible to the state you left it in, use this option (don't just run
       the logfile). With -logplay, IPython will try to reconstruct the
       previous working environment in full, not just execute the commands in
       the logfile.
 
       When a session is restored, logging is automatically turned on
       again with the name of the logfile it was invoked with (it is
       read from the log header). So once you've turned logging on for
       a session, you can quit IPython and reload it as many times as
       you want and it will continue to log its history and restore
       from the beginning every time.
       
       Caveats: there are limitations in this option. The history
       variables _i*,_* and _dh don't get restored properly. In the
       future we will try to implement full session saving by writing
       and retrieving a 'snapshot' of the memory state of IPython. But
       our first attempts failed because of inherent limitations of
       Python's Pickle module, so this may have to wait.
 
     -[no]messages   
 	Print messages which IPython collects about its startup
 	process (default on).
 
     -[no]pdb
 	Automatically call the pdb debugger after every uncaught
 	exception. If you are used to debugging using pdb, this puts
 	you automatically inside of it after any call (either in
 	IPython or in code called by it) which triggers an exception
 	which goes uncaught.
 
     -pydb
 	Makes IPython use the third party "pydb" package as debugger,
 	instead of pdb. Requires that pydb is installed.
 	
     -[no]pprint
 	ipython can optionally use the pprint (pretty printer) module
 	for displaying results. pprint tends to give a nicer display
 	of nested data structures. If you like it, you can turn it on
 	permanently in your config file (default off).
 
     -profile, p <name>
 
 	assume that your config file is ipythonrc-<name> or
 	ipy_profile_<name>.py (looks in current dir first, then in
 	IPYTHON_DIR).  This is a quick way to keep and load multiple
 	config files for different tasks, especially if you use the
 	include option of config files. You can keep a basic
 	IPYTHON_DIR/ipythonrc file and then have other 'profiles' which
 	include this one and load extra things for particular
 	tasks. For example:
 
-	1. $HOME/.ipython/ipythonrc : load basic things you always want.
-	2. $HOME/.ipython/ipythonrc-math : load (1) and basic math-related modules.
-	3. $HOME/.ipython/ipythonrc-numeric : load (1) and Numeric and plotting modules.
+	1. $IPYTHON_DIR/ipythonrc : load basic things you always want.
+	2. $IPYTHON_DIR/ipythonrc-math : load (1) and basic math-related modules.
+	3. $IPYTHON_DIR/ipythonrc-numeric : load (1) and Numeric and plotting modules.
 
 	Since it is possible to create an endless loop by having
 	circular file inclusions, IPython will stop if it reaches 15
 	recursive inclusions.
 
     -prompt_in1, pi1 <string>
 
         Specify the string used for input prompts. Note that if you are using
 	numbered prompts, the number is represented with a '\#' in the
 	string. Don't forget to quote strings with spaces embedded in
 	them. Default: 'In [\#]:'.  The :ref:`prompts section <prompts>`
 	discusses in detail all the available escapes to customize your
 	prompts.
 
     -prompt_in2, pi2 <string>
 	Similar to the previous option, but used for the continuation
 	prompts. The special sequence '\D' is similar to '\#', but
 	with all digits replaced dots (so you can have your
 	continuation prompt aligned with your input prompt).  Default:
 	' .\D.:' (note three spaces at the start for alignment with
 	'In [\#]').
 
     -prompt_out,po <string> 
 	String used for output prompts, also uses numbers like
 	prompt_in1. Default: 'Out[\#]:'
 
     -quick   start in bare bones mode (no config file loaded).
 
     -rcfile <name>   
 	name of your IPython resource configuration file. Normally
 	IPython loads ipythonrc (from current directory) or
 	IPYTHON_DIR/ipythonrc.
 
 	If the loading of your config file fails, IPython starts with
 	a bare bones configuration (no modules loaded at all).
 
     -[no]readline
 	use the readline library, which is needed to support name
 	completion and command history, among other things.  It is
 	enabled by default, but may cause problems for users of
 	X/Emacs in Python comint or shell buffers.
 
 	Note that X/Emacs 'eterm' buffers (opened with M-x term) support
 	IPython's readline and syntax coloring fine, only 'emacs' (M-x
 	shell and C-c !) buffers do not.
 
     -screen_length, sl <n>
 	number of lines of your screen. This is used to control
 	printing of very long strings. Strings longer than this number
 	of lines will be sent through a pager instead of directly
 	printed.
 
 	The default value for this is 0, which means IPython will
 	auto-detect your screen size every time it needs to print certain
 	potentially long strings (this doesn't change the behavior of the
 	'print' keyword, it's only triggered internally). If for some
 	reason this isn't working well (it needs curses support), specify
 	it yourself. Otherwise don't change the default.
 
     -separate_in, si <string>
 
 	separator before input prompts.
 	Default: '\n'
 
     -separate_out, so <string>  
         separator before output prompts.
         Default: nothing.
 
     -separate_out2, so2
 	separator after output prompts.
 	Default: nothing.
 	For these three options, use the value 0 to specify no separator.
 
     -nosep   
 	shorthand for '-SeparateIn 0 -SeparateOut 0 -SeparateOut2
 	0'. Simply removes all input/output separators.
 
     -upgrade 
 	allows you to upgrade your IPYTHON_DIR configuration when you
 	install a new version of IPython. Since new versions may
 	include new command line options or example files, this copies
 	updated ipythonrc-type files. However, it backs up (with a
 	.old extension) all files which it overwrites so that you can
 	merge back any customizations you might have in your personal
 	files. Note that you should probably use %upgrade instead,
 	it's a safer alternative.
 
 
     -Version    print version information and exit.
 
     -wxversion <string> 
     Deprecated.
 
     -xmode <modename>
 
 	Mode for exception reporting.
 
 	Valid modes: Plain, Context and Verbose.
 
 	* Plain: similar to python's normal traceback printing.
 	* Context: prints 5 lines of context source code around each
 	  line in the traceback.
 	* Verbose: similar to Context, but additionally prints the
 	  variables currently visible where the exception happened
 	  (shortening their strings if too long). This can potentially be
 	  very slow, if you happen to have a huge data structure whose
 	  string representation is complex to compute. Your computer may
 	  appear to freeze for a while with cpu usage at 100%. If this
 	  occurs, you can cancel the traceback with Ctrl-C (maybe hitting it
 	  more than once).
 
 Interactive use
 ===============
 
 Warning: IPython relies on the existence of a global variable called
 _ip which controls the shell itself. If you redefine _ip to anything,
 bizarre behavior will quickly occur.
 
 Other than the above warning, IPython is meant to work as a drop-in
 replacement for the standard interactive interpreter. As such, any code
 which is valid python should execute normally under IPython (cases where
 this is not true should be reported as bugs). It does, however, offer
 many features which are not available at a standard python prompt. What
 follows is a list of these.
 
 
 Caution for Windows users
 -------------------------
 
 Windows, unfortunately, uses the '\' character as a path
 separator. This is a terrible choice, because '\' also represents the
 escape character in most modern programming languages, including
 Python. For this reason, using '/' character is recommended if you
 have problems with ``\``.  However, in Windows commands '/' flags
 options, so you can not use it for the root directory. This means that
 paths beginning at the root must be typed in a contrived manner like:
 ``%copy \opt/foo/bar.txt \tmp``
 
 .. _magic:
 
 Magic command system
 --------------------
 
 IPython will treat any line whose first character is a % as a special
 call to a 'magic' function. These allow you to control the behavior of
 IPython itself, plus a lot of system-type features. They are all
 prefixed with a % character, but parameters are given without
 parentheses or quotes.
 
 Example: typing '%cd mydir' (without the quotes) changes you working
 directory to 'mydir', if it exists.
 
 If you have 'automagic' enabled (in your ipythonrc file, via the command
 line option -automagic or with the %automagic function), you don't need
 to type in the % explicitly. IPython will scan its internal list of
 magic functions and call one if it exists. With automagic on you can
 then just type 'cd mydir' to go to directory 'mydir'. The automagic
 system has the lowest possible precedence in name searches, so defining
 an identifier with the same name as an existing magic function will
 shadow it for automagic use. You can still access the shadowed magic
 function by explicitly using the % character at the beginning of the line.
 
 An example (with automagic on) should clarify all this::
 
     In [1]: cd ipython # %cd is called by automagic
 
     /home/fperez/ipython
 
     In [2]: cd=1 # now cd is just a variable
 
     In [3]: cd .. # and doesn't work as a function anymore
 
     ------------------------------
 
         File "<console>", line 1
 
           cd ..
 
               ^
 
     SyntaxError: invalid syntax
 
     In [4]: %cd .. # but %cd always works
 
     /home/fperez
 
     In [5]: del cd # if you remove the cd variable
 
     In [6]: cd ipython # automagic can work again
 
     /home/fperez/ipython
 
 You can define your own magic functions to extend the system. The
 following example defines a new magic command, %impall::
 
     import IPython.ipapi 
 
     ip = IPython.ipapi.get()
 
     def doimp(self, arg):
 
         ip = self.api
 
         ip.ex("import %s; reload(%s); from %s import *" % (
 
         arg,arg,arg)
 
         )
 
     ip.expose_magic('impall', doimp)
 
 You can also define your own aliased names for magic functions. In your
 ipythonrc file, placing a line like::
 
     execute __IP.magic_cl = __IP.magic_clear
 
 will define %cl as a new name for %clear.
 
 Type %magic for more information, including a list of all available
 magic functions at any time and their docstrings. You can also type
 %magic_function_name? (see sec. 6.4 <#sec:dyn-object-info> for
 information on the '?' system) to get information about any particular
 magic function you are interested in.
 
 The API documentation for the :mod:`IPython.Magic` module contains the full
 docstrings of all currently available magic commands.
 
 
 Access to the standard Python help
 ----------------------------------
 
 As of Python 2.1, a help system is available with access to object docstrings
 and the Python manuals. Simply type 'help' (no quotes) to access it. You can
 also type help(object) to obtain information about a given object, and
 help('keyword') for information on a keyword. As noted :ref:`here
 <accessing_help>`, you need to properly configure your environment variable
 PYTHONDOCS for this feature to work correctly.
 
 .. _dynamic_object_info:
 
 Dynamic object information
 --------------------------
 
 Typing ?word or word? prints detailed information about an object. If
 certain strings in the object are too long (docstrings, code, etc.) they
 get snipped in the center for brevity. This system gives access variable
 types and values, full source code for any object (if available),
 function prototypes and other useful information.
 
 Typing ??word or word?? gives access to the full information without
 snipping long strings. Long strings are sent to the screen through the
 less pager if longer than the screen and printed otherwise. On systems
 lacking the less command, IPython uses a very basic internal pager.
 
 The following magic functions are particularly useful for gathering
 information about your working environment. You can get more details by
 typing %magic or querying them individually (use %function_name? with or
 without the %), this is just a summary:
 
     * **%pdoc <object>**: Print (or run through a pager if too long) the
       docstring for an object. If the given object is a class, it will
       print both the class and the constructor docstrings.
     * **%pdef <object>**: Print the definition header for any callable
       object. If the object is a class, print the constructor information.
     * **%psource <object>**: Print (or run through a pager if too long)
       the source code for an object.
     * **%pfile <object>**: Show the entire source file where an object was
       defined via a pager, opening it at the line where the object
       definition begins.
     * **%who/%whos**: These functions give information about identifiers
       you have defined interactively (not things you loaded or defined
       in your configuration files). %who just prints a list of
       identifiers and %whos prints a table with some basic details about
       each identifier.
 
 Note that the dynamic object information functions (?/??, %pdoc, %pfile,
 %pdef, %psource) give you access to documentation even on things which
 are not really defined as separate identifiers. Try for example typing
 {}.get? or after doing import os, type os.path.abspath??.
 
 
 .. _readline:
 
 Readline-based features
 -----------------------
 
 These features require the GNU readline library, so they won't work if
 your Python installation lacks readline support. We will first describe
 the default behavior IPython uses, and then how to change it to suit
 your preferences.
 
 
 Command line completion
 +++++++++++++++++++++++
 
 At any time, hitting TAB will complete any available python commands or
 variable names, and show you a list of the possible completions if
 there's no unambiguous one. It will also complete filenames in the
 current directory if no python names match what you've typed so far.
 
 
 Search command history
 ++++++++++++++++++++++
 
 IPython provides two ways for searching through previous input and thus
 reduce the need for repetitive typing:
 
    1. Start typing, and then use Ctrl-p (previous,up) and Ctrl-n
       (next,down) to search through only the history items that match
       what you've typed so far. If you use Ctrl-p/Ctrl-n at a blank
       prompt, they just behave like normal arrow keys.
    2. Hit Ctrl-r: opens a search prompt. Begin typing and the system
       searches your history for lines that contain what you've typed so
       far, completing as much as it can.
 
 
 Persistent command history across sessions
 ++++++++++++++++++++++++++++++++++++++++++
 
 IPython will save your input history when it leaves and reload it next
 time you restart it. By default, the history file is named
 $IPYTHON_DIR/history, but if you've loaded a named profile,
 '-PROFILE_NAME' is appended to the name. This allows you to keep
 separate histories related to various tasks: commands related to
 numerical work will not be clobbered by a system shell history, for
 example.
 
 
 Autoindent
 ++++++++++
 
 IPython can recognize lines ending in ':' and indent the next line,
 while also un-indenting automatically after 'raise' or 'return'.
 
 This feature uses the readline library, so it will honor your ~/.inputrc
 configuration (or whatever file your INPUTRC variable points to). Adding
 the following lines to your .inputrc file can make indenting/unindenting
 more convenient (M-i indents, M-u unindents)::
 
     $if Python 
     "\M-i": "    "  
     "\M-u": "\d\d\d\d"  
     $endif
 
 Note that there are 4 spaces between the quote marks after "M-i" above.
 
 Warning: this feature is ON by default, but it can cause problems with
 the pasting of multi-line indented code (the pasted code gets
 re-indented on each line). A magic function %autoindent allows you to
 toggle it on/off at runtime. You can also disable it permanently on in
 your ipythonrc file (set autoindent 0).
 
 
 Customizing readline behavior
 +++++++++++++++++++++++++++++
 
 All these features are based on the GNU readline library, which has an
 extremely customizable interface. Normally, readline is configured via a
 file which defines the behavior of the library; the details of the
 syntax for this can be found in the readline documentation available
 with your system or on the Internet. IPython doesn't read this file (if
 it exists) directly, but it does support passing to readline valid
 options via a simple interface. In brief, you can customize readline by
 setting the following options in your ipythonrc configuration file (note
 that these options can not be specified at the command line):
 
     * **readline_parse_and_bind**: this option can appear as many times as
       you want, each time defining a string to be executed via a
       readline.parse_and_bind() command. The syntax for valid commands
       of this kind can be found by reading the documentation for the GNU
       readline library, as these commands are of the kind which readline
       accepts in its configuration file.
     * **readline_remove_delims**: a string of characters to be removed
       from the default word-delimiters list used by readline, so that
       completions may be performed on strings which contain them. Do not
       change the default value unless you know what you're doing.
     * **readline_omit__names**: when tab-completion is enabled, hitting
       <tab> after a '.' in a name will complete all attributes of an
       object, including all the special methods whose names include
       double underscores (like __getitem__ or __class__). If you'd
       rather not see these names by default, you can set this option to
       1. Note that even when this option is set, you can still see those
       names by explicitly typing a _ after the period and hitting <tab>:
       'name._<tab>' will always complete attribute names starting with '_'.
       
       This option is off by default so that new users see all
       attributes of any objects they are dealing with.
 
 You will find the default values along with a corresponding detailed
 explanation in your ipythonrc file.
 
 
 Session logging and restoring
 -----------------------------
 
 You can log all input from a session either by starting IPython with the
 command line switches -log or -logfile (see :ref:`here <command_line_options>`)
 or by activating the logging at any moment with the magic function %logstart.
 
 Log files can later be reloaded with the -logplay option and IPython
 will attempt to 'replay' the log by executing all the lines in it, thus
 restoring the state of a previous session. This feature is not quite
 perfect, but can still be useful in many cases.
 
 The log files can also be used as a way to have a permanent record of
 any code you wrote while experimenting. Log files are regular text files
 which you can later open in your favorite text editor to extract code or
 to 'clean them up' before using them to replay a session.
 
 The %logstart function for activating logging in mid-session is used as
 follows:
 
 %logstart [log_name [log_mode]]
 
 If no name is given, it defaults to a file named 'log' in your
 IPYTHON_DIR directory, in 'rotate' mode (see below).
 
 '%logstart name' saves to file 'name' in 'backup' mode. It saves your
 history up to that point and then continues logging.
 
 %logstart takes a second optional parameter: logging mode. This can be
 one of (note that the modes are given unquoted):
 
     * [over:] overwrite existing log_name.
     * [backup:] rename (if exists) to log_name~ and start log_name.
     * [append:] well, that says it.
     * [rotate:] create rotating logs log_name.1~, log_name.2~, etc.
 
 The %logoff and %logon functions allow you to temporarily stop and
 resume logging to a file which had previously been started with
 %logstart. They will fail (with an explanation) if you try to use them
 before logging has been started.
 
 .. _system_shell_access:
 
 System shell access
 -------------------
 
 Any input line beginning with a ! character is passed verbatim (minus
 the !, of course) to the underlying operating system. For example,
 typing !ls will run 'ls' in the current directory.
 
 Manual capture of command output
 --------------------------------
 
 If the input line begins with two exclamation marks, !!, the command is
 executed but its output is captured and returned as a python list, split
 on newlines. Any output sent by the subprocess to standard error is
 printed separately, so that the resulting list only captures standard
 output. The !! syntax is a shorthand for the %sx magic command.
 
 Finally, the %sc magic (short for 'shell capture') is similar to %sx,
 but allowing more fine-grained control of the capture details, and
 storing the result directly into a named variable. The direct use of 
 %sc is now deprecated, and you should ise the ``var = !cmd`` syntax
 instead.
 
 IPython also allows you to expand the value of python variables when
 making system calls. Any python variable or expression which you prepend
 with $ will get expanded before the system call is made::
 
     In [1]: pyvar='Hello world' 
     In [2]: !echo "A python variable: $pyvar"  
     A python variable: Hello world
 
 If you want the shell to actually see a literal $, you need to type it
 twice::
 
     In [3]: !echo "A system variable: $$HOME"  
     A system variable: /home/fperez
 
 You can pass arbitrary expressions, though you'll need to delimit them
 with {} if there is ambiguity as to the extent of the expression::
 
     In [5]: x=10 
     In [6]: y=20 
     In [13]: !echo $x+y  
     10+y  
     In [7]: !echo ${x+y}  
     30
 
 Even object attributes can be expanded::
 
     In [12]: !echo $sys.argv  
     [/home/fperez/usr/bin/ipython]
 
 
 System command aliases
 ----------------------
 
 The %alias magic function and the alias option in the ipythonrc
 configuration file allow you to define magic functions which are in fact
 system shell commands. These aliases can have parameters.
 
 '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
 
 Then, typing '%alias_name params' will execute the system command 'cmd
 params' (from your underlying operating system).
 
 You can also define aliases with parameters using %s specifiers (one per
 parameter). The following example defines the %parts function as an
 alias to the command 'echo first %s second %s' where each %s will be
 replaced by a positional parameter to the call to %parts::
 
     In [1]: alias parts echo first %s second %s 
     In [2]: %parts A B 
     first A second B  
     In [3]: %parts A  
     Incorrect number of arguments: 2 expected.  
     parts is an alias to: 'echo first %s second %s'
 
 If called with no parameters, %alias prints the table of currently
 defined aliases.
 
 The %rehash/rehashx magics allow you to load your entire $PATH as
 ipython aliases. See their respective docstrings (or sec. 6.2
 <#sec:magic> for further details).
 
 
 .. _dreload:
 
 Recursive reload
 ----------------
 
 The dreload function does a recursive reload of a module: changes made
 to the module since you imported will actually be available without
 having to exit.
 
 
 Verbose and colored exception traceback printouts
 -------------------------------------------------
 
 IPython provides the option to see very detailed exception tracebacks,
 which can be especially useful when debugging large programs. You can
 run any Python file with the %run function to benefit from these
 detailed tracebacks. Furthermore, both normal and verbose tracebacks can
 be colored (if your terminal supports it) which makes them much easier
 to parse visually.
 
 See the magic xmode and colors functions for details (just type %magic).
 
 These features are basically a terminal version of Ka-Ping Yee's cgitb
 module, now part of the standard Python library.
 
 
 .. _input_caching:
 
 Input caching system
 --------------------
 
 IPython offers numbered prompts (In/Out) with input and output caching
 (also referred to as 'input history'). All input is saved and can be 
 retrieved as variables (besides the usual arrow key recall), in 
 addition to the %rep magic command that brings a history entry
 up for editing on the next  command line.
 
 The following GLOBAL variables always exist (so don't overwrite them!):
 _i: stores previous input. _ii: next previous. _iii: next-next previous.
 _ih : a list of all input _ih[n] is the input from line n and this list
 is aliased to the global variable In. If you overwrite In with a
 variable of your own, you can remake the assignment to the internal list
 with a simple 'In=_ih'.
 
 Additionally, global variables named _i<n> are dynamically created (<n>
 being the prompt counter), such that
 _i<n> == _ih[<n>] == In[<n>].
 
 For example, what you typed at prompt 14 is available as _i14, _ih[14]
 and In[14].
 
 This allows you to easily cut and paste multi line interactive prompts
 by printing them out: they print like a clean string, without prompt
 characters. You can also manipulate them like regular variables (they
 are strings), modify or exec them (typing 'exec _i9' will re-execute the
 contents of input prompt 9, 'exec In[9:14]+In[18]' will re-execute lines
 9 through 13 and line 18).
 
 You can also re-execute multiple lines of input easily by using the
 magic %macro function (which automates the process and allows
 re-execution without having to type 'exec' every time). The macro system
 also allows you to re-execute previous lines which include magic
 function calls (which require special processing). Type %macro? or see
 sec. 6.2 <#sec:magic> for more details on the macro system.
 
 A history function %hist allows you to see any part of your input
 history by printing a range of the _i variables.
 
 You can also search ('grep') through your history by typing 
 '%hist -g somestring'. This also searches through the so called *shadow history*,
 which remembers all the commands (apart from multiline code blocks)
 you have ever entered. Handy for searching for svn/bzr URL's, IP adrresses
 etc. You can bring shadow history entries listed by '%hist -g' up for editing 
 (or re-execution by just pressing ENTER) with %rep command. Shadow history 
 entries are not available as _iNUMBER variables, and they are identified by 
 the '0' prefix in %hist -g output. That is, history entry 12 is a normal 
 history entry, but 0231 is a shadow history entry.
 
 Shadow history was added because the readline history is inherently very
 unsafe - if you have multiple IPython sessions open, the last session 
 to close will overwrite the history of previountly closed session. Likewise,
 if a crash occurs, history is never saved, whereas shadow history entries
 are added after entering every command (so a command executed 
 in another IPython session is immediately available in other IPython 
 sessions that are open).
 
 To conserve space, a command can exist in shadow history only once - it doesn't 
 make sense to store a common line like "cd .." a thousand times. The idea is 
 mainly to provide a reliable place where valuable, hard-to-remember commands can 
 always be retrieved, as opposed to providing an exact sequence of commands
 you have entered in actual order.
 
 Because shadow history has all the commands you have ever executed,
 time taken by %hist -g will increase oven time. If it ever starts to take
 too long (or it ends up containing sensitive information like passwords), 
 clear the shadow history by `%clear shadow_nuke`.
 
 Time taken to add entries to shadow history should be negligible, but 
 in any case, if you start noticing performance degradation after using 
 IPython for a long time (or running a script that floods the shadow history!), 
 you can 'compress' the shadow history by executing 
 `%clear shadow_compress`. In practice, this should never be necessary
 in normal use.
 
 .. _output_caching:
 
 Output caching system
 ---------------------
 
 For output that is returned from actions, a system similar to the input
 cache exists but using _ instead of _i. Only actions that produce a
 result (NOT assignments, for example) are cached. If you are familiar
 with Mathematica, IPython's _ variables behave exactly like
 Mathematica's % variables.
 
 The following GLOBAL variables always exist (so don't overwrite them!):
 
     * [_] (a single underscore) : stores previous output, like Python's
       default interpreter.
     * [__] (two underscores): next previous.
     * [___] (three underscores): next-next previous.
 
 Additionally, global variables named _<n> are dynamically created (<n>
 being the prompt counter), such that the result of output <n> is always
 available as _<n> (don't use the angle brackets, just the number, e.g.
 _21).
 
 These global variables are all stored in a global dictionary (not a
 list, since it only has entries for lines which returned a result)
 available under the names _oh and Out (similar to _ih and In). So the
 output from line 12 can be obtained as _12, Out[12] or _oh[12]. If you
 accidentally overwrite the Out variable you can recover it by typing
 'Out=_oh' at the prompt.
 
 This system obviously can potentially put heavy memory demands on your
 system, since it prevents Python's garbage collector from removing any
 previously computed results. You can control how many results are kept
 in memory with the option (at the command line or in your ipythonrc
 file) cache_size. If you set it to 0, the whole system is completely
 disabled and the prompts revert to the classic '>>>' of normal Python.
 
 
 Directory history
 -----------------
 
 Your history of visited directories is kept in the global list _dh, and
 the magic %cd command can be used to go to any entry in that list. The
 %dhist command allows you to view this history. Do ``cd -<TAB`` to
 conventiently view the directory history.
 
 
 Automatic parentheses and quotes
 --------------------------------
 
 These features were adapted from Nathan Gray's LazyPython. They are
 meant to allow less typing for common situations.
 
 
 Automatic parentheses
 ---------------------
 
 Callable objects (i.e. functions, methods, etc) can be invoked like this
 (notice the commas between the arguments)::
 
     >>> callable_ob arg1, arg2, arg3
 
 and the input will be translated to this::
 
     -> callable_ob(arg1, arg2, arg3)
 
 You can force automatic parentheses by using '/' as the first character
 of a line. For example::
 
     >>> /globals # becomes 'globals()'
 
 Note that the '/' MUST be the first character on the line! This won't work::
 
     >>> print /globals # syntax error
 
 In most cases the automatic algorithm should work, so you should rarely
 need to explicitly invoke /. One notable exception is if you are trying
 to call a function with a list of tuples as arguments (the parenthesis
 will confuse IPython)::
 
     In [1]: zip (1,2,3),(4,5,6) # won't work
 
 but this will work::
 
     In [2]: /zip (1,2,3),(4,5,6) 
     ---> zip ((1,2,3),(4,5,6))  
     Out[2]= [(1, 4), (2, 5), (3, 6)]
 
 IPython tells you that it has altered your command line by displaying
 the new command line preceded by ->. e.g.::
 
     In [18]: callable list 
     ----> callable (list)
 
 
 Automatic quoting
 -----------------
 
 You can force automatic quoting of a function's arguments by using ','
 or ';' as the first character of a line. For example::
 
     >>> ,my_function /home/me # becomes my_function("/home/me")
 
 If you use ';' instead, the whole argument is quoted as a single string
 (while ',' splits on whitespace)::
 
     >>> ,my_function a b c # becomes my_function("a","b","c")
 
     >>> ;my_function a b c # becomes my_function("a b c")
 
 Note that the ',' or ';' MUST be the first character on the line! This
 won't work::
 
     >>> x = ,my_function /home/me # syntax error
 
 IPython as your default Python environment
 ==========================================
 
 Python honors the environment variable PYTHONSTARTUP and will execute at
 startup the file referenced by this variable. If you put at the end of
 this file the following two lines of code::
 
     import IPython 
     IPython.Shell.IPShell().mainloop(sys_exit=1)
 
 then IPython will be your working environment anytime you start Python.
 The sys_exit=1 is needed to have IPython issue a call to sys.exit() when
 it finishes, otherwise you'll be back at the normal Python '>>>'
 prompt.
 
 This is probably useful to developers who manage multiple Python
 versions and don't want to have correspondingly multiple IPython
 versions. Note that in this mode, there is no way to pass IPython any
 command-line options, as those are trapped first by Python itself.
 
 .. _Embedding:
 
 Embedding IPython
 =================
 
 It is possible to start an IPython instance inside your own Python
 programs. This allows you to evaluate dynamically the state of your
 code, operate with your variables, analyze them, etc. Note however that
 any changes you make to values while in the shell do not propagate back
 to the running code, so it is safe to modify your values because you
 won't break your code in bizarre ways by doing so.
 
 This feature allows you to easily have a fully functional python
 environment for doing object introspection anywhere in your code with a
 simple function call. In some cases a simple print statement is enough,
 but if you need to do more detailed analysis of a code fragment this
 feature can be very valuable.
 
 It can also be useful in scientific computing situations where it is
 common to need to do some automatic, computationally intensive part and
 then stop to look at data, plots, etc.
 Opening an IPython instance will give you full access to your data and
 functions, and you can resume program execution once you are done with
 the interactive part (perhaps to stop again later, as many times as
 needed).
 
 The following code snippet is the bare minimum you need to include in
 your Python programs for this to work (detailed examples follow later)::
 
     from IPython.Shell import IPShellEmbed
 
     ipshell = IPShellEmbed()
 
     ipshell() # this call anywhere in your program will start IPython
 
 You can run embedded instances even in code which is itself being run at
 the IPython interactive prompt with '%run <filename>'. Since it's easy
 to get lost as to where you are (in your top-level IPython or in your
 embedded one), it's a good idea in such cases to set the in/out prompts
 to something different for the embedded instances. The code examples
 below illustrate this.
 
 You can also have multiple IPython instances in your program and open
 them separately, for example with different options for data
 presentation. If you close and open the same instance multiple times,
 its prompt counters simply continue from each execution to the next.
 
 Please look at the docstrings in the Shell.py module for more details on
 the use of this system.
 
 The following sample file illustrating how to use the embedding
 functionality is provided in the examples directory as example-embed.py.
 It should be fairly self-explanatory::
 
 
     #!/usr/bin/env python
 
     """An example of how to embed an IPython shell into a running program.
 
     Please see the documentation in the IPython.Shell module for more details.
 
     The accompanying file example-embed-short.py has quick code fragments for
     embedding which you can cut and paste in your code once you understand how
     things work.
 
     The code in this file is deliberately extra-verbose, meant for learning."""
 
     # The basics to get you going:
 
     # IPython sets the __IPYTHON__ variable so you can know if you have nested
     # copies running.
 
     # Try running this code both at the command line and from inside IPython (with
     # %run example-embed.py)
     try:
         __IPYTHON__
     except NameError:
         nested = 0
         args = ['']
     else:
         print "Running nested copies of IPython."
         print "The prompts for the nested copy have been modified"
         nested = 1
         # what the embedded instance will see as sys.argv:
         args = ['-pi1','In <\\#>: ','-pi2','   .\\D.: ',
                 '-po','Out<\\#>: ','-nosep']
 
     # First import the embeddable shell class
     from IPython.Shell import IPShellEmbed
 
     # Now create an instance of the embeddable shell. The first argument is a
     # string with options exactly as you would type them if you were starting
     # IPython at the system command line. Any parameters you want to define for
     # configuration can thus be specified here.
     ipshell = IPShellEmbed(args,
                            banner = 'Dropping into IPython',
                            exit_msg = 'Leaving Interpreter, back to program.')
 
     # Make a second instance, you can have as many as you want.
     if nested:
         args[1] = 'In2<\\#>'
     else:
         args = ['-pi1','In2<\\#>: ','-pi2','   .\\D.: ',
                 '-po','Out<\\#>: ','-nosep']
     ipshell2 = IPShellEmbed(args,banner = 'Second IPython instance.')
 
     print '\nHello. This is printed from the main controller program.\n'
 
     # You can then call ipshell() anywhere you need it (with an optional
     # message):
     ipshell('***Called from top level. '
             'Hit Ctrl-D to exit interpreter and continue program.\n'
             'Note that if you use %kill_embedded, you can fully deactivate\n'
             'This embedded instance so it will never turn on again')
 
     print '\nBack in caller program, moving along...\n'
 
     #---------------------------------------------------------------------------
     # More details:
 
     # IPShellEmbed instances don't print the standard system banner and
     # messages. The IPython banner (which actually may contain initialization
     # messages) is available as <instance>.IP.BANNER in case you want it.
 
     # IPShellEmbed instances print the following information everytime they
     # start:
 
     # - A global startup banner.
 
     # - A call-specific header string, which you can use to indicate where in the
     # execution flow the shell is starting.
 
     # They also print an exit message every time they exit.
 
     # Both the startup banner and the exit message default to None, and can be set
     # either at the instance constructor or at any other time with the
     # set_banner() and set_exit_msg() methods.
 
     # The shell instance can be also put in 'dummy' mode globally or on a per-call
     # basis. This gives you fine control for debugging without having to change
     # code all over the place.
 
     # The code below illustrates all this.
 
 
     # This is how the global banner and exit_msg can be reset at any point
     ipshell.set_banner('Entering interpreter - New Banner')
     ipshell.set_exit_msg('Leaving interpreter - New exit_msg')
 
     def foo(m):
         s = 'spam'
         ipshell('***In foo(). Try @whos, or print s or m:')
         print 'foo says m = ',m
 
     def bar(n):
         s = 'eggs'
         ipshell('***In bar(). Try @whos, or print s or n:')
         print 'bar says n = ',n
 
     # Some calls to the above functions which will trigger IPython:
     print 'Main program calling foo("eggs")\n'
     foo('eggs')
 
     # The shell can be put in 'dummy' mode where calls to it silently return. This
     # allows you, for example, to globally turn off debugging for a program with a
     # single call.
     ipshell.set_dummy_mode(1)
     print '\nTrying to call IPython which is now "dummy":'
     ipshell()
     print 'Nothing happened...'
     # The global 'dummy' mode can still be overridden for a single call
     print '\nOverriding dummy mode manually:'
     ipshell(dummy=0)
 
     # Reactivate the IPython shell
     ipshell.set_dummy_mode(0)
 
     print 'You can even have multiple embedded instances:'
     ipshell2()
 
     print '\nMain program calling bar("spam")\n'
     bar('spam')
 
     print 'Main program finished. Bye!'
 
     #********************** End of file <example-embed.py> ***********************
 
 Once you understand how the system functions, you can use the following
 code fragments in your programs which are ready for cut and paste::
 
 
     """Quick code snippets for embedding IPython into other programs.
 
     See example-embed.py for full details, this file has the bare minimum code for
     cut and paste use once you understand how to use the system."""
 
     #---------------------------------------------------------------------------
     # This code loads IPython but modifies a few things if it detects it's running
     # embedded in another IPython session (helps avoid confusion)
 
     try:
         __IPYTHON__
     except NameError:
         argv = ['']
         banner = exit_msg = ''
     else:
         # Command-line options for IPython (a list like sys.argv)
         argv = ['-pi1','In <\\#>:','-pi2','   .\\D.:','-po','Out<\\#>:']
         banner = '*** Nested interpreter ***'
         exit_msg = '*** Back in main IPython ***'
 
     # First import the embeddable shell class
     from IPython.Shell import IPShellEmbed
     # Now create the IPython shell instance. Put ipshell() anywhere in your code
     # where you want it to open.
     ipshell = IPShellEmbed(argv,banner=banner,exit_msg=exit_msg)
 
     #---------------------------------------------------------------------------
     # This code will load an embeddable IPython shell always with no changes for
     # nested embededings.
 
     from IPython.Shell import IPShellEmbed
     ipshell = IPShellEmbed()
     # Now ipshell() will open IPython anywhere in the code.
 
     #---------------------------------------------------------------------------
     # This code loads an embeddable shell only if NOT running inside
     # IPython. Inside IPython, the embeddable shell variable ipshell is just a
     # dummy function.
 
     try:
         __IPYTHON__
     except NameError:
         from IPython.Shell import IPShellEmbed
         ipshell = IPShellEmbed()
         # Now ipshell() will open IPython anywhere in the code
     else:
         # Define a dummy ipshell() so the same code doesn't crash inside an
         # interactive IPython
         def ipshell(): pass
 
     #******************* End of file <example-embed-short.py> ********************
 
 Using the Python debugger (pdb)
 ===============================
 
 Running entire programs via pdb
 -------------------------------
 
 pdb, the Python debugger, is a powerful interactive debugger which
 allows you to step through code, set breakpoints, watch variables,
 etc.  IPython makes it very easy to start any script under the control
 of pdb, regardless of whether you have wrapped it into a 'main()'
 function or not. For this, simply type '%run -d myscript' at an
 IPython prompt. See the %run command's documentation (via '%run?' or
 in Sec. magic_ for more details, including how to control where pdb
 will stop execution first.
 
 For more information on the use of the pdb debugger, read the included
 pdb.doc file (part of the standard Python distribution). On a stock
 Linux system it is located at /usr/lib/python2.3/pdb.doc, but the
 easiest way to read it is by using the help() function of the pdb module
 as follows (in an IPython prompt):
 
 In [1]: import pdb 
 In [2]: pdb.help()
 
 This will load the pdb.doc document in a file viewer for you automatically.
 
 
 Automatic invocation of pdb on exceptions
 -----------------------------------------
 
 IPython, if started with the -pdb option (or if the option is set in
 your rc file) can call the Python pdb debugger every time your code
 triggers an uncaught exception. This feature
 can also be toggled at any time with the %pdb magic command. This can be
 extremely useful in order to find the origin of subtle bugs, because pdb
 opens up at the point in your code which triggered the exception, and
 while your program is at this point 'dead', all the data is still
 available and you can walk up and down the stack frame and understand
 the origin of the problem.
 
 Furthermore, you can use these debugging facilities both with the
 embedded IPython mode and without IPython at all. For an embedded shell
 (see sec. Embedding_), simply call the constructor with
 '-pdb' in the argument string and automatically pdb will be called if an
 uncaught exception is triggered by your code.
 
 For stand-alone use of the feature in your programs which do not use
 IPython at all, put the following lines toward the top of your 'main'
 routine::
 
     import sys
     from IPython.core import ultratb
     sys.excepthook = ultratb.FormattedTB(mode='Verbose',
     color_scheme='Linux', call_pdb=1)
 
 The mode keyword can be either 'Verbose' or 'Plain', giving either very
 detailed or normal tracebacks respectively. The color_scheme keyword can
 be one of 'NoColor', 'Linux' (default) or 'LightBG'. These are the same
 options which can be set in IPython with -colors and -xmode.
 
 This will give any of your programs detailed, colored tracebacks with
 automatic invocation of pdb.
 
 
 Extensions for syntax processing
 ================================
 
 This isn't for the faint of heart, because the potential for breaking
 things is quite high. But it can be a very powerful and useful feature.
 In a nutshell, you can redefine the way IPython processes the user input
 line to accept new, special extensions to the syntax without needing to
 change any of IPython's own code.
 
 In the IPython/extensions directory you will find some examples
 supplied, which we will briefly describe now. These can be used 'as is'
 (and both provide very useful functionality), or you can use them as a
 starting point for writing your own extensions.
 
 
 Pasting of code starting with '>>> ' or '... '
 ----------------------------------------------
 
 In the python tutorial it is common to find code examples which have
 been taken from real python sessions. The problem with those is that all
 the lines begin with either '>>> ' or '... ', which makes it impossible
 to paste them all at once. One must instead do a line by line manual
 copying, carefully removing the leading extraneous characters.
 
 This extension identifies those starting characters and removes them
 from the input automatically, so that one can paste multi-line examples
 directly into IPython, saving a lot of time. Please look at the file
 InterpreterPasteInput.py in the IPython/extensions directory for details
 on how this is done.
 
 IPython comes with a special profile enabling this feature, called
 tutorial. Simply start IPython via 'ipython -p tutorial' and the feature
 will be available. In a normal IPython session you can activate the
 feature by importing the corresponding module with:
 In [1]: import IPython.extensions.InterpreterPasteInput
 
 The following is a 'screenshot' of how things work when this extension
 is on, copying an example from the standard tutorial::
 
     IPython profile: tutorial 
 
     *** Pasting of code with ">>>" or "..." has been enabled. 
 
     In [1]: >>> def fib2(n): # return Fibonacci series up to n 
        ...: ...     """Return a list containing the Fibonacci series up to
     n.""" 
        ...: ...     result = [] 
        ...: ...     a, b = 0, 1 
        ...: ...     while b < n: 
        ...: ...         result.append(b)    # see below 
        ...: ...         a, b = b, a+b 
        ...: ...     return result 
        ...: 
 
     In [2]: fib2(10) 
     Out[2]: [1, 1, 2, 3, 5, 8]
 
 Note that as currently written, this extension does not recognize
 IPython's prompts for pasting. Those are more complicated, since the
 user can change them very easily, they involve numbers and can vary in
 length. One could however extract all the relevant information from the
 IPython instance and build an appropriate regular expression. This is
 left as an exercise for the reader.
 
 
 Input of physical quantities with units
 ---------------------------------------
 
 The module PhysicalQInput allows a simplified form of input for physical
 quantities with units. This file is meant to be used in conjunction with
 the PhysicalQInteractive module (in the same directory) and
 Physics.PhysicalQuantities from Konrad Hinsen's ScientificPython
 (http://dirac.cnrs-orleans.fr/ScientificPython/).
 
 The Physics.PhysicalQuantities module defines PhysicalQuantity objects,
 but these must be declared as instances of a class. For example, to
 define v as a velocity of 3 m/s, normally you would write::
 
     In [1]: v = PhysicalQuantity(3,'m/s')
 
 Using the PhysicalQ_Input extension this can be input instead as: 
 In [1]: v = 3 m/s
 which is much more convenient for interactive use (even though it is
 blatantly invalid Python syntax).
 
 The physics profile supplied with IPython (enabled via 'ipython -p
 physics') uses these extensions, which you can also activate with:
 
 from math import * # math MUST be imported BEFORE PhysicalQInteractive 
 from IPython.extensions.PhysicalQInteractive import * 
 import IPython.extensions.PhysicalQInput
 
 .. _gui_support:
 
 GUI event loop support support
 ==============================
 
 .. versionadded:: 0.11
     The ``%gui`` magic and :mod:`IPython.lib.inputhook`.
 
 IPython has excellent support for working interactively with Graphical User
 Interface (GUI) toolkits, such as wxPython, PyQt4, PyGTK and Tk. This is
 implemented using Python's builtin ``PyOSInputHook`` hook. This implementation
 is extremely robust compared to our previous threaded based version. The
 advantages of this are:
 
 * GUIs can be enabled and disabled dynamically at runtime.
 * The active GUI can be switched dynamically at runtime.
 * In some cases, multiple GUIs can run simultaneously with no problems.
 * There is a developer API in :mod:`IPython.lib.inputhook` for customizing 
   all of these things.
 
 For users, enabling GUI event loop integration is simple.  You simple use the
 ``%gui`` magic as follows::
 
     %gui [-a] [GUINAME]
 
 With no arguments, ``%gui`` removes all GUI support.  Valid ``GUINAME``
 arguments are ``wx``, ``qt4``, ``gtk`` and ``tk``.  The ``-a`` option will
 create and return a running application object for the selected GUI toolkit.
 
 Thus, to use wxPython interactively and create a running :class:`wx.App`
 object, do::
 
     %gui -a wx
 
 For information on IPython's Matplotlib integration (and the ``pylab`` mode)
 see :ref:`this section <matplotlib_support>`.
 
 For developers that want to use IPython's GUI event loop integration in
 the form of a library, these capabilities are exposed in library form
 in the :mod:`IPython.lib.inputhook`.  Interested developers should see the
 module docstrings for more information, but there are a few points that
 should be mentioned here.
 
 First, the ``PyOSInputHook`` approach only works in command line settings 
 where readline is activated.
 
 Second, when using the ``PyOSInputHook`` approach, a GUI application should
 *not* start its event loop. Instead all of this is handled by the
 ``PyOSInputHook``. This means that applications that are meant to be used both
 in IPython and as standalone apps need to have special code to detects how the
 application is being run. We highly recommend using IPython's
 :func:`appstart_` functions for this. Here is a simple example that shows the
 recommended code that should be at the bottom of a wxPython using GUI
 application::
 
   try:
       from IPython import appstart_wx
       appstart_wx(app)
   except ImportError:
       app.MainLoop()
 
 This pattern should be used instead of the simple ``app.MainLoop()`` code
 that a standalone wxPython application would have.
 
 Third, unlike previous versions of IPython, we no longer "hijack" (replace
 them with no-ops) the event loops. This is done to allow applications that
 actually need to run the real event loops to do so. This is often needed to
 process pending events at critical points.
 
 Finally, we also have a number of examples in our source directory
 :file:`docs/examples/lib` that demonstrate these capabilities.
 
 .. _matplotlib_support:
 
 Plotting with matplotlib
 ========================
 
 
 `Matplotlib`_ provides high quality 2D and
 3D plotting for Python. Matplotlib can produce plots on screen using a variety
 of GUI toolkits, including Tk, PyGTK, PyQt4 and wxPython. It also provides a
 number of commands useful for scientific computing, all with a syntax
 compatible with that of the popular Matlab program.
 
 Many IPython users have come to rely on IPython's ``-pylab`` mode which
 automates the integration of Matplotlib with IPython. We are still in the
 process of working with the Matplotlib developers to finalize the new pylab
 API, but for now you can use Matplotlib interactively using the following
 commands::
 
     %gui -a wx
     import matplotlib
     matplotlib.use('wxagg')
     from matplotlib import pylab
     pylab.interactive(True)
 
 All of this will soon be automated as Matplotlib beings to include
 new logic that uses our new GUI support.
 
 .. _interactive_demos:
 
 Interactive demos with IPython
 ==============================
 
 IPython ships with a basic system for running scripts interactively in
 sections, useful when presenting code to audiences. A few tags embedded
 in comments (so that the script remains valid Python code) divide a file
 into separate blocks, and the demo can be run one block at a time, with
 IPython printing (with syntax highlighting) the block before executing
 it, and returning to the interactive prompt after each block. The
 interactive namespace is updated after each block is run with the
 contents of the demo's namespace.
 
 This allows you to show a piece of code, run it and then execute
 interactively commands based on the variables just created. Once you
 want to continue, you simply execute the next block of the demo. The
 following listing shows the markup necessary for dividing a script into
 sections for execution as a demo::
 
 
     """A simple interactive demo to illustrate the use of IPython's Demo class.
 
     Any python script can be run as a demo, but that does little more than showing
     it on-screen, syntax-highlighted in one shot.  If you add a little simple
     markup, you can stop at specified intervals and return to the ipython prompt,
     resuming execution later.
     """
 
     print 'Hello, welcome to an interactive IPython demo.'
     print 'Executing this block should require confirmation before proceeding,'
     print 'unless auto_all has been set to true in the demo object'
 
     # The mark below defines a block boundary, which is a point where IPython will
     # stop execution and return to the interactive prompt.
     # Note that in actual interactive execution, 
     # <demo> --- stop ---
 
     x = 1
     y = 2
 
     # <demo> --- stop ---
 
     # the mark below makes this block as silent
     # <demo> silent
 
     print 'This is a silent block, which gets executed but not printed.'
 
     # <demo> --- stop ---
     # <demo> auto
     print 'This is an automatic block.'
     print 'It is executed without asking for confirmation, but printed.'
     z = x+y
 
     print 'z=',x
 
     # <demo> --- stop ---
     # This is just another normal block.
     print 'z is now:', z
 
     print 'bye!'
 
 In order to run a file as a demo, you must first make a Demo object out
 of it. If the file is named myscript.py, the following code will make a
 demo::
 
     from IPython.demo import Demo
 
     mydemo = Demo('myscript.py')
 
 This creates the mydemo object, whose blocks you run one at a time by
 simply calling the object with no arguments. If you have autocall active
 in IPython (the default), all you need to do is type::
 
     mydemo 
 
 and IPython will call it, executing each block. Demo objects can be
 restarted, you can move forward or back skipping blocks, re-execute the
 last block, etc. Simply use the Tab key on a demo object to see its
 methods, and call '?' on them to see their docstrings for more usage
 details. In addition, the demo module itself contains a comprehensive
 docstring, which you can access via::
 
     from IPython import demo
 
     demo?
 
 Limitations: It is important to note that these demos are limited to
 fairly simple uses. In particular, you can not put division marks in
 indented code (loops, if statements, function definitions, etc.)
 Supporting something like this would basically require tracking the
 internal execution state of the Python interpreter, so only top-level
 divisions are allowed. If you want to be able to open an IPython
 instance at an arbitrary point in a program, you can use IPython's
 embedding facilities, described in detail in Sec. 9
 
 .. [Matplotlib] Matplotlib. http://matplotlib.sourceforge.net
 

docs/source/interactive/shell.txt

@@ -1,293 +1,293 b''
 .. _ipython_as_shell:
 
 =========================
 IPython as a system shell
 =========================
 
 .. warning::
 
     As of the 0.11 version of IPython, some of the features and APIs
     described in this section have been deprecated or are broken.  Our plan
     is to continue to support these features, but they need to be updated
     to take advantage of recent API changes.  Furthermore, this section
     of the documentation need to be updated to reflect all of these changes.
 
 Overview
 ========
 
 The 'sh' profile optimizes IPython for system shell usage. Apart from
 certain job control functionality that is present in unix (ctrl+z does
 "suspend"), the sh profile should provide you with most of the
 functionality you use daily in system shell, and more. Invoke IPython
 in 'sh' profile by doing 'ipython -p sh', or (in win32) by launching
 the "pysh" shortcut in start menu.
 
 If you want to use the features of sh profile as your defaults (which
 might be a good idea if you use other profiles a lot of the time but
 still want the convenience of sh profile), add ``import ipy_profile_sh``
-to your ~/.ipython/ipy_user_conf.py.
+to your $IPYTHON_DIR/ipy_user_conf.py.
 
 The 'sh' profile is different from the default profile in that:
 
  * Prompt shows the current directory
  * Spacing between prompts and input is more compact (no padding with
    empty lines). The startup banner is more compact as well.
  * System commands are directly available (in alias table) without
    requesting %rehashx - however, if you install new programs along
    your PATH, you might want to run %rehashx to update the persistent
    alias table
  * Macros are stored in raw format by default. That is, instead of
    '_ip.system("cat foo"), the macro will contain text 'cat foo')
  * Autocall is in full mode
  * Calling "up" does "cd .."
 
 The 'sh' profile is different from the now-obsolete (and unavailable)
 'pysh' profile in that:
 
  * '$$var = command' and '$var = command' syntax is not supported
  * anymore. Use 'var = !command' instead (incidentally, this is
  * available in all IPython profiles). Note that !!command *will*
  * work.
 
 Aliases
 =======
 
 All of your $PATH has been loaded as IPython aliases, so you should be
 able to type any normal system command and have it executed. See
 %alias?  and %unalias? for details on the alias facilities. See also
 %rehashx? for details on the mechanism used to load $PATH.
 
 
 Directory management
 ====================
 
 Since each command passed by ipython to the underlying system is executed
 in a subshell which exits immediately, you can NOT use !cd to navigate
 the filesystem.
 
 IPython provides its own builtin '%cd' magic command to move in the
 filesystem (the % is not required with automagic on). It also maintains
 a list of visited directories (use %dhist to see it) and allows direct
 switching to any of them. Type 'cd?' for more details.
 
 %pushd, %popd and %dirs are provided for directory stack handling.
 
 
 Enabled extensions
 ==================
 
 Some extensions, listed below, are enabled as default in this profile.
 
 envpersist
 ----------
 
 %env can be used to "remember" environment variable manipulations. Examples::
 
 	%env - Show all environment variables
 	%env VISUAL=jed  - set VISUAL to jed
 	%env PATH+=;/foo - append ;foo to PATH
 	%env PATH+=;/bar - also append ;bar to PATH
 	%env PATH-=/wbin; - prepend /wbin; to PATH
 	%env -d VISUAL   - forget VISUAL persistent val
 	%env -p          - print all persistent env modifications
 
 ipy_which
 ---------
 
 %which magic command. Like 'which' in unix, but knows about ipython aliases.
 
 Example::
 
  [C:/ipython]|14> %which st
  st -> start .
  [C:/ipython]|15> %which d
  d -> dir /w /og /on
  [C:/ipython]|16> %which cp
  cp -> cp
    == c:\bin\cp.exe
  c:\bin\cp.exe
 
 ipy_app_completers
 ------------------
 
 Custom tab completers for some apps like svn, hg, bzr, apt-get. Try 'apt-get install <TAB>' in debian/ubuntu.
 
 ipy_rehashdir
 -------------
 
 Allows you to add system command aliases for commands that are not along your path. Let's say that you just installed Putty and want to be able to invoke it without adding it to path, you can create the alias for it with rehashdir::
 
  [~]|22> cd c:/opt/PuTTY/
  [c:opt/PuTTY]|23> rehashdir .
               <23> ['pageant', 'plink', 'pscp', 'psftp', 'putty', 'puttygen', 'unins000']
 
 Now, you can execute any of those commams directly::
 
  [c:opt/PuTTY]|24> cd
  [~]|25> putty
 
 (the putty window opens).
 
 If you want to store the alias so that it will always be available, do '%store putty'. If you want to %store all these aliases persistently, just do it in a for loop::
 
  [~]|27> for a in _23:
     |..>     %store $a
     |..>
     |..>
  Alias stored: pageant (0, 'c:\\opt\\PuTTY\\pageant.exe')
  Alias stored: plink (0, 'c:\\opt\\PuTTY\\plink.exe')
  Alias stored: pscp (0, 'c:\\opt\\PuTTY\\pscp.exe')
  Alias stored: psftp (0, 'c:\\opt\\PuTTY\\psftp.exe')
  ...
 
 mglob
 -----
 
 Provide the magic function %mglob, which makes it easier (than the 'find' command) to collect (possibly recursive) file lists. Examples::
 
  [c:/ipython]|9> mglob *.py
  [c:/ipython]|10> mglob *.py rec:*.txt
  [c:/ipython]|19> workfiles = %mglob !.svn/ !.hg/ !*_Data/ !*.bak rec:.
 
 Note that the first 2 calls will put the file list in result history (_, _9, _10), and the last one will assign it to 'workfiles'.
 
 
 Prompt customization
 ====================
 
 The sh profile uses the following prompt configurations::
 
     o.prompt_in1= r'\C_LightBlue[\C_LightCyan\Y2\C_LightBlue]\C_Green|\#>'
     o.prompt_in2= r'\C_Green|\C_LightGreen\D\C_Green>'
 
 You can change the prompt configuration to your liking by editing
 ipy_user_conf.py.  
 
 String lists
 ============
 
 String lists (IPython.utils.text.SList) are handy way to process output
 from system commands. They are produced by ``var = !cmd`` syntax.
 
 First, we acquire the output of 'ls -l'::
 
     [Q:doc/examples]|2> lines = !ls -l
      ==
     ['total 23',
      '-rw-rw-rw- 1 ville None 1163 Sep 30  2006 example-demo.py',
      '-rw-rw-rw- 1 ville None 1927 Sep 30  2006 example-embed-short.py',
      '-rwxrwxrwx 1 ville None 4606 Sep  1 17:15 example-embed.py',
      '-rwxrwxrwx 1 ville None 1017 Sep 30  2006 example-gnuplot.py',
      '-rwxrwxrwx 1 ville None  339 Jun 11 18:01 extension.py',
      '-rwxrwxrwx 1 ville None  113 Dec 20  2006 seteditor.py',
      '-rwxrwxrwx 1 ville None  245 Dec 12  2006 seteditor.pyc']
 
 Now, let's take a look at the contents of 'lines' (the first number is
 the list element number)::
 
     [Q:doc/examples]|3> lines
                     <3> SList (.p, .n, .l, .s, .grep(), .fields() available). Value:
 
     0: total 23
     1: -rw-rw-rw- 1 ville None 1163 Sep 30  2006 example-demo.py
     2: -rw-rw-rw- 1 ville None 1927 Sep 30  2006 example-embed-short.py
     3: -rwxrwxrwx 1 ville None 4606 Sep  1 17:15 example-embed.py
     4: -rwxrwxrwx 1 ville None 1017 Sep 30  2006 example-gnuplot.py
     5: -rwxrwxrwx 1 ville None  339 Jun 11 18:01 extension.py
     6: -rwxrwxrwx 1 ville None  113 Dec 20  2006 seteditor.py
     7: -rwxrwxrwx 1 ville None  245 Dec 12  2006 seteditor.pyc
 
 Now, let's filter out the 'embed' lines::
 
     [Q:doc/examples]|4> l2 = lines.grep('embed',prune=1)
     [Q:doc/examples]|5> l2
                     <5> SList (.p, .n, .l, .s, .grep(), .fields() available). Value:
 
     0: total 23
     1: -rw-rw-rw- 1 ville None 1163 Sep 30  2006 example-demo.py
     2: -rwxrwxrwx 1 ville None 1017 Sep 30  2006 example-gnuplot.py
     3: -rwxrwxrwx 1 ville None  339 Jun 11 18:01 extension.py
     4: -rwxrwxrwx 1 ville None  113 Dec 20  2006 seteditor.py
     5: -rwxrwxrwx 1 ville None  245 Dec 12  2006 seteditor.pyc
 
 Now, we want strings having just file names and permissions::
 
     [Q:doc/examples]|6> l2.fields(8,0)
                     <6> SList (.p, .n, .l, .s, .grep(), .fields() available). Value:
 
     0: total
     1: example-demo.py -rw-rw-rw-
     2: example-gnuplot.py -rwxrwxrwx
     3: extension.py -rwxrwxrwx
     4: seteditor.py -rwxrwxrwx
     5: seteditor.pyc -rwxrwxrwx
 
 Note how the line with 'total' does not raise IndexError.
 
 If you want to split these (yielding lists), call fields() without
 arguments::
 
     [Q:doc/examples]|7> _.fields()
                     <7>
     [['total'],
      ['example-demo.py', '-rw-rw-rw-'],
      ['example-gnuplot.py', '-rwxrwxrwx'],
      ['extension.py', '-rwxrwxrwx'],
      ['seteditor.py', '-rwxrwxrwx'],
      ['seteditor.pyc', '-rwxrwxrwx']]
 
 If you want to pass these separated with spaces to a command (typical
 for lists if files), use the .s property::
 
 
     [Q:doc/examples]|13> files = l2.fields(8).s
     [Q:doc/examples]|14> files
                     <14> 'example-demo.py example-gnuplot.py extension.py seteditor.py seteditor.pyc'
     [Q:doc/examples]|15> ls $files
     example-demo.py  example-gnuplot.py  extension.py  seteditor.py  seteditor.pyc
 
 SLists are inherited from normal python lists, so every list method is
 available::
 
     [Q:doc/examples]|21> lines.append('hey')
 
 
 Real world example: remove all files outside version control
 ============================================================
 
 First, capture output of "hg status"::
 
     [Q:/ipython]|28> out = !hg status
      ==
     ['M IPython\\extensions\\ipy_kitcfg.py',
      'M IPython\\extensions\\ipy_rehashdir.py',
     ...
      '? build\\lib\\IPython\\Debugger.py',
      '? build\\lib\\IPython\\extensions\\InterpreterExec.py',
      '? build\\lib\\IPython\\extensions\\InterpreterPasteInput.py',
     ...
 
 (lines starting with ? are not under version control).
 
 ::
 
     [Q:/ipython]|35> junk = out.grep(r'^\?').fields(1)
     [Q:/ipython]|36> junk
                 <36> SList (.p, .n, .l, .s, .grep(), .fields() availab
     ...
     10: build\bdist.win32\winexe\temp\_ctypes.py
     11: build\bdist.win32\winexe\temp\_hashlib.py
     12: build\bdist.win32\winexe\temp\_socket.py
 
 Now we can just remove these files by doing 'rm $junk.s'. 
 
 The .s, .n, .p properties
 =========================
 
 The '.s' property returns one string where lines are separated by
 single space (for convenient passing to system commands). The '.n'
 property return one string where the lines are separated by '\n'
 (i.e. the original output of the function). If the items in string
 list are file names, '.p' can be used to get a list of "path" objects
 for convenient file manipulation.
 

docs/source/parallel/parallel_intro.txt

@@ -1,237 +1,237 b''
 .. _ip1par:
 
 ============================
 Overview and getting started
 ============================
 
 Introduction
 ============
 
 This section gives an overview of IPython's sophisticated and powerful
 architecture for parallel and distributed computing. This architecture
 abstracts out parallelism in a very general way, which enables IPython to
 support many different styles of parallelism including:
 
 * Single program, multiple data (SPMD) parallelism.
 * Multiple program, multiple data (MPMD) parallelism.
 * Message passing using MPI.
 * Task farming.
 * Data parallel.
 * Combinations of these approaches.
 * Custom user defined approaches.
 
 Most importantly, IPython enables all types of parallel applications to
 be developed, executed, debugged and monitored *interactively*. Hence,
 the ``I`` in IPython.  The following are some example usage cases for IPython:
 
 * Quickly parallelize algorithms that are embarrassingly parallel
   using a number of simple approaches.  Many simple things can be
   parallelized interactively in one or two lines of code.
 
 * Steer traditional MPI applications on a supercomputer from an
   IPython session on your laptop.
 
 * Analyze and visualize large datasets (that could be remote and/or
   distributed) interactively using IPython and tools like
   matplotlib/TVTK.
 
 * Develop, test and debug new parallel algorithms 
   (that may use MPI) interactively.
 
 * Tie together multiple MPI jobs running on different systems into
   one giant distributed and parallel system.
 
 * Start a parallel job on your cluster and then have a remote 	
   collaborator connect to it and pull back data into their 
   local IPython session for plotting and analysis.
 
 * Run a set of tasks on a set of CPUs using dynamic load balancing.
 
 Architecture overview
 =====================
 
 The IPython architecture consists of three components:
 
 * The IPython engine.
 * The IPython controller.
 * Various controller clients.
 
 These components live in the :mod:`IPython.kernel` package and are
 installed with IPython.  They do, however, have additional dependencies
 that must be installed.  For more information, see our
 :ref:`installation documentation <install_index>`.
 
 IPython engine
 ---------------
 
 The IPython engine is a Python instance that takes Python commands over a
 network connection. Eventually, the IPython engine will be a full IPython
 interpreter, but for now, it is a regular Python interpreter. The engine
 can also handle incoming and outgoing Python objects sent over a network
 connection.  When multiple engines are started, parallel and distributed
 computing becomes possible. An important feature of an IPython engine is
 that it blocks while user code is being executed. Read on for how the
 IPython controller solves this problem to expose a clean asynchronous API
 to the user.
 
 IPython controller
 ------------------
 
 The IPython controller provides an interface for working with a set of
 engines. At an general level, the controller is a process to which
 IPython engines can connect. For each connected engine, the controller
 manages a queue. All actions that can be performed on the engine go
 through this queue. While the engines themselves block when user code is
 run, the controller hides that from the user to provide a fully
 asynchronous interface to a set of engines. 
 
 .. note::
 
     Because the controller listens on a network port for engines to 
     connect to it, it must be started *before* any engines are started.  
 
 The controller also provides a single point of contact for users who wish to
 utilize the engines connected to the controller. There are different ways of
 working with a controller. In IPython these ways correspond to different
 interfaces that the controller is adapted to. Currently we have two default
 interfaces to the controller:
 
 * The MultiEngine interface, which provides the simplest possible way of
   working with engines interactively.
 * The Task interface, which presents the engines as a load balanced
   task farming system.
 
 Advanced users can easily add new custom interfaces to enable other
 styles of parallelism. 
 
 .. note:: 
 
 	A single controller and set of engines can be accessed 
 	through multiple interfaces simultaneously.  This opens the
 	door for lots of interesting things.  
 
 Controller clients
 ------------------
 
 For each controller interface, there is a corresponding client. These
 clients allow users to interact with a set of engines through the
 interface.  Here are the two default clients:
 
 * The :class:`MultiEngineClient` class.
 * The :class:`TaskClient` class.
 
 Security
 --------
 
 By default (as long as `pyOpenSSL` is installed) all network connections
 between the controller and engines and the controller and clients are secure.
 What does this mean? First of all, all of the connections will be encrypted
 using SSL. Second, the connections are authenticated. We handle authentication
 in a capability based security model [Capability]_. In this model, a
 "capability (known in some systems as a key) is a communicable, unforgeable
 token of authority". Put simply, a capability is like a key to your house. If
 you have the key to your house, you can get in. If not, you can't.
 
 In our architecture, the controller is the only process that listens on
 network ports, and is thus responsible to creating these keys. In IPython,
 these keys are known as Foolscap URLs, or FURLs, because of the underlying
 network protocol we are using. As a user, you don't need to know anything
 about the details of these FURLs, other than that when the controller starts,
 it saves a set of FURLs to files named :file:`something.furl`. The default
-location of these files is the :file:`~./ipython/security` directory.
+location of these files is the :file:`$IPYTHON_DIR/cluster_<profile>/security` directory.
 
 To connect and authenticate to the controller an engine or client simply needs
 to present an appropriate FURL (that was originally created by the controller)
 to the controller. Thus, the FURL files need to be copied to a location where
 the clients and engines can find them. Typically, this is the
-:file:`~./ipython/security` directory on the host where the client/engine is
+:file:`$IPYTHON_DIR/cluster_<profile>/security` directory on the host where the client/engine is
 running (which could be a different host than the controller). Once the FURL
 files are copied over, everything should work fine.
 
 Currently, there are three FURL files that the controller creates:
 
 ipcontroller-engine.furl
     This FURL file is the key that gives an engine the ability to connect
     to a controller.
 
 ipcontroller-tc.furl
     This FURL file is the key that a :class:`TaskClient` must use to
     connect to the task interface of a controller.
     
 ipcontroller-mec.furl
     This FURL file is the key that a :class:`MultiEngineClient` must use
     to connect to the multiengine interface of a controller.
 
 More details of how these FURL files are used are given below.
 
 A detailed description of the security model and its implementation in IPython
 can be found :ref:`here <parallelsecurity>`.
 
 Getting Started
 ===============
 
 To use IPython for parallel computing, you need to start one instance of the
 controller and one or more instances of the engine. Initially, it is best to
 simply start a controller and engines on a single host using the
 :command:`ipcluster` command. To start a controller and 4 engines on your
 localhost, just do::
 
     $ ipcluster local -n 4
 
 More details about starting the IPython controller and engines can be found
 :ref:`here <parallel_process>`
 
 Once you have started the IPython controller and one or more engines, you
 are ready to use the engines to do something useful. To make sure
 everything is working correctly, try the following commands:
 
 .. sourcecode:: ipython
 
 	In [1]: from IPython.kernel import client
 	
 	In [2]: mec = client.MultiEngineClient()
 	
 	In [4]: mec.get_ids()
 	Out[4]: [0, 1, 2, 3]
 	
 	In [5]: mec.execute('print "Hello World"')
 	Out[5]: 
 	<Results List>
 	[0] In [1]: print "Hello World"
 	[0] Out[1]: Hello World
 
 	[1] In [1]: print "Hello World"
 	[1] Out[1]: Hello World
 
 	[2] In [1]: print "Hello World"
 	[2] Out[1]: Hello World
 
 	[3] In [1]: print "Hello World"
 	[3] Out[1]: Hello World
 
 Remember, a client also needs to present a FURL file to the controller. How
 does this happen? When a multiengine client is created with no arguments, the
 client tries to find the corresponding FURL file in the local
-:file:`~./ipython/security` directory. If it finds it, you are set. If you
+:file:`$IPYTHON_DIR/cluster_<profile>/security` directory. If it finds it, you are set. If you
 have put the FURL file in a different location or it has a different name,
 create the client like this::
 
     mec = client.MultiEngineClient('/path/to/my/ipcontroller-mec.furl')
 
 Same thing hold true of creating a task client::
 
     tc = client.TaskClient('/path/to/my/ipcontroller-tc.furl')
 
 You are now ready to learn more about the :ref:`MultiEngine
 <parallelmultiengine>` and :ref:`Task <paralleltask>` interfaces to the
 controller.
 
 .. note:: 
 
     Don't forget that the engine, multiengine client and task client all have
     *different* furl files. You must move *each* of these around to an
     appropriate location so that the engines and clients can use them to
     connect to the controller.
 
 .. [Capability] Capability-based security, http://en.wikipedia.org/wiki/Capability-based_security
 

docs/source/parallel/parallel_multiengine.txt

@@ -1,835 +1,835 b''
 .. _parallelmultiengine:
 
 ===============================
 IPython's multiengine interface
 ===============================
 
 The multiengine interface represents one possible way of working with a set of
 IPython engines. The basic idea behind the multiengine interface is that the
 capabilities of each engine are directly and explicitly exposed to the user.
 Thus, in the multiengine interface, each engine is given an id that is used to
 identify the engine and give it work to do. This interface is very intuitive
 and is designed with interactive usage in mind, and is thus the best place for
 new users of IPython to begin.
 
 Starting the IPython controller and engines
 ===========================================
 
 To follow along with this tutorial, you will need to start the IPython
 controller and four IPython engines. The simplest way of doing this is to use
 the :command:`ipcluster` command::
 
 	$ ipcluster local -n 4
 	
 For more detailed information about starting the controller and engines, see
 our :ref:`introduction <ip1par>` to using IPython for parallel computing.
 
 Creating a ``MultiEngineClient`` instance
 =========================================
 
 The first step is to import the IPython :mod:`IPython.kernel.client` module
 and then create a :class:`MultiEngineClient` instance:
 
 .. sourcecode:: ipython
 
 	In [1]: from IPython.kernel import client
 	
 	In [2]: mec = client.MultiEngineClient()
 
 This form assumes that the :file:`ipcontroller-mec.furl` is in the
-:file:`~./ipython/security` directory on the client's host. If not, the
+:file:`$IPYTHON_DIR/cluster_<profile>/security` directory on the client's host. If not, the
 location of the FURL file must be given as an argument to the
 constructor:
 
 .. sourcecode:: ipython
 
     In [2]: mec = client.MultiEngineClient('/path/to/my/ipcontroller-mec.furl')
 
 To make sure there are engines connected to the controller, use can get a list
 of engine ids:
 
 .. sourcecode:: ipython
 
 	In [3]: mec.get_ids()
 	Out[3]: [0, 1, 2, 3]
 
 Here we see that there are four engines ready to do work for us.
 
 Quick and easy parallelism
 ==========================
 
 In many cases, you simply want to apply a Python function to a sequence of
 objects, but *in parallel*. The multiengine interface provides two simple ways
 of accomplishing this: a parallel version of :func:`map` and ``@parallel``
 function decorator.
 
 Parallel map
 ------------
 
 Python's builtin :func:`map` functions allows a function to be applied to a
 sequence element-by-element. This type of code is typically trivial to
 parallelize. In fact, the multiengine interface in IPython already has a
 parallel version of :meth:`map` that works just like its serial counterpart:
 
 .. sourcecode:: ipython
 
 	In [63]: serial_result = map(lambda x:x**10, range(32))
 
 	In [64]: parallel_result = mec.map(lambda x:x**10, range(32))
 
 	In [65]: serial_result==parallel_result
 	Out[65]: True
 
 .. note::
 
     The multiengine interface version of :meth:`map` does not do any load
     balancing.  For a load balanced version, see the task interface.
 
 .. seealso::
 
     The :meth:`map` method has a number of options that can be controlled by
     the :meth:`mapper` method.  See its docstring for more information.
 
 Parallel function decorator
 ---------------------------
 
 Parallel functions are just like normal function, but they can be called on
 sequences and *in parallel*. The multiengine interface provides a decorator
 that turns any Python function into a parallel function:
 
 .. sourcecode:: ipython
 
     In [10]: @mec.parallel()
        ....: def f(x):
        ....:     return 10.0*x**4
        ....: 
 
     In [11]: f(range(32))    # this is done in parallel
     Out[11]: 
     [0.0,10.0,160.0,...]
 
 See the docstring for the :meth:`parallel` decorator for options.
 
 Running Python commands
 =======================
 
 The most basic type of operation that can be performed on the engines is to
 execute Python code. Executing Python code can be done in blocking or
 non-blocking mode (blocking is default) using the :meth:`execute` method.
 
 Blocking execution
 ------------------
 
 In blocking mode, the :class:`MultiEngineClient` object (called ``mec`` in
 these examples) submits the command to the controller, which places the
 command in the engines' queues for execution. The :meth:`execute` call then
 blocks until the engines are done executing the command:
 
 .. sourcecode:: ipython
 
 	# The default is to run on all engines
 	In [4]: mec.execute('a=5')
 	Out[4]: 
 	<Results List>
 	[0] In [1]: a=5
 	[1] In [1]: a=5
 	[2] In [1]: a=5
 	[3] In [1]: a=5
 
 	In [5]: mec.execute('b=10')
 	Out[5]: 
 	<Results List>
 	[0] In [2]: b=10
 	[1] In [2]: b=10
 	[2] In [2]: b=10
 	[3] In [2]: b=10
 
 Python commands can be executed on specific engines by calling execute using
 the ``targets`` keyword argument:
 
 .. sourcecode:: ipython
 
 	In [6]: mec.execute('c=a+b',targets=[0,2])
 	Out[6]: 
 	<Results List>
 	[0] In [3]: c=a+b
 	[2] In [3]: c=a+b
 
 
 	In [7]: mec.execute('c=a-b',targets=[1,3])
 	Out[7]: 
 	<Results List>
 	[1] In [3]: c=a-b
 	[3] In [3]: c=a-b
 
 
 	In [8]: mec.execute('print c')
 	Out[8]: 
 	<Results List>
 	[0] In [4]: print c
 	[0] Out[4]: 15
 
 	[1] In [4]: print c
 	[1] Out[4]: -5
 
 	[2] In [4]: print c
 	[2] Out[4]: 15
 
 	[3] In [4]: print c
 	[3] Out[4]: -5
 
 This example also shows one of the most important things about the IPython
 engines: they have a persistent user namespaces. The :meth:`execute` method
 returns a Python ``dict`` that contains useful information:
 
 .. sourcecode:: ipython
 
 	In [9]: result_dict = mec.execute('d=10; print d')
 
 	In [10]: for r in result_dict:
 	   ....:     print r
 	   ....:     
 	   ....:     
 	{'input': {'translated': 'd=10; print d', 'raw': 'd=10; print d'}, 'number': 5, 'id': 0, 'stdout': '10\n'}
 	{'input': {'translated': 'd=10; print d', 'raw': 'd=10; print d'}, 'number': 5, 'id': 1, 'stdout': '10\n'}
 	{'input': {'translated': 'd=10; print d', 'raw': 'd=10; print d'}, 'number': 5, 'id': 2, 'stdout': '10\n'}
 	{'input': {'translated': 'd=10; print d', 'raw': 'd=10; print d'}, 'number': 5, 'id': 3, 'stdout': '10\n'}
 
 Non-blocking execution
 ----------------------
 
 In non-blocking mode, :meth:`execute` submits the command to be executed and
 then returns a :class:`PendingResult` object immediately. The
 :class:`PendingResult` object gives you a way of getting a result at a later
 time through its :meth:`get_result` method or :attr:`r` attribute. This allows
 you to quickly submit long running commands without blocking your local
 Python/IPython session:
 
 .. sourcecode:: ipython
 
 	# In blocking mode
 	In [6]: mec.execute('import time')
 	Out[6]:
 	<Results List>
 	[0] In [1]: import time
 	[1] In [1]: import time
 	[2] In [1]: import time
 	[3] In [1]: import time
 
 	# In non-blocking mode
 	In [7]: pr = mec.execute('time.sleep(10)',block=False)
 
 	# Now block for the result
 	In [8]: pr.get_result()
 	Out[8]:
 	<Results List>
 	[0] In [2]: time.sleep(10)
 	[1] In [2]: time.sleep(10)
 	[2] In [2]: time.sleep(10)
 	[3] In [2]: time.sleep(10)
 
 	# Again in non-blocking mode
 	In [9]: pr = mec.execute('time.sleep(10)',block=False)
 
 	# Poll to see if the result is ready
 	In [10]: pr.get_result(block=False)
 
 	# A shorthand for get_result(block=True)
 	In [11]: pr.r
 	Out[11]:
 	<Results List>
 	[0] In [3]: time.sleep(10)
 	[1] In [3]: time.sleep(10)
 	[2] In [3]: time.sleep(10)
 	[3] In [3]: time.sleep(10)
 
 Often, it is desirable to wait until a set of :class:`PendingResult` objects
 are done. For this, there is a the method :meth:`barrier`. This method takes a
 tuple of :class:`PendingResult` objects and blocks until all of the associated
 results are ready:
 
 .. sourcecode:: ipython
 
 	In [72]: mec.block=False
 
 	# A trivial list of PendingResults objects
 	In [73]: pr_list = [mec.execute('time.sleep(3)') for i in range(10)]
 
 	# Wait until all of them are done
 	In [74]: mec.barrier(pr_list)
 
 	# Then, their results are ready using get_result or the r attribute
 	In [75]: pr_list[0].r
 	Out[75]: 
 	<Results List>
 	[0] In [20]: time.sleep(3)
 	[1] In [19]: time.sleep(3)
 	[2] In [20]: time.sleep(3)
 	[3] In [19]: time.sleep(3)
 
 
 The ``block`` and ``targets`` keyword arguments and attributes
 --------------------------------------------------------------
 
 Most methods in the multiengine interface (like :meth:`execute`) accept
 ``block`` and ``targets`` as keyword arguments. As we have seen above, these
 keyword arguments control the blocking mode and which engines the command is
 applied to. The :class:`MultiEngineClient` class also has :attr:`block` and
 :attr:`targets` attributes that control the default behavior when the keyword
 arguments are not provided. Thus the following logic is used for :attr:`block`
 and :attr:`targets`:
 
 * If no keyword argument is provided, the instance attributes are used.
 * Keyword argument, if provided override the instance attributes.
 
 The following examples demonstrate how to use the instance attributes:
 
 .. sourcecode:: ipython
 
 	In [16]: mec.targets = [0,2]
 
 	In [17]: mec.block = False
 
 	In [18]: pr = mec.execute('a=5')
 
 	In [19]: pr.r
 	Out[19]: 
 	<Results List>
 	[0] In [6]: a=5
 	[2] In [6]: a=5
 
 	# Note targets='all' means all engines
 	In [20]: mec.targets = 'all'
 
 	In [21]: mec.block = True
 
 	In [22]: mec.execute('b=10; print b')
 	Out[22]: 
 	<Results List>
 	[0] In [7]: b=10; print b
 	[0] Out[7]: 10
 
 	[1] In [6]: b=10; print b
 	[1] Out[6]: 10
 
 	[2] In [7]: b=10; print b
 	[2] Out[7]: 10
 
 	[3] In [6]: b=10; print b
 	[3] Out[6]: 10
 
 The :attr:`block` and :attr:`targets` instance attributes also determine the
 behavior of the parallel magic commands.
 
 
 Parallel magic commands
 -----------------------
 
 We provide a few IPython magic commands (``%px``, ``%autopx`` and ``%result``)
 that make it more pleasant to execute Python commands on the engines
 interactively. These are simply shortcuts to :meth:`execute` and
 :meth:`get_result`. The ``%px`` magic executes a single Python command on the
 engines specified by the :attr:`targets` attribute of the
 :class:`MultiEngineClient` instance (by default this is ``'all'``):
 
 .. sourcecode:: ipython
 
 	# Make this MultiEngineClient active for parallel magic commands
 	In [23]: mec.activate()
 
 	In [24]: mec.block=True
 
 	In [25]: import numpy
 
 	In [26]: %px import numpy
 	Executing command on Controller
 	Out[26]:
 	<Results List>
 	[0] In [8]: import numpy
 	[1] In [7]: import numpy
 	[2] In [8]: import numpy
 	[3] In [7]: import numpy
 
 
 	In [27]: %px a = numpy.random.rand(2,2)
 	Executing command on Controller
 	Out[27]:
 	<Results List>
 	[0] In [9]: a = numpy.random.rand(2,2)
 	[1] In [8]: a = numpy.random.rand(2,2)
 	[2] In [9]: a = numpy.random.rand(2,2)
 	[3] In [8]: a = numpy.random.rand(2,2)
 
 
 	In [28]: %px print numpy.linalg.eigvals(a)
 	Executing command on Controller
 	Out[28]:
 	<Results List>
 	[0] In [10]: print numpy.linalg.eigvals(a)
 	[0] Out[10]: [ 1.28167017  0.14197338]
 
 	[1] In [9]: print numpy.linalg.eigvals(a)
 	[1] Out[9]: [-0.14093616  1.27877273]
 
 	[2] In [10]: print numpy.linalg.eigvals(a)
 	[2] Out[10]: [-0.37023573  1.06779409]
 
 	[3] In [9]: print numpy.linalg.eigvals(a)
 	[3] Out[9]: [ 0.83664764 -0.25602658]
 
 The ``%result`` magic gets and prints the stdin/stdout/stderr of the last
 command executed on each engine. It is simply a shortcut to the
 :meth:`get_result` method:
 
 .. sourcecode:: ipython
 
 	In [29]: %result
 	Out[29]:
 	<Results List>
 	[0] In [10]: print numpy.linalg.eigvals(a)
 	[0] Out[10]: [ 1.28167017  0.14197338]
 
 	[1] In [9]: print numpy.linalg.eigvals(a)
 	[1] Out[9]: [-0.14093616  1.27877273]
 
 	[2] In [10]: print numpy.linalg.eigvals(a)
 	[2] Out[10]: [-0.37023573  1.06779409]
 
 	[3] In [9]: print numpy.linalg.eigvals(a)
 	[3] Out[9]: [ 0.83664764 -0.25602658]
 
 The ``%autopx`` magic switches to a mode where everything you type is executed
 on the engines given by the :attr:`targets` attribute:
 
 .. sourcecode:: ipython
 
 	In [30]: mec.block=False
 
 	In [31]: %autopx
 	Auto Parallel Enabled
 	Type %autopx to disable
 
 	In [32]: max_evals = []
 	<IPython.kernel.multiengineclient.PendingResult object at 0x17b8a70>
 
 	In [33]: for i in range(100):
 	   ....:     a = numpy.random.rand(10,10)
 	   ....:     a = a+a.transpose()
 	   ....:     evals = numpy.linalg.eigvals(a)
 	   ....:     max_evals.append(evals[0].real)
 	   ....:
 	   ....:
 	<IPython.kernel.multiengineclient.PendingResult object at 0x17af8f0>
 
 	In [34]: %autopx
 	Auto Parallel Disabled
 
 	In [35]: mec.block=True
 
 	In [36]: px print "Average max eigenvalue is: ", sum(max_evals)/len(max_evals)
 	Executing command on Controller
 	Out[36]:
 	<Results List>
 	[0] In [13]: print "Average max eigenvalue is: ", sum(max_evals)/len(max_evals)
 	[0] Out[13]: Average max eigenvalue is:  10.1387247332
 
 	[1] In [12]: print "Average max eigenvalue is: ", sum(max_evals)/len(max_evals)
 	[1] Out[12]: Average max eigenvalue is:  10.2076902286
 
 	[2] In [13]: print "Average max eigenvalue is: ", sum(max_evals)/len(max_evals)
 	[2] Out[13]: Average max eigenvalue is:  10.1891484655
 
 	[3] In [12]: print "Average max eigenvalue is: ", sum(max_evals)/len(max_evals)
 	[3] Out[12]: Average max eigenvalue is:  10.1158837784
 
 
 Moving Python objects around
 ============================
 
 In addition to executing code on engines, you can transfer Python objects to
 and from your IPython session and the engines. In IPython, these operations
 are called :meth:`push` (sending an object to the engines) and :meth:`pull`
 (getting an object from the engines).
 
 Basic push and pull
 -------------------
 
 Here are some examples of how you use :meth:`push` and :meth:`pull`:
 
 .. sourcecode:: ipython
 
 	In [38]: mec.push(dict(a=1.03234,b=3453))
 	Out[38]: [None, None, None, None]
 
 	In [39]: mec.pull('a')
 	Out[39]: [1.03234, 1.03234, 1.03234, 1.03234]
 
 	In [40]: mec.pull('b',targets=0)
 	Out[40]: [3453]
 
 	In [41]: mec.pull(('a','b'))
 	Out[41]: [[1.03234, 3453], [1.03234, 3453], [1.03234, 3453], [1.03234, 3453]]
 
 	In [42]: mec.zip_pull(('a','b'))
 	Out[42]: [(1.03234, 1.03234, 1.03234, 1.03234), (3453, 3453, 3453, 3453)]
 
 	In [43]: mec.push(dict(c='speed'))
 	Out[43]: [None, None, None, None]
 
 	In [44]: %px print c
 	Executing command on Controller
 	Out[44]: 
 	<Results List>
 	[0] In [14]: print c
 	[0] Out[14]: speed
 
 	[1] In [13]: print c
 	[1] Out[13]: speed
 
 	[2] In [14]: print c
 	[2] Out[14]: speed
 
 	[3] In [13]: print c
 	[3] Out[13]: speed
 
 In non-blocking mode :meth:`push` and :meth:`pull` also return
 :class:`PendingResult` objects:
 
 .. sourcecode:: ipython
 
 	In [47]: mec.block=False
 
 	In [48]: pr = mec.pull('a')
 
 	In [49]: pr.r
 	Out[49]: [1.03234, 1.03234, 1.03234, 1.03234]
 
 
 Push and pull for functions
 ---------------------------
 
 Functions can also be pushed and pulled using :meth:`push_function` and
 :meth:`pull_function`:
 
 .. sourcecode:: ipython
 
     In [52]: mec.block=True
 
 	In [53]: def f(x):
 	   ....:     return 2.0*x**4
 	   ....: 
 
 	In [54]: mec.push_function(dict(f=f))
 	Out[54]: [None, None, None, None]
 
 	In [55]: mec.execute('y = f(4.0)')
 	Out[55]: 
 	<Results List>
 	[0] In [15]: y = f(4.0)
 	[1] In [14]: y = f(4.0)
 	[2] In [15]: y = f(4.0)
 	[3] In [14]: y = f(4.0)
 
 
 	In [56]: px print y
 	Executing command on Controller
 	Out[56]: 
 	<Results List>
 	[0] In [16]: print y
 	[0] Out[16]: 512.0
 
 	[1] In [15]: print y
 	[1] Out[15]: 512.0
 
 	[2] In [16]: print y
 	[2] Out[16]: 512.0
 
 	[3] In [15]: print y
 	[3] Out[15]: 512.0
 
 
 Dictionary interface
 --------------------
 
 As a shorthand to :meth:`push` and :meth:`pull`, the
 :class:`MultiEngineClient` class implements some of the Python dictionary
 interface. This make the remote namespaces of the engines appear as a local
 dictionary. Underneath, this uses :meth:`push` and :meth:`pull`:
 
 .. sourcecode:: ipython
 
 	In [50]: mec.block=True
 
 	In [51]: mec['a']=['foo','bar']
 
 	In [52]: mec['a']
 	Out[52]: [['foo', 'bar'], ['foo', 'bar'], ['foo', 'bar'], ['foo', 'bar']]
 
 Scatter and gather
 ------------------
 
 Sometimes it is useful to partition a sequence and push the partitions to
 different engines. In MPI language, this is know as scatter/gather and we
 follow that terminology. However, it is important to remember that in
 IPython's :class:`MultiEngineClient` class, :meth:`scatter` is from the
 interactive IPython session to the engines and :meth:`gather` is from the
 engines back to the interactive IPython session. For scatter/gather operations
 between engines, MPI should be used:
 
 .. sourcecode:: ipython
 
 	In [58]: mec.scatter('a',range(16))
 	Out[58]: [None, None, None, None]
 
 	In [59]: px print a
 	Executing command on Controller
 	Out[59]: 
 	<Results List>
 	[0] In [17]: print a
 	[0] Out[17]: [0, 1, 2, 3]
 
 	[1] In [16]: print a
 	[1] Out[16]: [4, 5, 6, 7]
 
 	[2] In [17]: print a
 	[2] Out[17]: [8, 9, 10, 11]
 
 	[3] In [16]: print a
 	[3] Out[16]: [12, 13, 14, 15]
 
 
 	In [60]: mec.gather('a')
 	Out[60]: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
 
 Other things to look at
 =======================
 
 How to do parallel list comprehensions
 --------------------------------------
 
 In many cases list comprehensions are nicer than using the map function. While
 we don't have fully parallel list comprehensions, it is simple to get the
 basic effect using :meth:`scatter` and :meth:`gather`:
 
 .. sourcecode:: ipython
 
 	In [66]: mec.scatter('x',range(64))
 	Out[66]: [None, None, None, None]
 
 	In [67]: px y = [i**10 for i in x]
 	Executing command on Controller
 	Out[67]: 
 	<Results List>
 	[0] In [19]: y = [i**10 for i in x]
 	[1] In [18]: y = [i**10 for i in x]
 	[2] In [19]: y = [i**10 for i in x]
 	[3] In [18]: y = [i**10 for i in x]
 
 
 	In [68]: y = mec.gather('y')
 
 	In [69]: print y
 	[0, 1, 1024, 59049, 1048576, 9765625, 60466176, 282475249, 1073741824,...]
 
 Parallel exceptions
 -------------------
 
 In the multiengine interface, parallel commands can raise Python exceptions,
 just like serial commands. But, it is a little subtle, because a single
 parallel command can actually raise multiple exceptions (one for each engine
 the command was run on). To express this idea, the MultiEngine interface has a
 :exc:`CompositeError` exception class that will be raised in most cases. The
 :exc:`CompositeError` class is a special type of exception that wraps one or
 more other types of exceptions. Here is how it works:
 
 .. sourcecode:: ipython
 
 	In [76]: mec.block=True
 
 	In [77]: mec.execute('1/0')
 	---------------------------------------------------------------------------
 	CompositeError                            Traceback (most recent call last)
 
 	/ipython1-client-r3021/docs/examples/<ipython console> in <module>()
 
 	/ipython1-client-r3021/ipython1/kernel/multiengineclient.pyc in execute(self, lines, targets, block)
 	    432         targets, block = self._findTargetsAndBlock(targets, block)
 	    433         result = blockingCallFromThread(self.smultiengine.execute, lines,
 	--> 434             targets=targets, block=block)
 	    435         if block:
 	    436             result = ResultList(result)
 
 	/ipython1-client-r3021/ipython1/kernel/twistedutil.pyc in blockingCallFromThread(f, *a, **kw)
 	     72             result.raiseException()
 	     73         except Exception, e:
 	---> 74             raise e
 	     75     return result
 	     76 
 
 	CompositeError: one or more exceptions from call to method: execute
 	[0:execute]: ZeroDivisionError: integer division or modulo by zero
 	[1:execute]: ZeroDivisionError: integer division or modulo by zero
 	[2:execute]: ZeroDivisionError: integer division or modulo by zero
 	[3:execute]: ZeroDivisionError: integer division or modulo by zero
 
 Notice how the error message printed when :exc:`CompositeError` is raised has
 information about the individual exceptions that were raised on each engine.
 If you want, you can even raise one of these original exceptions:
 
 .. sourcecode:: ipython
 
 	In [80]: try:
 	   ....:     mec.execute('1/0')
 	   ....: except client.CompositeError, e:
 	   ....:     e.raise_exception()
 	   ....:     
 	   ....:     
 	---------------------------------------------------------------------------
 	ZeroDivisionError                         Traceback (most recent call last)
 
 	/ipython1-client-r3021/docs/examples/<ipython console> in <module>()
 
 	/ipython1-client-r3021/ipython1/kernel/error.pyc in raise_exception(self, excid)
 	    156             raise IndexError("an exception with index %i does not exist"%excid)
 	    157         else:
 	--> 158             raise et, ev, etb
 	    159 
 	    160 def collect_exceptions(rlist, method):
 
 	ZeroDivisionError: integer division or modulo by zero
 
 If you are working in IPython, you can simple type ``%debug`` after one of
 these :exc:`CompositeError` exceptions is raised, and inspect the exception
 instance:
 
 .. sourcecode:: ipython
 
 	In [81]: mec.execute('1/0')
 	---------------------------------------------------------------------------
 	CompositeError                            Traceback (most recent call last)
 
 	/ipython1-client-r3021/docs/examples/<ipython console> in <module>()
 
 	/ipython1-client-r3021/ipython1/kernel/multiengineclient.pyc in execute(self, lines, targets, block)
 	    432         targets, block = self._findTargetsAndBlock(targets, block)
 	    433         result = blockingCallFromThread(self.smultiengine.execute, lines,
 	--> 434             targets=targets, block=block)
 	    435         if block:
 	    436             result = ResultList(result)
 
 	/ipython1-client-r3021/ipython1/kernel/twistedutil.pyc in blockingCallFromThread(f, *a, **kw)
 	     72             result.raiseException()
 	     73         except Exception, e:
 	---> 74             raise e
 	     75     return result
 	     76 
 
 	CompositeError: one or more exceptions from call to method: execute
 	[0:execute]: ZeroDivisionError: integer division or modulo by zero
 	[1:execute]: ZeroDivisionError: integer division or modulo by zero
 	[2:execute]: ZeroDivisionError: integer division or modulo by zero
 	[3:execute]: ZeroDivisionError: integer division or modulo by zero
 
 	In [82]: %debug
 	> 
 	
 	/ipython1-client-r3021/ipython1/kernel/twistedutil.py(74)blockingCallFromThread()
 	     73         except Exception, e:
 	---> 74             raise e
 	     75     return result
 
 	# With the debugger running, e is the exceptions instance.  We can tab complete
 	# on it and see the extra methods that are available.
 	ipdb> e.
 	e.__class__         e.__getitem__       e.__new__           e.__setstate__      e.args
 	e.__delattr__       e.__getslice__      e.__reduce__        e.__str__           e.elist
 	e.__dict__          e.__hash__          e.__reduce_ex__     e.__weakref__       e.message
 	e.__doc__           e.__init__          e.__repr__          e._get_engine_str   e.print_tracebacks
 	e.__getattribute__  e.__module__        e.__setattr__       e._get_traceback    e.raise_exception
 	ipdb> e.print_tracebacks()
 	[0:execute]: 
 	---------------------------------------------------------------------------
 	ZeroDivisionError                         Traceback (most recent call last)
 
 	/ipython1-client-r3021/docs/examples/<string> in <module>()
 
 	ZeroDivisionError: integer division or modulo by zero
 
 	[1:execute]: 
 	---------------------------------------------------------------------------
 	ZeroDivisionError                         Traceback (most recent call last)
 
 	/ipython1-client-r3021/docs/examples/<string> in <module>()
 
 	ZeroDivisionError: integer division or modulo by zero
 
 	[2:execute]: 
 	---------------------------------------------------------------------------
 	ZeroDivisionError                         Traceback (most recent call last)
 
 	/ipython1-client-r3021/docs/examples/<string> in <module>()
 
 	ZeroDivisionError: integer division or modulo by zero
 
 	[3:execute]: 
 	---------------------------------------------------------------------------
 	ZeroDivisionError                         Traceback (most recent call last)
 
 	/ipython1-client-r3021/docs/examples/<string> in <module>()
 
 	ZeroDivisionError: integer division or modulo by zero
 
 .. note::
 
     The above example appears to be broken right now because of a change in 
     how we are using Twisted.
 
 All of this same error handling magic even works in non-blocking mode:
 
 .. sourcecode:: ipython
 
 	In [83]: mec.block=False
 
 	In [84]: pr = mec.execute('1/0')
 
 	In [85]: pr.r
 	---------------------------------------------------------------------------
 	CompositeError                            Traceback (most recent call last)
 
 	/ipython1-client-r3021/docs/examples/<ipython console> in <module>()
 
 	/ipython1-client-r3021/ipython1/kernel/multiengineclient.pyc in _get_r(self)
 	    170 
 	    171     def _get_r(self):
 	--> 172         return self.get_result(block=True)
 	    173 
 	    174     r = property(_get_r)
 
 	/ipython1-client-r3021/ipython1/kernel/multiengineclient.pyc in get_result(self, default, block)
 	    131                 return self.result
 	    132         try:
 	--> 133             result = self.client.get_pending_deferred(self.result_id, block)
 	    134         except error.ResultNotCompleted:
 	    135             return default
 
 	/ipython1-client-r3021/ipython1/kernel/multiengineclient.pyc in get_pending_deferred(self, deferredID, block)
 	    385 
 	    386     def get_pending_deferred(self, deferredID, block):
 	--> 387         return blockingCallFromThread(self.smultiengine.get_pending_deferred, deferredID, block)
 	    388 
 	    389     def barrier(self, pendingResults):
 
 	/ipython1-client-r3021/ipython1/kernel/twistedutil.pyc in blockingCallFromThread(f, *a, **kw)
 	     72             result.raiseException()
 	     73         except Exception, e:
 	---> 74             raise e
 	     75     return result
 	     76 
 
 	CompositeError: one or more exceptions from call to method: execute
 	[0:execute]: ZeroDivisionError: integer division or modulo by zero
 	[1:execute]: ZeroDivisionError: integer division or modulo by zero
 	[2:execute]: ZeroDivisionError: integer division or modulo by zero
 	[3:execute]: ZeroDivisionError: integer division or modulo by zero
 
 

docs/source/parallel/parallel_process.txt

@@ -1,389 +1,389 b''
 .. _parallel_process:
 
 ===========================================
 Starting the IPython controller and engines
 ===========================================
 
 To use IPython for parallel computing, you need to start one instance of
 the controller and one or more instances of the engine. The controller
 and each engine can run on different machines or on the same machine.
 Because of this, there are many different possibilities.
 
 Broadly speaking, there are two ways of going about starting a controller and engines:
 
 * In an automated manner using the :command:`ipcluster` command.
 * In a more manual way using the :command:`ipcontroller` and 
   :command:`ipengine` commands.
 
 This document describes both of these methods. We recommend that new users
 start with the :command:`ipcluster` command as it simplifies many common usage
 cases.
 
 General considerations
 ======================
 
 Before delving into the details about how you can start a controller and
 engines using the various methods, we outline some of the general issues that
 come up when starting the controller and engines. These things come up no
 matter which method you use to start your IPython cluster.
 
 Let's say that you want to start the controller on ``host0`` and engines on
 hosts ``host1``-``hostn``. The following steps are then required:
 
 1. Start the controller on ``host0`` by running :command:`ipcontroller` on
    ``host0``.
 2. Move the FURL file (:file:`ipcontroller-engine.furl`) created by the 
    controller from ``host0`` to hosts ``host1``-``hostn``.
 3. Start the engines on hosts ``host1``-``hostn`` by running 
    :command:`ipengine`.  This command has to be told where the FURL file 
    (:file:`ipcontroller-engine.furl`) is located.
 
 At this point, the controller and engines will be connected. By default, the
 FURL files created by the controller are put into the
-:file:`~/.ipython/security` directory. If the engines share a filesystem with
+:file:`$IPYTHON_DIR/cluster_<profile>/security` directory. If the engines share a filesystem with
 the controller, step 2 can be skipped as the engines will automatically look
 at that location.
 
 The final step required required to actually use the running controller from a
 client is to move the FURL files :file:`ipcontroller-mec.furl` and
 :file:`ipcontroller-tc.furl` from ``host0`` to the host where the clients will
-be run. If these file are put into the :file:`~/.ipython/security` directory
+be run. If these file are put into the :file:`$IPYTHON_DIR/cluster_<profile>/security` directory
 of the client's host, they will be found automatically. Otherwise, the full
 path to them has to be passed to the client's constructor.
 
 Using :command:`ipcluster`
 ==========================
 
 The :command:`ipcluster` command provides a simple way of starting a
 controller and engines in the following situations:
 
 1. When the controller and engines are all run on localhost. This is useful
    for testing or running on a multicore computer.
 2. When engines are started using the :command:`mpirun` command that comes 
    with most MPI [MPI]_ implementations
 3. When engines are started using the PBS [PBS]_ batch system.
 4. When the controller is started on localhost and the engines are started on 
    remote nodes using :command:`ssh`.
 
 .. note::
 
     It is also possible for advanced users to add support to
     :command:`ipcluster` for starting controllers and engines using other
     methods (like Sun's Grid Engine for example).
 
 .. note::
 
     Currently :command:`ipcluster` requires that the
-    :file:`~/.ipython/security` directory live on a shared filesystem that is
+    :file:`$IPYTHON_DIR/cluster_<profile>/security` directory live on a shared filesystem that is
     seen by both the controller and engines. If you don't have a shared file
     system you will need to use :command:`ipcontroller` and
     :command:`ipengine` directly.  This constraint can be relaxed if you are 
     using the :command:`ssh` method to start the cluster.
 
 Underneath the hood, :command:`ipcluster` just uses :command:`ipcontroller`
 and :command:`ipengine` to perform the steps described above.
 
 Using :command:`ipcluster` in local mode
 ----------------------------------------
 
 To start one controller and 4 engines on localhost, just do::
 
     $ ipcluster local -n 4
 
 To see other command line options for the local mode, do::
 
     $ ipcluster local -h
 
 Using :command:`ipcluster` in mpiexec/mpirun mode
 -------------------------------------------------
 
 The mpiexec/mpirun mode is useful if you:
 
 1. Have MPI installed.
 2. Your systems are configured to use the :command:`mpiexec` or
    :command:`mpirun` commands to start MPI processes.
 
 .. note::
 
     The preferred command to use is :command:`mpiexec`.  However, we also 
     support :command:`mpirun` for backwards compatibility.  The underlying
     logic used is exactly the same, the only difference being the name of the 
     command line program that is called.
 
 If these are satisfied, you can start an IPython cluster using::
 
     $ ipcluster mpiexec -n 4
 
 This does the following:
 
 1. Starts the IPython controller on current host.
 2. Uses :command:`mpiexec` to start 4 engines.
 
 On newer MPI implementations (such as OpenMPI), this will work even if you
 don't make any calls to MPI or call :func:`MPI_Init`. However, older MPI
 implementations actually require each process to call :func:`MPI_Init` upon
 starting. The easiest way of having this done is to install the mpi4py
 [mpi4py]_ package and then call ipcluster with the ``--mpi`` option::
 
     $ ipcluster mpiexec -n 4 --mpi=mpi4py
 
 Unfortunately, even this won't work for some MPI implementations. If you are
 having problems with this, you will likely have to use a custom Python
 executable that itself calls :func:`MPI_Init` at the appropriate time.
 Fortunately, mpi4py comes with such a custom Python executable that is easy to
 install and use. However, this custom Python executable approach will not work
 with :command:`ipcluster` currently.
 
 Additional command line options for this mode can be found by doing::
 
     $ ipcluster mpiexec -h
 
 More details on using MPI with IPython can be found :ref:`here <parallelmpi>`.
 
 
 Using :command:`ipcluster` in PBS mode
 --------------------------------------
 
 The PBS mode uses the Portable Batch System [PBS]_ to start the engines. To
 use this mode, you first need to create a PBS script template that will be
 used to start the engines. Here is a sample PBS script template:
 
 .. sourcecode:: bash
 
     #PBS -N ipython
     #PBS -j oe
     #PBS -l walltime=00:10:00
     #PBS -l nodes=${n/4}:ppn=4
     #PBS -q parallel
 
     cd $$PBS_O_WORKDIR
     export PATH=$$HOME/usr/local/bin
     export PYTHONPATH=$$HOME/usr/local/lib/python2.4/site-packages
     /usr/local/bin/mpiexec -n ${n} ipengine --logfile=$$PBS_O_WORKDIR/ipengine
 
 There are a few important points about this template:
 
 1. This template will be rendered at runtime using IPython's :mod:`Itpl` 
    template engine.
 
 2. Instead of putting in the actual number of engines, use the notation
    ``${n}`` to indicate the number of engines to be started. You can also uses
    expressions like ``${n/4}`` in the template to indicate the number of 
    nodes.
 
 3. Because ``$`` is a special character used by the template engine, you must  
    escape any ``$`` by using ``$$``.  This is important when referring to 
    environment variables in the template.
 
 4. Any options to :command:`ipengine` should be given in the batch script 
    template.
 
 5. Depending on the configuration of you system, you may have to set 
    environment variables in the script template.
 
 Once you have created such a script, save it with a name like
 :file:`pbs.template`. Now you are ready to start your job::
 
     $ ipcluster pbs -n 128 --pbs-script=pbs.template
 
 Additional command line options for this mode can be found by doing::
 
     $ ipcluster pbs -h
 
 Using :command:`ipcluster` in SSH mode
 --------------------------------------
 
 The SSH mode uses :command:`ssh` to execute :command:`ipengine` on remote
 nodes and the :command:`ipcontroller` on localhost.
 
 When using using this mode it highly recommended that you have set up SSH keys
 and are using ssh-agent [SSH]_ for password-less logins.
 
 To use this mode you need a python file describing the cluster, here is an
 example of such a "clusterfile":
 
 .. sourcecode:: python
     
     send_furl = True
     engines = { 'host1.example.com' : 2,
                 'host2.example.com' : 5,
                 'host3.example.com' : 1,
                 'host4.example.com' : 8 }
 
 Since this is a regular python file usual python syntax applies. Things to
 note:
 
 * The `engines` dict, where the keys is the host we want to run engines on and
   the value is the number of engines to run on that host.
 * send_furl can either be `True` or `False`, if `True` it will copy over the
   furl needed for :command:`ipengine` to each host.
 
 The ``--clusterfile`` command line option lets you specify the file to use for
 the cluster definition. Once you have your cluster file and you can
 :command:`ssh` into the remote hosts with out an password you are ready to
 start your cluster like so:
 
 .. sourcecode:: bash 
 
    $ ipcluster ssh --clusterfile /path/to/my/clusterfile.py
 
 
 Two helper shell scripts are used to start and stop :command:`ipengine` on
 remote hosts:
 
 * sshx.sh 
 * engine_killer.sh
 
 Defaults for both of these are contained in the source code for
 :command:`ipcluster`. The default scripts are written to a local file in a
 tmep directory and then copied to a temp directory on the remote host and
 executed from there. On most Unix, Linux and OS X systems this is /tmp.
 
 The default sshx.sh is the following:
 
 .. sourcecode:: bash
 
    #!/bin/sh
    "$@" &> /dev/null &
    echo $!
 
 If you want to use a custom sshx.sh script you need to use the ``--sshx``
 option and specify the file to use. Using a custom sshx.sh file could be
 helpful when you need to setup the environment on the remote host before
 executing :command:`ipengine`.
 
 For a detailed options list:
 
 .. sourcecode:: bash
 
    $ ipcluster ssh -h
 
 Current limitations of the SSH mode of :command:`ipcluster` are:
 
 * Untested on Windows. Would require a working :command:`ssh` on Windows.
   Also, we are using shell scripts to setup and execute commands on remote
   hosts.
 * :command:`ipcontroller` is started on localhost, with no option to start it
   on a remote node.
 
 Using the :command:`ipcontroller` and :command:`ipengine` commands
 ==================================================================
 
 It is also possible to use the :command:`ipcontroller` and :command:`ipengine`
 commands to start your controller and engines. This approach gives you full
 control over all aspects of the startup process.
 
 Starting the controller and engine on your local machine
 --------------------------------------------------------
 
 To use :command:`ipcontroller` and :command:`ipengine` to start things on your
 local machine, do the following.
 
 First start the controller::
 
 	$ ipcontroller
 	
 Next, start however many instances of the engine you want using (repeatedly)
 the command::
 
 	$ ipengine
 
 The engines should start and automatically connect to the controller using the
-FURL files in :file:`~./ipython/security`. You are now ready to use the
+FURL files in :file:`$IPYTHON_DIR/cluster_<profile>/security`. You are now ready to use the
 controller and engines from IPython.
 
 .. warning:: 
 	
 	The order of the above operations is very important.  You *must*
  	start the controller before the engines, since the engines connect 
 	to the controller as they get started.
 
 .. note::
 
     On some platforms (OS X), to put the controller and engine into the
     background you may need to give these commands in the form ``(ipcontroller
     &)`` and ``(ipengine &)`` (with the parentheses) for them to work
     properly.
 
 Starting the controller and engines on different hosts
 ------------------------------------------------------
 
 When the controller and engines are running on different hosts, things are
 slightly more complicated, but the underlying ideas are the same:
 
 1. Start the controller on a host using :command:`ipcontroller`.
-2. Copy :file:`ipcontroller-engine.furl` from :file:`~./ipython/security` on
+2. Copy :file:`ipcontroller-engine.furl` from :file:`$IPYTHON_DIR/cluster_<profile>/security` on
    the controller's host to the host where the engines will run.
 3. Use :command:`ipengine` on the engine's hosts to start the engines.
 
 The only thing you have to be careful of is to tell :command:`ipengine` where
 the :file:`ipcontroller-engine.furl` file is located. There are two ways you
 can do this:
 
-* Put :file:`ipcontroller-engine.furl` in the :file:`~./ipython/security`
+* Put :file:`ipcontroller-engine.furl` in the :file:`$IPYTHON_DIR/cluster_<profile>/security`
   directory on the engine's host, where it will be found automatically.
 * Call :command:`ipengine` with the ``--furl-file=full_path_to_the_file``
   flag.
 
 The ``--furl-file`` flag works like this::
 
     $ ipengine --furl-file=/path/to/my/ipcontroller-engine.furl
 
 .. note::
     
     If the controller's and engine's hosts all have a shared file system  
-    (:file:`~./ipython/security` is the same on all of them), then things
+    (:file:`$IPYTHON_DIR/cluster_<profile>/security` is the same on all of them), then things
     will just work!
 
 Make FURL files persistent
 ---------------------------
 
 At fist glance it may seem that that managing the FURL files is a bit
 annoying. Going back to the house and key analogy, copying the FURL around
 each time you start the controller is like having to make a new key every time
 you want to unlock the door and enter your house. As with your house, you want
 to be able to create the key (or FURL file) once, and then simply use it at
 any point in the future.
 
 This is possible, but before you do this, you **must** remove any old FURL
-files in the :file:`~/.ipython/security` directory.
+files in the :file:`$IPYTHON_DIR/cluster_<profile>/security` directory.
 
 .. warning::
 
     You **must** remove old FURL files before using persistent FURL files.
 
 Then, The only thing you have to do is decide what ports the controller will
 listen on for the engines and clients. This is done as follows::
 
     $ ipcontroller -r --client-port=10101 --engine-port=10102
 
 These options also work with all of the various modes of
 :command:`ipcluster`::
 
     $ ipcluster local -n 2 -r --client-port=10101 --engine-port=10102
 
 Then, just copy the furl files over the first time and you are set. You can
 start and stop the controller and engines any many times as you want in the
 future, just make sure to tell the controller to use the *same* ports.
 
 .. note::
 
     You may ask the question: what ports does the controller listen on if you
     don't tell is to use specific ones? The default is to use high random port
     numbers. We do this for two reasons: i) to increase security through
     obscurity and ii) to multiple controllers on a given host to start and
     automatically use different ports.
 
 Log files
 ---------
 
 All of the components of IPython have log files associated with them.
 These log files can be extremely useful in debugging problems with
-IPython and can be found in the directory :file:`~/.ipython/log`.  Sending
+IPython and can be found in the directory :file:`$IPYTHON_DIR/cluster_<profile>/log`.  Sending
 the log files to us will often help us to debug any problems.
 
 
 .. [PBS] Portable Batch System.  http://www.openpbs.org/
 .. [SSH] SSH-Agent http://en.wikipedia.org/wiki/Ssh-agent
 
 

docs/source/parallel/parallel_security.txt

@@ -1,366 +1,366 b''
 .. _parallelsecurity:
 
 ===========================
 Security details of IPython
 ===========================
 
 IPython's :mod:`IPython.kernel` package exposes the full power of the Python
 interpreter over a TCP/IP network for the purposes of parallel computing. This
 feature brings up the important question of IPython's security model. This
 document gives details about this model and how it is implemented in IPython's
 architecture.
 
 Processs and network topology
 =============================
 
 To enable parallel computing, IPython has a number of different processes that
 run. These processes are discussed at length in the IPython documentation and
 are summarized here:
 
 * The IPython *engine*.  This process is a full blown Python
   interpreter in which user code is executed.  Multiple
   engines are started to make parallel computing possible.
 * The IPython *controller*.  This process manages a set of 
   engines, maintaining a queue for each and presenting
   an asynchronous interface to the set of engines.
 * The IPython *client*.  This process is typically an
   interactive Python process that is used to coordinate the
   engines to get a parallel computation done.
 
 Collectively, these three processes are called the IPython *kernel*.
 
 These three processes communicate over TCP/IP connections with a well defined
 topology. The IPython controller is the only process that listens on TCP/IP
 sockets. Upon starting, an engine connects to a controller and registers
 itself with the controller. These engine/controller TCP/IP connections persist
 for the lifetime of each engine.
 
 The IPython client also connects to the controller using one or more TCP/IP
 connections. These connections persist for the lifetime of the client only.
 
 A given IPython controller and set of engines typically has a relatively short
 lifetime. Typically this lifetime corresponds to the duration of a single
 parallel simulation performed by a single user. Finally, the controller,
 engines and client processes typically execute with the permissions of that
 same user. More specifically, the controller and engines are *not* executed as
 root or with any other superuser permissions.
 
 Application logic
 =================
 
 When running the IPython kernel to perform a parallel computation, a user
 utilizes the IPython client to send Python commands and data through the
 IPython controller to the IPython engines, where those commands are executed
 and the data processed. The design of IPython ensures that the client is the
 only access point for the capabilities of the engines. That is, the only way
 of addressing the engines is through a client.
 
 A user can utilize the client to instruct the IPython engines to execute
 arbitrary Python commands. These Python commands can include calls to the
 system shell, access the filesystem, etc., as required by the user's
 application code. From this perspective, when a user runs an IPython engine on
 a host, that engine has the same capabilities and permissions as the user
 themselves (as if they were logged onto the engine's host with a terminal).
 
 Secure network connections
 ==========================
 
 Overview
 --------
 
 All TCP/IP connections between the client and controller as well as the
 engines and controller are fully encrypted and authenticated. This section
 describes the details of the encryption and authentication approached used
 within IPython.
 
 IPython uses the Foolscap network protocol [Foolscap]_ for all communications
 between processes. Thus, the details of IPython's security model are directly
 related to those of Foolscap. Thus, much of the following discussion is
 actually just a discussion of the security that is built in to Foolscap.
 
 Encryption
 ----------
 
 For encryption purposes, IPython and Foolscap use the well known Secure Socket
 Layer (SSL) protocol [RFC5246]_. We use the implementation of this protocol
 provided by the OpenSSL project through the pyOpenSSL [pyOpenSSL]_ Python
 bindings to OpenSSL.
 
 Authentication
 --------------
 
 IPython clients and engines must also authenticate themselves with the
 controller. This is handled in a capabilities based security model
 [Capability]_. In this model, the controller creates a strong cryptographic
 key or token that represents each set of capability that the controller
 offers. Any party who has this key and presents it to the controller has full
 access to the corresponding capabilities of the controller. This model is
 analogous to using a physical key to gain access to physical items
 (capabilities) behind a locked door.
 
 For a capabilities based authentication system to prevent unauthorized access,
 two things must be ensured:
 
 * The keys must be cryptographically strong.  Otherwise attackers could gain 
   access by a simple brute force key guessing attack.
 * The actual keys must be distributed only to authorized parties.
 
 The keys in Foolscap are called Foolscap URL's or FURLs. The following section
 gives details about how these FURLs are created in Foolscap. The IPython
 controller creates a number of FURLs for different purposes:
 
 * One FURL that grants IPython engines access to the controller.  Also
   implicit in this access is permission to execute code sent by an
   authenticated IPython client.
 * Two or more FURLs that grant IPython clients access to the controller.
   Implicit in this access is permission to give the controller's engine code 
   to execute.
 
 Upon starting, the controller creates these different FURLS and writes them
-files in the user-read-only directory :file:`$HOME/.ipython/security`. Thus,
+files in the user-read-only directory :file:`$IPYTHON_DIR/cluster_default/security`. Thus,
 only the user who starts the controller has access to the FURLs.
 
 For an IPython client or engine to authenticate with a controller, it must
 present the appropriate FURL to the controller upon connecting. If the
 FURL matches what the controller expects for a given capability, access is
 granted. If not, access is denied. The exchange of FURLs is done after
 encrypted communications channels have been established to prevent attackers
 from capturing them.
 
 .. note::
 
     The FURL is similar to an unsigned private key in SSH.
 
 Details of the Foolscap handshake
 ---------------------------------
 
 In this section we detail the precise security handshake that takes place at
 the beginning of any network connection in IPython. For the purposes of this
 discussion, the SERVER is the IPython controller process and the CLIENT is the
 IPython engine or client process.
 
 Upon starting, all IPython processes do the following:
 
 1. Create a public key x509 certificate (ISO/IEC 9594).
 2. Create a hash of the contents of the certificate using the SHA-1 algorithm.
    The base-32 encoded version of this hash is saved by the process as its 
    process id (actually in Foolscap, this is the Tub id, but here refer to
    it as the process id).
 
 Upon starting, the IPython controller also does the following:
 
 1. Save the x509 certificate to disk in a secure location. The CLIENT
    certificate is never saved to disk.
 2. Create a FURL for each capability that the controller has.  There are
    separate capabilities the controller offers for clients and engines.  The
    FURL is created using:  a) the process id of the SERVER, b) the IP 
    address and port the SERVER is listening on and c) a 160 bit,    
    cryptographically secure string that represents the capability (the
    "capability id").
 3. The FURLs are saved to disk in a secure location on the SERVER's host.
 
 For a CLIENT to be able to connect to the SERVER and access a capability of
 that SERVER, the CLIENT must have knowledge of the FURL for that SERVER's
 capability. This typically requires that the file containing the FURL be
 moved from the SERVER's host to the CLIENT's host. This is done by the end
 user who started the SERVER and wishes to have a CLIENT connect to the SERVER.
 
 When a CLIENT connects to the SERVER, the following handshake protocol takes
 place:
 
 1. The CLIENT tells the SERVER what process (or Tub) id it expects the SERVER
    to have.
 2. If the SERVER has that process id, it notifies the CLIENT that it will now 
    enter encrypted mode.  If the SERVER has a different id, the SERVER aborts.
 3. Both CLIENT and SERVER initiate the SSL handshake protocol.
 4. Both CLIENT and SERVER request the certificate of their peer and verify
    that certificate.  If this succeeds, all further communications are 
    encrypted.
 5. Both CLIENT and SERVER send a hello block containing connection parameters
    and their process id.
 6. The CLIENT and SERVER check that their peer's stated process id matches the
    hash of the x509 certificate the peer presented.  If not, the connection is
    aborted.
 7. The CLIENT verifies that the SERVER's stated id matches the id of the
    SERVER the CLIENT is intending to connect to.  If not, the connection is 
    aborted.
 8. The CLIENT and SERVER elect a master who decides on the final connection
    parameters.
 
 The public/private key pair associated with each process's x509 certificate
 are completely hidden from this handshake protocol. There are however, used
 internally by OpenSSL as part of the SSL handshake protocol. Each process
 keeps their own private key hidden and sends its peer only the public key
 (embedded in the certificate).
 
 Finally, when the CLIENT requests access to a particular SERVER capability,
 the following happens:
 
 1. The CLIENT asks the SERVER for access to a capability by presenting that 
    capabilities id.
 2. If the SERVER has a capability with that id, access is granted. If not,
    access is not granted.
 3. Once access has been gained, the CLIENT can use the capability.
 
 Specific security vulnerabilities
 =================================
 
 There are a number of potential security vulnerabilities present in IPython's
 architecture. In this section we discuss those vulnerabilities and detail how
 the security architecture described above prevents them from being exploited.
 
 Unauthorized clients
 --------------------
 
 The IPython client can instruct the IPython engines to execute arbitrary
 Python code with the permissions of the user who started the engines. If an
 attacker were able to connect their own hostile IPython client to the IPython
 controller, they could instruct the engines to execute code.
 
 This attack is prevented by the capabilities based client authentication
 performed after the encrypted channel has been established. The relevant
 authentication information is encoded into the FURL that clients must
 present to gain access to the IPython controller. By limiting the distribution
 of those FURLs, a user can grant access to only authorized persons.
 
 It is highly unlikely that a client FURL could be guessed by an attacker
 in a brute force guessing attack. A given instance of the IPython controller
 only runs for a relatively short amount of time (on the order of hours). Thus
 an attacker would have only a limited amount of time to test a search space of
 size 2**320. Furthermore, even if a controller were to run for a longer amount
 of time, this search space is quite large (larger for instance than that of
 typical username/password pair).
 
 Unauthorized engines
 --------------------
 
 If an attacker were able to connect a hostile engine to a user's controller,
 the user might unknowingly send sensitive code or data to the hostile engine.
 This attacker's engine would then have full access to that code and data.
 
 This type of attack is prevented in the same way as the unauthorized client
 attack, through the usage of the capabilities based authentication scheme.
 
 Unauthorized controllers
 ------------------------
 
 It is also possible that an attacker could try to convince a user's IPython
 client or engine to connect to a hostile IPython controller. That controller
 would then have full access to the code and data sent between the IPython
 client and the IPython engines.
 
 Again, this attack is prevented through the FURLs, which ensure that a
 client or engine connects to the correct controller. It is also important to
 note that the FURLs also encode the IP address and port that the
 controller is listening on, so there is little chance of mistakenly connecting
 to a controller running on a different IP address and port.
 
 When starting an engine or client, a user must specify which FURL to use
 for that connection. Thus, in order to introduce a hostile controller, the
 attacker must convince the user to use the FURLs associated with the
 hostile controller. As long as a user is diligent in only using FURLs from
 trusted sources, this attack is not possible.
 
 Other security measures
 =======================
 
 A number of other measures are taken to further limit the security risks
 involved in running the IPython kernel.
 
 First, by default, the IPython controller listens on random port numbers.
 While this can be overridden by the user, in the default configuration, an
 attacker would have to do a port scan to even find a controller to attack.
 When coupled with the relatively short running time of a typical controller
 (on the order of hours), an attacker would have to work extremely hard and
 extremely *fast* to even find a running controller to attack.
 
 Second, much of the time, especially when run on supercomputers or clusters,
 the controller is running behind a firewall. Thus, for engines or client to
 connect to the controller:
 
 * The different processes have to all be behind the firewall.
 
 or:
 
 * The user has to use SSH port forwarding to tunnel the
   connections through the firewall.
    
 In either case, an attacker is presented with addition barriers that prevent
 attacking or even probing the system.
 
 Summary
 =======
 
 IPython's architecture has been carefully designed with security in mind. The
 capabilities based authentication model, in conjunction with the encrypted
 TCP/IP channels, address the core potential vulnerabilities in the system,
 while still enabling user's to use the system in open networks.
 
 Other questions
 ===============
 
 About keys
 ----------
 
 Can you clarify the roles of the certificate and its keys versus the FURL,
 which is also called a key?
 
 The certificate created by IPython processes is a standard public key x509
 certificate, that is used by the SSL handshake protocol to setup encrypted
 channel between the controller and the IPython engine or client. This public
 and private key associated with this certificate are used only by the SSL
 handshake protocol in setting up this encrypted channel.
 
 The FURL serves a completely different and independent purpose from the
 key pair associated with the certificate. When we refer to a FURL as a
 key, we are using the word "key" in the capabilities based security model
 sense. This has nothing to do with "key" in the public/private key sense used
 in the SSL protocol.
 
 With that said the FURL is used as an cryptographic key, to grant
 IPython engines and clients access to particular capabilities that the
 controller offers.
 
 Self signed certificates
 ------------------------
 
 Is the controller creating a self-signed certificate? Is this created for per
 instance/session, one-time-setup or each-time the controller is started?
 
 The Foolscap network protocol, which handles the SSL protocol details, creates
 a self-signed x509 certificate using OpenSSL for each IPython process. The
 lifetime of the certificate is handled differently for the IPython controller
 and the engines/client.
 
 For the IPython engines and client, the certificate is only held in memory for
 the lifetime of its process. It is never written to disk.
 
 For the controller, the certificate can be created anew each time the
 controller starts or it can be created once and reused each time the
 controller starts. If at any point, the certificate is deleted, a new one is
 created the next time the controller starts.
 
 SSL private key
 ---------------
 
 How the private key (associated with the certificate) is distributed?
 
 In the usual implementation of the SSL protocol, the private key is never
 distributed. We follow this standard always.
 
 SSL versus Foolscap authentication
 ----------------------------------
 
 Many SSL connections only perform one sided authentication (the server to the
 client). How is the client authentication in IPython's system related to SSL
 authentication?
 
 We perform a two way SSL handshake in which both parties request and verify
 the certificate of their peer. This mutual authentication is handled by the
 SSL handshake and is separate and independent from the additional
 authentication steps that the CLIENT and SERVER perform after an encrypted
 channel is established.
 
 .. [RFC5246] <http://tools.ietf.org/html/rfc5246>
 
 

docs/source/parallel/parallel_task.txt

@@ -1,121 +1,121 b''
 .. _paralleltask:
 
 ==========================
 The IPython task interface
 ==========================
 
 The task interface to the controller presents the engines as a fault tolerant,
 dynamic load-balanced system or workers. Unlike the multiengine interface, in
 the task interface, the user have no direct access to individual engines. In
 some ways, this interface is simpler, but in other ways it is more powerful.
 
 Best of all the user can use both of these interfaces running at the same time
 to take advantage or both of their strengths. When the user can break up the
 user's work into segments that do not depend on previous execution, the task
 interface is ideal. But it also has more power and flexibility, allowing the
 user to guide the distribution of jobs, without having to assign tasks to
 engines explicitly.
 
 Starting the IPython controller and engines
 ===========================================
 
 To follow along with this tutorial, you will need to start the IPython
 controller and four IPython engines. The simplest way of doing this is to use
 the :command:`ipcluster` command::
 
 	$ ipcluster local -n 4
 	
 For more detailed information about starting the controller and engines, see
 our :ref:`introduction <ip1par>` to using IPython for parallel computing.
 
 Creating a ``TaskClient`` instance
 =========================================
 
 The first step is to import the IPython :mod:`IPython.kernel.client` module
 and then create a :class:`TaskClient` instance:
 
 .. sourcecode:: ipython
 
 	In [1]: from IPython.kernel import client
 	
 	In [2]: tc = client.TaskClient()
 
 This form assumes that the :file:`ipcontroller-tc.furl` is in the
-:file:`~./ipython/security` directory on the client's host. If not, the
+:file:`$IPYTHON_DIR/cluster_<profile>/security` directory on the client's host. If not, the
 location of the FURL file must be given as an argument to the
 constructor:
 
 .. sourcecode:: ipython
 
     In [2]: mec = client.TaskClient('/path/to/my/ipcontroller-tc.furl')
 
 Quick and easy parallelism
 ==========================
 
 In many cases, you simply want to apply a Python function to a sequence of
 objects, but *in parallel*. Like the multiengine interface, the task interface
 provides two simple ways of accomplishing this: a parallel version of
 :func:`map` and ``@parallel`` function decorator. However, the verions in the
 task interface have one important difference: they are dynamically load
 balanced. Thus, if the execution time per item varies significantly, you
 should use the versions in the task interface.
 
 Parallel map
 ------------
 
 The parallel :meth:`map` in the task interface is similar to that in the
 multiengine interface:
 
 .. sourcecode:: ipython
 
 	In [63]: serial_result = map(lambda x:x**10, range(32))
 
 	In [64]: parallel_result = tc.map(lambda x:x**10, range(32))
 
 	In [65]: serial_result==parallel_result
 	Out[65]: True
 
 Parallel function decorator
 ---------------------------
 
 Parallel functions are just like normal function, but they can be called on
 sequences and *in parallel*. The multiengine interface provides a decorator
 that turns any Python function into a parallel function:
 
 .. sourcecode:: ipython
 
     In [10]: @tc.parallel()
        ....: def f(x):
        ....:     return 10.0*x**4
        ....: 
 
     In [11]: f(range(32))    # this is done in parallel
     Out[11]: 
     [0.0,10.0,160.0,...]
 
 More details
 ============
 
 The :class:`TaskClient` has many more powerful features that allow quite a bit
 of flexibility in how tasks are defined and run. The next places to look are
 in the following classes:
 
 * :class:`IPython.kernel.client.TaskClient`
 * :class:`IPython.kernel.client.StringTask`
 * :class:`IPython.kernel.client.MapTask`
 
 The following is an overview of how to use these classes together:
 
 1. Create a :class:`TaskClient`.
 2. Create one or more instances of :class:`StringTask` or :class:`MapTask`
    to define your tasks.
 3. Submit your tasks to using the :meth:`run` method of your
    :class:`TaskClient` instance.
 4. Use :meth:`TaskClient.get_task_result` to get the results of the
    tasks.
 
 We are in the process of developing more detailed information about the task
 interface. For now, the docstrings of the :class:`TaskClient`,
 :class:`StringTask` and :class:`MapTask` classes should be consulted.