Merging with upstream

IPython/testing/decorators_trial.py

@@ -0,0 +1,132 b''
+# encoding: utf-8
+"""
+Testing related decorators for use with twisted.trial.
+
+The decorators in this files are designed to follow the same API as those
+in the decorators module (in this same directory).  But they can be used
+with twisted.trial
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import sys
+
+from IPython.testing.decorators import make_label_dec
+
+#-----------------------------------------------------------------------------
+# Testing decorators
+#-----------------------------------------------------------------------------
+
+
+def skipif(skip_condition, msg=None):
+    """Create a decorator that marks a test function for skipping.
+
+    The is a decorator factory that returns a decorator that will 
+    conditionally skip a test based on the value of skip_condition.  The
+    skip_condition argument can either be a boolean or a callable that returns
+    a boolean.
+
+    Parameters
+    ----------
+    skip_condition : boolean or callable
+        If this evaluates to True, the test is skipped.
+    msg : str
+        The message to print if the test is skipped.
+
+    Returns
+    -------
+    decorator : function
+        The decorator function that can be applied to the test function.
+    """
+
+    def skip_decorator(f):
+
+        # Allow for both boolean or callable skip conditions.
+        if callable(skip_condition):
+            skip_val = lambda : skip_condition()
+        else:
+            skip_val = lambda : skip_condition
+
+        if msg is None:
+            out = 'Test skipped due to test condition.'
+        else: 
+            out = msg
+        final_msg = "Skipping test: %s. %s" % (f.__name__,out)
+
+        if skip_val():
+            f.skip = final_msg
+
+        return f
+    return skip_decorator
+
+
+def skip(msg=None):
+    """Create a decorator that marks a test function for skipping.
+
+    This is a decorator factory that returns a decorator that will cause
+    tests to be skipped.
+
+    Parameters
+    ----------
+    msg : str
+        Optional message to be added.
+
+    Returns
+    -------
+    decorator : function
+        Decorator, which, when applied to a function, sets the skip
+        attribute of the function causing `twisted.trial` to skip it.
+    """
+
+    return skipif(True,msg)
+
+
+def numpy_not_available():
+    """Can numpy be imported?  Returns true if numpy does NOT import.
+
+    This is used to make a decorator to skip tests that require numpy to be
+    available, but delay the 'import numpy' to test execution time.
+    """
+    try:
+        import numpy
+        np_not_avail = False
+    except ImportError:
+        np_not_avail = True
+
+    return np_not_avail
+
+#-----------------------------------------------------------------------------
+# Decorators for public use
+#-----------------------------------------------------------------------------
+
+# Decorators to skip certain tests on specific platforms.
+skip_win32 = skipif(sys.platform == 'win32',
+                    "This test does not run under Windows")
+skip_linux = skipif(sys.platform == 'linux2',
+                    "This test does not run under Linux")
+skip_osx = skipif(sys.platform == 'darwin',"This test does not run under OS X")
+
+# Decorators to skip tests if not on specific platforms.
+skip_if_not_win32 = skipif(sys.platform != 'win32',
+                           "This test only runs under Windows")
+skip_if_not_linux = skipif(sys.platform != 'linux2',
+                           "This test only runs under Linux")
+skip_if_not_osx = skipif(sys.platform != 'darwin',
+                         "This test only runs under OSX")
+
+# Other skip decorators
+skipif_not_numpy = skipif(numpy_not_available,"This test requires numpy")
+
+skipknownfailure = skip('This test is known to fail')
+
+

IPython/utils/attic.py

@@ -0,0 +1,167 b''
+# encoding: utf-8
+"""
+Older utilities that are not being used.
+
+WARNING: IF YOU NEED TO USE ONE OF THESE FUNCTIONS, PLEASE FIRST MOVE IT
+TO ANOTHER APPROPRIATE MODULE IN IPython.utils.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+import warnings
+
+from IPython.utils.warn import warn
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+def mutex_opts(dict,ex_op):
+    """Check for presence of mutually exclusive keys in a dict.
+
+    Call: mutex_opts(dict,[[op1a,op1b],[op2a,op2b]...]"""
+    for op1,op2 in ex_op:
+        if op1 in dict and op2 in dict:
+            raise ValueError,'\n*** ERROR in Arguments *** '\
+                  'Options '+op1+' and '+op2+' are mutually exclusive.'
+
+
+class EvalDict:
+    """
+    Emulate a dict which evaluates its contents in the caller's frame.
+
+    Usage:
+    >>> number = 19
+
+    >>> text = "python"
+
+    >>> print "%(text.capitalize())s %(number/9.0).1f rules!" % EvalDict()
+    Python 2.1 rules!
+    """
+
+    # This version is due to [e-mail] on c.l.py, and is basically a
+    # modified (shorter) version of:
+    # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66018 by
+    # Skip Montanaro ([e-mail]).
+
+    def __getitem__(self, name):
+        frame = sys._getframe(1)
+        return eval(name, frame.f_globals, frame.f_locals)
+
+EvalString = EvalDict  # for backwards compatibility
+
+
+def all_belong(candidates,checklist):
+    """Check whether a list of items ALL appear in a given list of options.
+
+    Returns a single 1 or 0 value."""
+
+    return 1-(0 in [x in checklist for x in candidates])
+
+
+def belong(candidates,checklist):
+    """Check whether a list of items appear in a given list of options.
+
+    Returns a list of 1 and 0, one for each candidate given."""
+
+    return [x in checklist for x in candidates]
+
+
+def with_obj(object, **args):
+    """Set multiple attributes for an object, similar to Pascal's with.
+
+    Example:
+    with_obj(jim,
+             born = 1960,
+             haircolour = 'Brown',
+             eyecolour = 'Green')
+
+    Credit: Greg Ewing, in
+    http://mail.python.org/pipermail/python-list/2001-May/040703.html.
+
+    NOTE: up until IPython 0.7.2, this was called simply 'with', but 'with'
+    has become a keyword for Python 2.5, so we had to rename it."""
+
+    object.__dict__.update(args)
+
+
+def map_method(method,object_list,*argseq,**kw):
+    """map_method(method,object_list,*args,**kw) -> list
+
+    Return a list of the results of applying the methods to the items of the
+    argument sequence(s).  If more than one sequence is given, the method is
+    called with an argument list consisting of the corresponding item of each
+    sequence. All sequences must be of the same length.
+
+    Keyword arguments are passed verbatim to all objects called.
+
+    This is Python code, so it's not nearly as fast as the builtin map()."""
+
+    out_list = []
+    idx = 0
+    for object in object_list:
+        try:
+            handler = getattr(object, method)
+        except AttributeError:
+            out_list.append(None)
+        else:
+            if argseq:
+                args = map(lambda lst:lst[idx],argseq)
+                #print 'ob',object,'hand',handler,'ar',args # dbg
+                out_list.append(handler(args,**kw))
+            else:
+                out_list.append(handler(**kw))
+        idx += 1
+    return out_list
+
+
+def import_fail_info(mod_name,fns=None):
+    """Inform load failure for a module."""
+
+    if fns == None:
+        warn("Loading of %s failed.\n" % (mod_name,))
+    else:
+        warn("Loading of %s from %s failed.\n" % (fns,mod_name))
+
+
+class NotGiven: pass
+
+def popkey(dct,key,default=NotGiven):
+    """Return dct[key] and delete dct[key].
+
+    If default is given, return it if dct[key] doesn't exist, otherwise raise
+    KeyError.  """
+
+    try:
+        val = dct[key]
+    except KeyError:
+        if default is NotGiven:
+            raise
+        else:
+            return default
+    else:
+        del dct[key]
+        return val
+
+
+def wrap_deprecated(func, suggest = '<nothing>'):
+    def newFunc(*args, **kwargs):
+        warnings.warn("Call to deprecated function %s, use %s instead" %
+                      ( func.__name__, suggest),
+                      category=DeprecationWarning,
+                      stacklevel = 2)
+        return func(*args, **kwargs)
+    return newFunc
+
+

IPython/utils/cursesimport.py

@@ -0,0 +1,31 b''
+# encoding: utf-8
+"""
+See if we have curses.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+# Curses and termios are Unix-only modules
+try:
+    import curses
+    # We need termios as well, so if its import happens to raise, we bail on
+    # using curses altogether.
+    import termios
+except ImportError:
+    use_curses = False
+else:
+    # Curses on Solaris may not be complete, so we can't use it there
+    use_curses = hasattr(curses,'initscr')
  No newline at end of file

IPython/utils/data.py

@@ -0,0 +1,106 b''
+# encoding: utf-8
+"""Utilities for working with data structures like lists, dicts and tuples.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import types
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def uniq_stable(elems):
+    """uniq_stable(elems) -> list
+
+    Return from an iterable, a list of all the unique elements in the input,
+    but maintaining the order in which they first appear.
+
+    A naive solution to this problem which just makes a dictionary with the
+    elements as keys fails to respect the stability condition, since
+    dictionaries are unsorted by nature.
+
+    Note: All elements in the input must be valid dictionary keys for this
+    routine to work, as it internally uses a dictionary for efficiency
+    reasons."""
+
+    unique = []
+    unique_dict = {}
+    for nn in elems:
+        if nn not in unique_dict:
+            unique.append(nn)
+            unique_dict[nn] = None
+    return unique
+
+
+def sort_compare(lst1, lst2, inplace=1):
+    """Sort and compare two lists.
+
+    By default it does it in place, thus modifying the lists. Use inplace = 0
+    to avoid that (at the cost of temporary copy creation)."""
+    if not inplace:
+        lst1 = lst1[:]
+        lst2 = lst2[:]
+    lst1.sort(); lst2.sort()
+    return lst1 == lst2
+
+
+def list2dict(lst):
+    """Takes a list of (key,value) pairs and turns it into a dict."""
+
+    dic = {}
+    for k,v in lst: dic[k] = v
+    return dic
+
+
+def list2dict2(lst, default=''):
+    """Takes a list and turns it into a dict.
+    Much slower than list2dict, but more versatile. This version can take
+    lists with sublists of arbitrary length (including sclars)."""
+
+    dic = {}
+    for elem in lst:
+        if type(elem) in (types.ListType,types.TupleType):
+            size = len(elem)
+            if  size == 0:
+                pass
+            elif size == 1:
+                dic[elem] = default
+            else:
+                k,v = elem[0], elem[1:]
+                if len(v) == 1: v = v[0]
+                dic[k] = v
+        else:
+            dic[elem] = default
+    return dic
+
+
+def flatten(seq):
+    """Flatten a list of lists (NOT recursive, only works for 2d lists)."""
+
+    return [x for subseq in seq for x in subseq]
+
+
+def get_slice(seq, start=0, stop=None, step=1):
+    """Get a slice of a sequence with variable step. Specify start,stop,step."""
+    if stop == None:
+        stop = len(seq)
+    item = lambda i: seq[i]
+    return map(item,xrange(start,stop,step))
+
+
+def chop(seq, size):
+    """Chop a sequence into chunks of the given size."""
+    chunk = lambda i: seq[i:i+size]
+    return map(chunk,xrange(0,len(seq),size))
+
+

IPython/utils/decorators.py

@@ -0,0 +1,46 b''
+# encoding: utf-8
+"""Decorators that don't go anywhere else.
+
+This module contains misc. decorators that don't really go with another module
+in :mod:`IPython.utils`. Beore putting something here please see if it should
+go into another topical module in :mod:`IPython.utils`.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def flag_calls(func):
+    """Wrap a function to detect and flag when it gets called.
+
+    This is a decorator which takes a function and wraps it in a function with
+    a 'called' attribute. wrapper.called is initialized to False.
+
+    The wrapper.called attribute is set to False right before each call to the
+    wrapped function, so if the call fails it remains False.  After the call
+    completes, wrapper.called is set to True and the output is returned.
+
+    Testing for truth in wrapper.called allows you to determine if a call to
+    func() was attempted and succeeded."""
+
+    def wrapper(*args,**kw):
+        wrapper.called = False
+        out = func(*args,**kw)
+        wrapper.called = True
+        return out
+
+    wrapper.called = False
+    wrapper.__doc__ = func.__doc__
+    return wrapper
+

IPython/utils/dir2.py

@@ -0,0 +1,82 b''
+# encoding: utf-8
+"""A fancy version of Python's builtin :func:`dir` function.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def get_class_members(cls):
+    ret = dir(cls)
+    if hasattr(cls,'__bases__'):
+        for base in cls.__bases__:
+            ret.extend(get_class_members(base))
+    return ret
+
+
+def dir2(obj):
+    """dir2(obj) -> list of strings
+
+    Extended version of the Python builtin dir(), which does a few extra
+    checks, and supports common objects with unusual internals that confuse
+    dir(), such as Traits and PyCrust.
+
+    This version is guaranteed to return only a list of true strings, whereas
+    dir() returns anything that objects inject into themselves, even if they
+    are later not really valid for attribute access (many extension libraries
+    have such bugs).
+    """
+
+    # Start building the attribute list via dir(), and then complete it
+    # with a few extra special-purpose calls.
+    words = dir(obj)
+
+    if hasattr(obj,'__class__'):
+        words.append('__class__')
+        words.extend(get_class_members(obj.__class__))
+    #if '__base__' in words: 1/0
+
+    # Some libraries (such as traits) may introduce duplicates, we want to
+    # track and clean this up if it happens
+    may_have_dupes = False
+
+    # this is the 'dir' function for objects with Enthought's traits
+    if hasattr(obj, 'trait_names'):
+        try:
+            words.extend(obj.trait_names())
+            may_have_dupes = True
+        except TypeError:
+            # This will happen if `obj` is a class and not an instance.
+            pass
+
+    # Support for PyCrust-style _getAttributeNames magic method.
+    if hasattr(obj, '_getAttributeNames'):
+        try:
+            words.extend(obj._getAttributeNames())
+            may_have_dupes = True
+        except TypeError:
+            # `obj` is a class and not an instance.  Ignore
+            # this error.
+            pass
+
+    if may_have_dupes:
+        # eliminate possible duplicates, as some traits may also
+        # appear as normal attributes in the dir() call.
+        words = list(set(words))
+        words.sort()
+
+    # filter out non-string attributes which may be stuffed by dir() calls
+    # and poor coding in third-party modules
+    return [w for w in words if isinstance(w, basestring)]
+

IPython/utils/doctestreload.py

@@ -0,0 +1,75 b''
+# encoding: utf-8
+"""
+A utility for handling the reloading of doctest.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def dhook_wrap(func,*a,**k):
+    """Wrap a function call in a sys.displayhook controller.
+
+    Returns a wrapper around func which calls func, with all its arguments and
+    keywords unmodified, using the default sys.displayhook.  Since IPython
+    modifies sys.displayhook, it breaks the behavior of certain systems that
+    rely on the default behavior, notably doctest.
+    """
+
+    def f(*a,**k):
+
+        dhook_s = sys.displayhook
+        sys.displayhook = sys.__displayhook__
+        try:
+            out = func(*a,**k)
+        finally:
+            sys.displayhook = dhook_s
+
+        return out
+
+    f.__doc__ = func.__doc__
+    return f
+
+
+def doctest_reload():
+    """Properly reload doctest to reuse it interactively.
+
+    This routine:
+
+      - imports doctest but does NOT reload it (see below).
+
+      - resets its global 'master' attribute to None, so that multiple uses of
+      the module interactively don't produce cumulative reports.
+
+      - Monkeypatches its core test runner method to protect it from IPython's
+      modified displayhook.  Doctest expects the default displayhook behavior
+      deep down, so our modification breaks it completely.  For this reason, a
+      hard monkeypatch seems like a reasonable solution rather than asking
+      users to manually use a different doctest runner when under IPython.
+
+    Notes
+    -----
+
+    This function *used to* reload doctest, but this has been disabled because
+    reloading doctest unconditionally can cause massive breakage of other
+    doctest-dependent modules already in memory, such as those for IPython's
+    own testing system.  The name wasn't changed to avoid breaking people's
+    code, but the reload call isn't actually made anymore."""
+
+    import doctest
+    doctest.master = None
+    doctest.DocTestRunner.run = dhook_wrap(doctest.DocTestRunner.run)
+

IPython/utils/frame.py

@@ -0,0 +1,85 b''
+# encoding: utf-8
+"""
+Utilities for working with stack frames.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def extract_vars(*names,**kw):
+    """Extract a set of variables by name from another frame.
+
+    :Parameters:
+      - `*names`: strings
+        One or more variable names which will be extracted from the caller's
+    frame.
+
+    :Keywords:
+      - `depth`: integer (0)
+        How many frames in the stack to walk when looking for your variables.
+
+
+    Examples:
+
+        In [2]: def func(x):
+           ...:     y = 1
+           ...:     print extract_vars('x','y')
+           ...:
+
+        In [3]: func('hello')
+        {'y': 1, 'x': 'hello'}
+    """
+
+    depth = kw.get('depth',0)
+    
+    callerNS = sys._getframe(depth+1).f_locals
+    return dict((k,callerNS[k]) for k in names)
+
+
+def extract_vars_above(*names):
+    """Extract a set of variables by name from another frame.
+
+    Similar to extractVars(), but with a specified depth of 1, so that names
+    are exctracted exactly from above the caller.
+
+    This is simply a convenience function so that the very common case (for us)
+    of skipping exactly 1 frame doesn't have to construct a special dict for
+    keyword passing."""
+
+    callerNS = sys._getframe(2).f_locals
+    return dict((k,callerNS[k]) for k in names)
+
+
+def debugx(expr,pre_msg=''):
+    """Print the value of an expression from the caller's frame.
+
+    Takes an expression, evaluates it in the caller's frame and prints both
+    the given expression and the resulting value (as well as a debug mark
+    indicating the name of the calling function.  The input must be of a form
+    suitable for eval().
+
+    An optional message can be passed, which will be prepended to the printed
+    expr->value pair."""
+
+    cf = sys._getframe(1)
+    print '[DBG:%s] %s%s -> %r' % (cf.f_code.co_name,pre_msg,expr,
+                                   eval(expr,cf.f_globals,cf.f_locals))
+
+
+# deactivate it by uncommenting the following line, which makes it a no-op
+#def debugx(expr,pre_msg=''): pass
+

IPython/utils/io.py

@@ -0,0 +1,292 b''
+# encoding: utf-8
+"""
+IO related utilities.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+import tempfile
+
+from IPython.external.Itpl import itpl, printpl
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+class IOStream:
+
+    def __init__(self,stream,fallback):
+        if not hasattr(stream,'write') or not hasattr(stream,'flush'):
+            stream = fallback
+        self.stream = stream
+        self._swrite = stream.write
+        self.flush = stream.flush
+
+    def write(self,data):
+        try:
+            self._swrite(data)
+        except:
+            try:
+                # print handles some unicode issues which may trip a plain
+                # write() call.  Attempt to emulate write() by using a
+                # trailing comma
+                print >> self.stream, data,
+            except:
+                # if we get here, something is seriously broken.
+                print >> sys.stderr, \
+                      'ERROR - failed to write data to stream:', self.stream
+
+    # This class used to have a writeln method, but regular files and streams
+    # in Python don't have this method. We need to keep this completely
+    # compatible so we removed it.
+
+    def close(self):
+        pass
+
+
+class IOTerm:
+    """ Term holds the file or file-like objects for handling I/O operations.
+
+    These are normally just sys.stdin, sys.stdout and sys.stderr but for
+    Windows they can can replaced to allow editing the strings before they are
+    displayed."""
+
+    # In the future, having IPython channel all its I/O operations through
+    # this class will make it easier to embed it into other environments which
+    # are not a normal terminal (such as a GUI-based shell)
+    def __init__(self,cin=None,cout=None,cerr=None):
+        self.cin  = IOStream(cin,sys.stdin)
+        self.cout = IOStream(cout,sys.stdout)
+        self.cerr = IOStream(cerr,sys.stderr)
+
+
+# Global variable to be used for all I/O
+Term = IOTerm()
+
+
+import IPython.utils.rlineimpl as readline
+# Remake Term to use the readline i/o facilities
+if sys.platform == 'win32' and readline.have_readline:
+
+    Term = IOTerm(cout=readline._outputfile,cerr=readline._outputfile)
+
+
+class Tee(object):
+    """A class to duplicate an output stream to stdout/err.
+
+    This works in a manner very similar to the Unix 'tee' command.
+
+    When the object is closed or deleted, it closes the original file given to
+    it for duplication.
+    """
+    # Inspired by:
+    # http://mail.python.org/pipermail/python-list/2007-May/442737.html
+
+    def __init__(self, file_or_name, mode=None, channel='stdout'):
+        """Construct a new Tee object.
+
+        Parameters
+        ----------
+        file_or_name : filename or open filehandle (writable)
+          File that will be duplicated
+
+        mode : optional, valid mode for open().
+          If a filename was give, open with this mode.
+
+        channel : str, one of ['stdout', 'stderr']  
+        """
+        if channel not in ['stdout', 'stderr']:
+            raise ValueError('Invalid channel spec %s' % channel)
+        
+        if hasattr(file, 'write') and hasattr(file, 'seek'):
+            self.file = file_or_name
+        else:
+            self.file = open(file_or_name, mode)
+        self.channel = channel
+        self.ostream = getattr(sys, channel)
+        setattr(sys, channel, self)
+        self._closed = False
+
+    def close(self):
+        """Close the file and restore the channel."""
+        self.flush()
+        setattr(sys, self.channel, self.ostream)
+        self.file.close()
+        self._closed = True
+
+    def write(self, data):
+        """Write data to both channels."""
+        self.file.write(data)
+        self.ostream.write(data)
+        self.ostream.flush()
+
+    def flush(self):
+        """Flush both channels."""
+        self.file.flush()
+        self.ostream.flush()
+
+    def __del__(self):
+        if not self._closed:
+            self.close()
+
+
+def file_read(filename):
+    """Read a file and close it.  Returns the file source."""
+    fobj = open(filename,'r');
+    source = fobj.read();
+    fobj.close()
+    return source
+
+
+def file_readlines(filename):
+    """Read a file and close it.  Returns the file source using readlines()."""
+    fobj = open(filename,'r');
+    lines = fobj.readlines();
+    fobj.close()
+    return lines
+
+
+def raw_input_multi(header='', ps1='==> ', ps2='..> ',terminate_str = '.'):
+    """Take multiple lines of input.
+
+    A list with each line of input as a separate element is returned when a
+    termination string is entered (defaults to a single '.'). Input can also
+    terminate via EOF (^D in Unix, ^Z-RET in Windows).
+
+    Lines of input which end in \\ are joined into single entries (and a
+    secondary continuation prompt is issued as long as the user terminates
+    lines with \\). This allows entering very long strings which are still
+    meant to be treated as single entities.
+    """
+
+    try:
+        if header:
+            header += '\n'
+        lines = [raw_input(header + ps1)]
+    except EOFError:
+        return []
+    terminate = [terminate_str]
+    try:
+        while lines[-1:] != terminate:
+            new_line = raw_input(ps1)
+            while new_line.endswith('\\'):
+                new_line = new_line[:-1] + raw_input(ps2)
+            lines.append(new_line)
+
+        return lines[:-1]  # don't return the termination command
+    except EOFError:
+        print
+        return lines
+
+
+def raw_input_ext(prompt='',  ps2='... '):
+    """Similar to raw_input(), but accepts extended lines if input ends with \\."""
+
+    line = raw_input(prompt)
+    while line.endswith('\\'):
+        line = line[:-1] + raw_input(ps2)
+    return line
+
+
+def ask_yes_no(prompt,default=None):
+    """Asks a question and returns a boolean (y/n) answer.
+
+    If default is given (one of 'y','n'), it is used if the user input is
+    empty. Otherwise the question is repeated until an answer is given.
+
+    An EOF is treated as the default answer.  If there is no default, an
+    exception is raised to prevent infinite loops.
+
+    Valid answers are: y/yes/n/no (match is not case sensitive)."""
+
+    answers = {'y':True,'n':False,'yes':True,'no':False}
+    ans = None
+    while ans not in answers.keys():
+        try:
+            ans = raw_input(prompt+' ').lower()
+            if not ans:  # response was an empty string
+                ans = default
+        except KeyboardInterrupt:
+            pass
+        except EOFError:
+            if default in answers.keys():
+                ans = default
+                print
+            else:
+                raise
+
+    return answers[ans]
+
+
+class NLprinter:
+    """Print an arbitrarily nested list, indicating index numbers.
+
+    An instance of this class called nlprint is available and callable as a
+    function.
+
+    nlprint(list,indent=' ',sep=': ') -> prints indenting each level by 'indent'
+    and using 'sep' to separate the index from the value. """
+
+    def __init__(self):
+        self.depth = 0
+
+    def __call__(self,lst,pos='',**kw):
+        """Prints the nested list numbering levels."""
+        kw.setdefault('indent',' ')
+        kw.setdefault('sep',': ')
+        kw.setdefault('start',0)
+        kw.setdefault('stop',len(lst))
+        # we need to remove start and stop from kw so they don't propagate
+        # into a recursive call for a nested list.
+        start = kw['start']; del kw['start']
+        stop = kw['stop']; del kw['stop']
+        if self.depth == 0 and 'header' in kw.keys():
+            print kw['header']
+
+        for idx in range(start,stop):
+            elem = lst[idx]
+            if type(elem)==type([]):
+                self.depth += 1
+                self.__call__(elem,itpl('$pos$idx,'),**kw)
+                self.depth -= 1
+            else:
+                printpl(kw['indent']*self.depth+'$pos$idx$kw["sep"]$elem')
+
+nlprint = NLprinter()
+
+
+def temp_pyfile(src, ext='.py'):
+    """Make a temporary python file, return filename and filehandle.
+
+    Parameters
+    ----------
+    src : string or list of strings (no need for ending newlines if list)
+      Source code to be written to the file.
+
+    ext : optional, string
+      Extension for the generated file.
+
+    Returns
+    -------
+    (filename, open filehandle)
+      It is the caller's responsibility to close the open file and unlink it.
+    """
+    fname = tempfile.mkstemp(ext)[1]
+    f = open(fname,'w')
+    f.write(src)
+    f.flush()
+    return fname, f
+
+
+

IPython/utils/path.py

@@ -0,0 +1,345 b''
+# encoding: utf-8
+"""
+Utilities for path handling.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import sys
+
+import IPython
+from IPython.utils.process import xsys
+from IPython.utils.importstring import import_item
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+def _get_long_path_name(path):
+    """Dummy no-op."""
+    return path
+
+
+if sys.platform == 'win32':
+    def _get_long_path_name(path):
+        """Get a long path name (expand ~) on Windows using ctypes.
+
+        Examples
+        --------
+
+        >>> get_long_path_name('c:\\docume~1')
+        u'c:\\\\Documents and Settings'
+
+        """
+        try:
+            import ctypes
+        except ImportError:
+            raise ImportError('you need to have ctypes installed for this to work')
+        _GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
+        _GetLongPathName.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p,
+            ctypes.c_uint ]
+
+        buf = ctypes.create_unicode_buffer(260)
+        rv = _GetLongPathName(path, buf, 260)
+        if rv == 0 or rv > 260:
+            return path
+        else:
+            return buf.value
+
+
+def get_long_path_name(path):
+    """Expand a path into its long form.
+
+    On Windows this expands any ~ in the paths. On other platforms, it is
+    a null operation.
+    """
+    return _get_long_path_name(path)
+
+
+def get_py_filename(name):
+    """Return a valid python filename in the current directory.
+
+    If the given name is not a file, it adds '.py' and searches again.
+    Raises IOError with an informative message if the file isn't found."""
+
+    name = os.path.expanduser(name)
+    if not os.path.isfile(name) and not name.endswith('.py'):
+        name += '.py'
+    if os.path.isfile(name):
+        return name
+    else:
+        raise IOError,'File `%s` not found.' % name
+
+
+def filefind(filename, path_dirs=None):
+    """Find a file by looking through a sequence of paths.
+
+    This iterates through a sequence of paths looking for a file and returns
+    the full, absolute path of the first occurence of the file.  If no set of
+    path dirs is given, the filename is tested as is, after running through
+    :func:`expandvars` and :func:`expanduser`.  Thus a simple call::
+
+        filefind('myfile.txt')
+
+    will find the file in the current working dir, but::
+
+        filefind('~/myfile.txt')
+
+    Will find the file in the users home directory.  This function does not
+    automatically try any paths, such as the cwd or the user's home directory.
+    
+    Parameters
+    ----------
+    filename : str
+        The filename to look for.
+    path_dirs : str, None or sequence of str
+        The sequence of paths to look for the file in.  If None, the filename
+        need to be absolute or be in the cwd.  If a string, the string is
+        put into a sequence and the searched.  If a sequence, walk through
+        each element and join with ``filename``, calling :func:`expandvars`
+        and :func:`expanduser` before testing for existence.
+        
+    Returns
+    -------
+    Raises :exc:`IOError` or returns absolute path to file.
+    """
+    
+    # If paths are quoted, abspath gets confused, strip them...
+    filename = filename.strip('"').strip("'")
+    # If the input is an absolute path, just check it exists
+    if os.path.isabs(filename) and os.path.isfile(filename):
+        return filename
+        
+    if path_dirs is None:
+        path_dirs = ("",)
+    elif isinstance(path_dirs, basestring):
+        path_dirs = (path_dirs,)
+        
+    for path in path_dirs:
+        if path == '.': path = os.getcwd()
+        testname = expand_path(os.path.join(path, filename))
+        if os.path.isfile(testname):
+            return os.path.abspath(testname)
+        
+    raise IOError("File %r does not exist in any of the search paths: %r" % 
+                  (filename, path_dirs) )
+
+
+class HomeDirError(Exception):
+    pass
+
+
+def get_home_dir():
+    """Return the closest possible equivalent to a 'home' directory.
+
+    * On POSIX, we try $HOME.
+    * On Windows we try:
+      - %HOMESHARE%
+      - %HOMEDRIVE\%HOMEPATH%
+      - %USERPROFILE%
+      - Registry hack for My Documents
+      - %HOME%: rare, but some people with unix-like setups may have defined it
+    * On Dos C:\
+ 
+    Currently only Posix and NT are implemented, a HomeDirError exception is
+    raised for all other OSes.
+    """
+
+    isdir = os.path.isdir
+    env = os.environ
+
+    # first, check py2exe distribution root directory for _ipython.
+    # This overrides all. Normally does not exist.
+
+    if hasattr(sys, "frozen"): #Is frozen by py2exe
+        if '\\library.zip\\' in IPython.__file__.lower():#libraries compressed to zip-file
+            root, rest = IPython.__file__.lower().split('library.zip')
+        else: 
+            root=os.path.join(os.path.split(IPython.__file__)[0],"../../")
+        root=os.path.abspath(root).rstrip('\\')
+        if isdir(os.path.join(root, '_ipython')):
+            os.environ["IPYKITROOT"] = root
+        return root.decode(sys.getfilesystemencoding())
+
+    if os.name == 'posix':
+        # Linux, Unix, AIX, OS X
+        try:
+            homedir = env['HOME']
+        except KeyError:
+            raise HomeDirError('Undefined $HOME, IPython cannot proceed.')
+        else:
+            return homedir.decode(sys.getfilesystemencoding())
+    elif os.name == 'nt':
+        # Now for win9x, XP, Vista, 7?
+        # For some strange reason all of these return 'nt' for os.name.
+        # First look for a network home directory. This will return the UNC
+        # path (\\server\\Users\%username%) not the mapped path (Z:\). This
+        # is needed when running IPython on cluster where all paths have to 
+        # be UNC.
+        try:
+            homedir = env['HOMESHARE']
+        except KeyError:
+            pass
+        else:
+            if isdir(homedir):
+                return homedir.decode(sys.getfilesystemencoding())
+
+        # Now look for a local home directory
+        try:
+            homedir = os.path.join(env['HOMEDRIVE'],env['HOMEPATH'])
+        except KeyError:
+            pass
+        else:
+            if isdir(homedir):
+                return homedir.decode(sys.getfilesystemencoding())
+
+        # Now the users profile directory
+        try:
+            homedir = os.path.join(env['USERPROFILE'])
+        except KeyError:
+            pass
+        else:
+            if isdir(homedir):
+                return homedir.decode(sys.getfilesystemencoding())
+
+        # Use the registry to get the 'My Documents' folder.
+        try:
+            import _winreg as wreg
+            key = wreg.OpenKey(
+                wreg.HKEY_CURRENT_USER,
+                "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
+            )
+            homedir = wreg.QueryValueEx(key,'Personal')[0]
+            key.Close()
+        except:
+            pass
+        else:
+            if isdir(homedir):
+                return homedir.decode(sys.getfilesystemencoding())
+
+        # A user with a lot of unix tools in win32 may have defined $HOME.
+        # Try this as a last ditch option.
+        try:
+            homedir = env['HOME']
+        except KeyError:
+            pass
+        else:
+            if isdir(homedir):
+                return homedir.decode(sys.getfilesystemencoding())
+
+        # If all else fails, raise HomeDirError
+        raise HomeDirError('No valid home directory could be found')
+    elif os.name == 'dos':
+        # Desperate, may do absurd things in classic MacOS. May work under DOS.
+        return 'C:\\'.decode(sys.getfilesystemencoding())
+    else:
+        raise HomeDirError('No valid home directory could be found for your OS')
+
+
+def get_ipython_dir():
+    """Get the IPython directory for this platform and user.
+    
+    This uses the logic in `get_home_dir` to find the home directory
+    and the adds .ipython to the end of the path.
+    """
+    ipdir_def = '.ipython'
+    home_dir = get_home_dir()
+    # import pdb; pdb.set_trace()  # dbg
+    ipdir = os.environ.get(
+        'IPYTHON_DIR', os.environ.get(
+            'IPYTHONDIR', os.path.join(home_dir, ipdir_def)
+        )
+    )
+    return ipdir.decode(sys.getfilesystemencoding())
+
+
+def get_ipython_package_dir():
+    """Get the base directory where IPython itself is installed."""
+    ipdir = os.path.dirname(IPython.__file__)
+    return ipdir.decode(sys.getfilesystemencoding())
+
+
+def get_ipython_module_path(module_str):
+    """Find the path to an IPython module in this version of IPython.
+
+    This will always find the version of the module that is in this importable
+    IPython package. This will always return the path to the ``.py``
+    version of the module.
+    """
+    if module_str == 'IPython':
+        return os.path.join(get_ipython_package_dir(), '__init__.py')
+    mod = import_item(module_str)
+    the_path = mod.__file__.replace('.pyc', '.py')
+    the_path = the_path.replace('.pyo', '.py')
+    return the_path.decode(sys.getfilesystemencoding())
+
+
+def expand_path(s):
+    """Expand $VARS and ~names in a string, like a shell
+
+    :Examples:
+    
+       In [2]: os.environ['FOO']='test'
+
+       In [3]: expand_path('variable FOO is $FOO')
+       Out[3]: 'variable FOO is test'
+    """
+    # This is a pretty subtle hack. When expand user is given a UNC path
+    # on Windows (\\server\share$\%username%), os.path.expandvars, removes
+    # the $ to get (\\server\share\%username%). I think it considered $
+    # alone an empty var. But, we need the $ to remains there (it indicates
+    # a hidden share).
+    if os.name=='nt':
+        s = s.replace('$\\', 'IPYTHON_TEMP')
+    s = os.path.expandvars(os.path.expanduser(s))
+    if os.name=='nt':
+        s = s.replace('IPYTHON_TEMP', '$\\')
+    return s
+
+
+def target_outdated(target,deps):
+    """Determine whether a target is out of date.
+
+    target_outdated(target,deps) -> 1/0
+
+    deps: list of filenames which MUST exist.
+    target: single filename which may or may not exist.
+
+    If target doesn't exist or is older than any file listed in deps, return
+    true, otherwise return false.
+    """
+    try:
+        target_time = os.path.getmtime(target)
+    except os.error:
+        return 1
+    for dep in deps:
+        dep_time = os.path.getmtime(dep)
+        if dep_time > target_time:
+            #print "For target",target,"Dep failed:",dep # dbg
+            #print "times (dep,tar):",dep_time,target_time # dbg
+            return 1
+    return 0
+
+
+def target_update(target,deps,cmd):
+    """Update a target with a given command given a list of dependencies.
+
+    target_update(target,deps,cmd) -> runs cmd if target is outdated.
+
+    This is just a wrapper around target_outdated() which calls the given
+    command if target is outdated."""
+
+    if target_outdated(target,deps):
+        xsys(cmd)
+

IPython/utils/process.py

@@ -0,0 +1,368 b''
+# encoding: utf-8
+"""
+Utilities for working with external processes.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import sys
+import shlex
+import subprocess
+
+from IPython.utils.terminal import set_term_title
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+class FindCmdError(Exception):
+    pass
+
+
+def _find_cmd(cmd):
+    """Find the full path to a command using which."""
+    return os.popen('which %s' % cmd).read().strip()
+
+
+if os.name == 'posix':
+    def _find_cmd(cmd):
+        """Find the full path to a command using which."""
+        return getoutputerror('/usr/bin/env which %s' % cmd)[0]
+
+
+if sys.platform == 'win32':
+    def _find_cmd(cmd):
+        """Find the full path to a .bat or .exe using the win32api module."""
+        try:
+            from win32api import SearchPath
+        except ImportError:
+            raise ImportError('you need to have pywin32 installed for this to work')
+        else:
+            PATH = os.environ['PATH']
+            extensions = ['.exe', '.com', '.bat', '.py']
+            path = None
+            for ext in extensions:
+                try:
+                    path = SearchPath(PATH,cmd + ext)[0]
+                except:
+                    pass
+            if path is None:
+                raise OSError("command %r not found" % cmd)
+            else:
+                return path
+
+
+def find_cmd(cmd):
+    """Find absolute path to executable cmd in a cross platform manner.
+    
+    This function tries to determine the full path to a command line program
+    using `which` on Unix/Linux/OS X and `win32api` on Windows.  Most of the
+    time it will use the version that is first on the users `PATH`.  If
+    cmd is `python` return `sys.executable`.
+
+    Warning, don't use this to find IPython command line programs as there
+    is a risk you will find the wrong one.  Instead find those using the
+    following code and looking for the application itself::
+    
+        from IPython.utils.path import get_ipython_module_path
+        from IPython.utils.process import pycmd2argv
+        argv = pycmd2argv(get_ipython_module_path('IPython.core.ipapp'))
+
+    Parameters
+    ----------
+    cmd : str
+        The command line program to look for.
+    """
+    if cmd == 'python':
+        return os.path.abspath(sys.executable)
+    try:
+        path = _find_cmd(cmd)
+    except OSError:
+        raise FindCmdError('command could not be found: %s' % cmd)
+    # which returns empty if not found
+    if path == '':
+        raise FindCmdError('command could not be found: %s' % cmd)
+    return os.path.abspath(path)
+
+
+def pycmd2argv(cmd):
+    r"""Take the path of a python command and return a list (argv-style).
+
+    This only works on Python based command line programs and will find the
+    location of the ``python`` executable using ``sys.executable`` to make
+    sure the right version is used.
+
+    For a given path ``cmd``, this returns [cmd] if cmd's extension is .exe,
+    .com or .bat, and [, cmd] otherwise.  
+
+    Parameters
+    ----------
+    cmd : string
+      The path of the command.
+
+    Returns
+    -------
+    argv-style list.
+    """
+    ext = os.path.splitext(cmd)[1]
+    if ext in ['.exe', '.com', '.bat']:
+        return [cmd]
+    else:
+        if sys.platform == 'win32':
+            # The -u option here turns on unbuffered output, which is required
+            # on Win32 to prevent wierd conflict and problems with Twisted.
+            # Also, use sys.executable to make sure we are picking up the 
+            # right python exe.
+            return [sys.executable, '-u', cmd]
+        else:
+            return [sys.executable, cmd]
+
+
+def arg_split(s, posix=False):
+    """Split a command line's arguments in a shell-like manner.
+
+    This is a modified version of the standard library's shlex.split()
+    function, but with a default of posix=False for splitting, so that quotes
+    in inputs are respected."""
+
+    # XXX - there may be unicode-related problems here!!!  I'm not sure that
+    # shlex is truly unicode-safe, so it might be necessary to do
+    #
+    # s = s.encode(sys.stdin.encoding)
+    #
+    # first, to ensure that shlex gets a normal string.  Input from anyone who
+    # knows more about unicode and shlex than I would be good to have here...
+    lex = shlex.shlex(s, posix=posix)
+    lex.whitespace_split = True
+    return list(lex)
+
+
+def system(cmd, verbose=0, debug=0, header=''):
+    """Execute a system command, return its exit status.
+
+    Options:
+
+    - verbose (0): print the command to be executed.
+
+    - debug (0): only print, do not actually execute.
+
+    - header (''): Header to print on screen prior to the executed command (it
+    is only prepended to the command, no newlines are added).
+
+    Note: a stateful version of this function is available through the
+    SystemExec class."""
+
+    stat = 0
+    if verbose or debug: print header+cmd
+    sys.stdout.flush()
+    if not debug: stat = os.system(cmd)
+    return stat
+
+
+def abbrev_cwd():
+    """ Return abbreviated version of cwd, e.g. d:mydir """
+    cwd = os.getcwd().replace('\\','/')
+    drivepart = ''
+    tail = cwd
+    if sys.platform == 'win32':
+        if len(cwd) < 4:
+            return cwd
+        drivepart,tail = os.path.splitdrive(cwd)
+
+
+    parts = tail.split('/')
+    if len(parts) > 2:
+        tail = '/'.join(parts[-2:])
+
+    return (drivepart + (
+        cwd == '/' and '/' or tail))
+
+
+# This function is used by ipython in a lot of places to make system calls.
+# We need it to be slightly different under win32, due to the vagaries of
+# 'network shares'.  A win32 override is below.
+
+def shell(cmd, verbose=0, debug=0, header=''):
+    """Execute a command in the system shell, always return None.
+
+    Options:
+
+    - verbose (0): print the command to be executed.
+
+    - debug (0): only print, do not actually execute.
+
+    - header (''): Header to print on screen prior to the executed command (it
+    is only prepended to the command, no newlines are added).
+
+    Note: this is similar to system(), but it returns None so it can
+    be conveniently used in interactive loops without getting the return value
+    (typically 0) printed many times."""
+
+    stat = 0
+    if verbose or debug: print header+cmd
+    # flush stdout so we don't mangle python's buffering
+    sys.stdout.flush()
+
+    if not debug:
+        set_term_title("IPy " + cmd)
+        os.system(cmd)
+        set_term_title("IPy " + abbrev_cwd())
+
+# override shell() for win32 to deal with network shares
+if os.name in ('nt','dos'):
+
+    shell_ori = shell
+
+    def shell(cmd, verbose=0, debug=0, header=''):
+        if os.getcwd().startswith(r"\\"):
+            path = os.getcwd()
+            # change to c drive (cannot be on UNC-share when issuing os.system,
+            # as cmd.exe cannot handle UNC addresses)
+            os.chdir("c:")
+            # issue pushd to the UNC-share and then run the command
+            try:
+                shell_ori('"pushd %s&&"'%path+cmd,verbose,debug,header)
+            finally:
+                os.chdir(path)
+        else:
+            shell_ori(cmd,verbose,debug,header)
+
+    shell.__doc__ = shell_ori.__doc__
+
+    
+def getoutput(cmd, verbose=0, debug=0, header='', split=0):
+    """Dummy substitute for perl's backquotes.
+
+    Executes a command and returns the output.
+
+    Accepts the same arguments as system(), plus:
+
+    - split(0): if true, the output is returned as a list split on newlines.
+
+    Note: a stateful version of this function is available through the
+    SystemExec class.
+
+    This is pretty much deprecated and rarely used, getoutputerror may be 
+    what you need.
+
+    """
+
+    if verbose or debug: print header+cmd
+    if not debug:
+        pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout
+        output = pipe.read()
+        # stipping last \n is here for backwards compat.
+        if output.endswith('\n'):
+            output = output[:-1]
+        if split:
+            return output.split('\n')
+        else:
+            return output
+
+
+# for compatibility with older naming conventions
+xsys = system
+
+
+def getoutputerror(cmd, verbose=0, debug=0, header='', split=0):
+    """Return (standard output,standard error) of executing cmd in a shell.
+
+    Accepts the same arguments as system(), plus:
+
+    - split(0): if true, each of stdout/err is returned as a list split on
+    newlines.
+
+    Note: a stateful version of this function is available through the
+    SystemExec class."""
+
+    if verbose or debug: print header+cmd
+    if not cmd:
+        if split:
+            return [],[]
+        else:
+            return '',''
+    if not debug:
+        p = subprocess.Popen(cmd, shell=True,
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE,
+                             close_fds=True)
+        pin, pout, perr = (p.stdin, p.stdout, p.stderr)
+
+        tout = pout.read().rstrip()
+        terr = perr.read().rstrip()
+        pin.close()
+        pout.close()
+        perr.close()
+        if split:
+            return tout.split('\n'),terr.split('\n')
+        else:
+            return tout,terr
+
+
+class SystemExec:
+    """Access the system and getoutput functions through a stateful interface.
+
+    Note: here we refer to the system and getoutput functions from this
+    library, not the ones from the standard python library.
+
+    This class offers the system and getoutput functions as methods, but the
+    verbose, debug and header parameters can be set for the instance (at
+    creation time or later) so that they don't need to be specified on each
+    call.
+
+    For efficiency reasons, there's no way to override the parameters on a
+    per-call basis other than by setting instance attributes. If you need
+    local overrides, it's best to directly call system() or getoutput().
+
+    The following names are provided as alternate options:
+     - xsys: alias to system
+     - bq: alias to getoutput
+
+    An instance can then be created as:
+    >>> sysexec = SystemExec(verbose=1,debug=0,header='Calling: ')
+    """
+
+    def __init__(self, verbose=0, debug=0, header='', split=0):
+        """Specify the instance's values for verbose, debug and header."""
+        self.verbose = verbose
+        self.debug = debug
+        self.header = header
+        self.split = split
+
+    def system(self, cmd):
+        """Stateful interface to system(), with the same keyword parameters."""
+
+        system(cmd, self.verbose, self.debug, self.header)
+
+    def shell(self, cmd):
+        """Stateful interface to shell(), with the same keyword parameters."""
+
+        shell(cmd, self.verbose, self.debug, self.header)
+
+    xsys = system  # alias
+
+    def getoutput(self, cmd):
+        """Stateful interface to getoutput()."""
+
+        return getoutput(cmd, self.verbose, self.debug, self.header, self.split)
+
+    def getoutputerror(self, cmd):
+        """Stateful interface to getoutputerror()."""
+
+        return getoutputerror(cmd, self.verbose, self.debug, self.header, self.split)
+
+    bq = getoutput  # alias
+
+

IPython/utils/sysinfo.py

@@ -0,0 +1,100 b''
+# encoding: utf-8
+"""
+Utilities for getting information about a system.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import platform
+import sys
+import subprocess
+
+from IPython.core import release
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def sys_info():
+    """Return useful information about IPython and the system, as a string.
+
+    Examples
+    --------
+    In [1]: print(sys_info())
+    IPython version: 0.11.bzr.r1340   # random
+    BZR revision   : 1340
+    Platform info  : os.name -> posix, sys.platform -> linux2
+                   : Linux-2.6.31-17-generic-i686-with-Ubuntu-9.10-karmic
+    Python info    : 2.6.4 (r264:75706, Dec  7 2009, 18:45:15) 
+    [GCC 4.4.1]
+    """
+    out = []
+    out.append('IPython version: %s' % release.version)
+    out.append('BZR revision   : %s' % release.revision)
+    out.append('Platform info  : os.name -> %s, sys.platform -> %s' %
+               (os.name,sys.platform) )
+    out.append('               : %s' % platform.platform())
+    out.append('Python info    : %s' % sys.version)
+    out.append('')  # ensure closing newline
+    return '\n'.join(out)
+
+
+def _num_cpus_unix():
+    """Return the number of active CPUs on a Unix system."""
+    return os.sysconf("SC_NPROCESSORS_ONLN")
+
+
+def _num_cpus_darwin():
+    """Return the number of active CPUs on a Darwin system."""
+    p = subprocess.Popen(['sysctl','-n','hw.ncpu'],stdout=subprocess.PIPE)
+    return p.stdout.read()
+
+
+def _num_cpus_windows():
+    """Return the number of active CPUs on a Windows system."""
+    return os.environ.get("NUMBER_OF_PROCESSORS")
+
+
+def num_cpus():
+   """Return the effective number of CPUs in the system as an integer.
+
+   This cross-platform function makes an attempt at finding the total number of
+   available CPUs in the system, as returned by various underlying system and
+   python calls.
+
+   If it can't find a sensible answer, it returns 1 (though an error *may* make
+   it return a large positive number that's actually incorrect).
+   """
+
+   # Many thanks to the Parallel Python project (http://www.parallelpython.com)
+   # for the names of the keys we needed to look up for this function.  This
+   # code was inspired by their equivalent function.
+
+   ncpufuncs = {'Linux':_num_cpus_unix,
+                'Darwin':_num_cpus_darwin,
+                'Windows':_num_cpus_windows,
+                # On Vista, python < 2.5.2 has a bug and returns 'Microsoft'
+                # See http://bugs.python.org/issue1082 for details.
+                'Microsoft':_num_cpus_windows,
+                }
+
+   ncpufunc = ncpufuncs.get(platform.system(),
+                            # default to unix version (Solaris, AIX, etc)
+                            _num_cpus_unix)
+
+   try:
+       ncpus = max(1,int(ncpufunc()))
+   except:
+       ncpus = 1
+   return ncpus
+

IPython/utils/terminal.py

@@ -0,0 +1,162 b''
+# encoding: utf-8
+"""
+Utilities for working with terminals.
+
+Authors:
+
+* Brian E. Granger
+* Fernando Perez
+* Alexander Belchenko (e-mail: bialix AT ukr.net)
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import struct
+import sys
+import warnings
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+# This variable is part of the expected API of the module:
+ignore_termtitle = True
+
+
+def _term_clear():
+    pass
+
+
+if os.name == 'posix':
+    def _term_clear():
+        os.system('clear')
+
+
+if sys.platform == 'win32':
+    def _term_clear():
+        os.system('cls')
+
+
+def term_clear():
+    _term_clear()
+
+
+def toggle_set_term_title(val):
+    """Control whether set_term_title is active or not.
+
+    set_term_title() allows writing to the console titlebar.  In embedded
+    widgets this can cause problems, so this call can be used to toggle it on
+    or off as needed.
+
+    The default state of the module is for the function to be disabled.
+
+    Parameters
+    ----------
+      val : bool
+        If True, set_term_title() actually writes to the terminal (using the
+        appropriate platform-specific module).  If False, it is a no-op.
+    """
+    global ignore_termtitle
+    ignore_termtitle = not(val)
+
+
+def _set_term_title(*args,**kw):
+    """Dummy no-op."""
+    pass
+
+
+def _set_term_title_xterm(title):
+    """ Change virtual terminal title in xterm-workalikes """
+    sys.stdout.write('\033]0;%s\007' % title)
+
+if os.name == 'posix':
+    TERM = os.environ.get('TERM','')
+    if (TERM == 'xterm') or (TERM == 'xterm-color'):
+        _set_term_title = _set_term_title_xterm
+
+
+if sys.platform == 'win32':
+    try:
+        import ctypes
+
+        SetConsoleTitleW = ctypes.windll.kernel32.SetConsoleTitleW
+        SetConsoleTitleW.argtypes = [ctypes.c_wchar_p]
+    
+        def _set_term_title(title):
+            """Set terminal title using ctypes to access the Win32 APIs."""
+            SetConsoleTitleW(title)
+    except ImportError:
+        def _set_term_title(title):
+            """Set terminal title using the 'title' command."""
+            global ignore_termtitle
+
+            try:
+                # Cannot be on network share when issuing system commands
+                curr = os.getcwd()
+                os.chdir("C:")
+                ret = os.system("title " + title)
+            finally:
+                os.chdir(curr)
+            if ret:
+                # non-zero return code signals error, don't try again
+                ignore_termtitle = True
+
+
+def set_term_title(title):
+    """Set terminal title using the necessary platform-dependent calls."""
+    if ignore_termtitle:
+        return
+    _set_term_title(title)
+
+
+def freeze_term_title():
+    warnings.warn("This function is deprecated, use toggle_set_term_title()")
+    global ignore_termtitle
+    ignore_termtitle = True
+
+
+def get_terminal_size(defaultx=80, defaulty=25):
+    return defaultx, defaulty
+
+
+if sys.platform == 'win32':
+    def get_terminal_size(defaultx=80, defaulty=25):
+        """Return size of current terminal console.
+
+        This function try to determine actual size of current working
+        console window and return tuple (sizex, sizey) if success,
+        or default size (defaultx, defaulty) otherwise.
+
+        Dependencies: ctypes should be installed.
+
+        Author: Alexander Belchenko (e-mail: bialix AT ukr.net)
+        """
+        try:
+            import ctypes
+        except ImportError:
+            return defaultx, defaulty
+
+        h = ctypes.windll.kernel32.GetStdHandle(-11)
+        csbi = ctypes.create_string_buffer(22)
+        res = ctypes.windll.kernel32.GetConsoleScreenBufferInfo(h, csbi)
+
+        if res:
+            (bufx, bufy, curx, cury, wattr,
+             left, top, right, bottom, maxx, maxy) = struct.unpack(
+                "hhhhHhhhhhh", csbi.raw)
+            sizex = right - left + 1
+            sizey = bottom - top + 1
+            return (sizex, sizey)
+        else:
+            return (defaultx, defaulty)
+

IPython/utils/tests/test_io.py

@@ -0,0 +1,61 b''
+# encoding: utf-8
+"""Tests for io.py"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+
+from cStringIO import StringIO
+
+import nose.tools as nt
+
+from IPython.testing import decorators as dec
+from IPython.utils.io import Tee
+
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
+
+
+def test_tee_simple():
+    "Very simple check with stdout only"
+    chan = StringIO()
+    text = 'Hello'
+    tee = Tee(chan, channel='stdout')
+    print >> chan, text,
+    nt.assert_equal(chan.getvalue(), text)
+
+
+class TeeTestCase(dec.ParametricTestCase):
+
+    def tchan(self, channel, check='close'):
+        trap = StringIO()
+        chan = StringIO()
+        text = 'Hello'
+        
+        std_ori = getattr(sys, channel)
+        setattr(sys, channel, trap)
+
+        tee = Tee(chan, channel=channel)
+        print >> chan, text,
+        setattr(sys, channel, std_ori)
+        trap_val = trap.getvalue()
+        nt.assert_equals(chan.getvalue(), text)
+        if check=='close':
+            tee.close()
+        else:
+            del tee
+
+    def test(self):
+        for chan in ['stdout', 'stderr']:
+            for check in ['close', 'del']:
+                yield self.tchan(chan, check)

IPython/utils/text.py

@@ -0,0 +1,473 b''
+# encoding: utf-8
+"""
+Utilities for working with strings and text.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import __main__
+
+import os
+import re
+import shutil
+import types
+
+from IPython.external.path import path
+
+from IPython.utils.generics import result_display
+from IPython.utils.io import nlprint
+from IPython.utils.data import flatten
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+StringTypes = types.StringTypes
+
+
+def unquote_ends(istr):
+    """Remove a single pair of quotes from the endpoints of a string."""
+
+    if not istr:
+        return istr
+    if (istr[0]=="'" and istr[-1]=="'") or \
+       (istr[0]=='"' and istr[-1]=='"'):
+        return istr[1:-1]
+    else:
+        return istr
+
+
+class LSString(str):
+    """String derivative with a special access attributes.
+
+    These are normal strings, but with the special attributes:
+
+        .l (or .list) : value as list (split on newlines).
+        .n (or .nlstr): original value (the string itself).
+        .s (or .spstr): value as whitespace-separated string.
+        .p (or .paths): list of path objects
+
+    Any values which require transformations are computed only once and
+    cached.
+
+    Such strings are very useful to efficiently interact with the shell, which
+    typically only understands whitespace-separated options for commands."""
+
+    def get_list(self):
+        try:
+            return self.__list
+        except AttributeError:
+            self.__list = self.split('\n')
+            return self.__list
+
+    l = list = property(get_list)
+
+    def get_spstr(self):
+        try:
+            return self.__spstr
+        except AttributeError:
+            self.__spstr = self.replace('\n',' ')
+            return self.__spstr
+
+    s = spstr = property(get_spstr)
+
+    def get_nlstr(self):
+        return self
+
+    n = nlstr = property(get_nlstr)
+
+    def get_paths(self):
+        try:
+            return self.__paths
+        except AttributeError:
+            self.__paths = [path(p) for p in self.split('\n') if os.path.exists(p)]
+            return self.__paths
+
+    p = paths = property(get_paths)
+
+
+def print_lsstring(arg):
+    """ Prettier (non-repr-like) and more informative printer for LSString """
+    print "LSString (.p, .n, .l, .s available). Value:"
+    print arg
+
+
+print_lsstring = result_display.when_type(LSString)(print_lsstring)
+
+
+class SList(list):
+    """List derivative with a special access attributes.
+
+    These are normal lists, but with the special attributes:
+
+        .l (or .list) : value as list (the list itself).
+        .n (or .nlstr): value as a string, joined on newlines.
+        .s (or .spstr): value as a string, joined on spaces.
+        .p (or .paths): list of path objects
+
+    Any values which require transformations are computed only once and
+    cached."""
+
+    def get_list(self):
+        return self
+
+    l = list = property(get_list)
+
+    def get_spstr(self):
+        try:
+            return self.__spstr
+        except AttributeError:
+            self.__spstr = ' '.join(self)
+            return self.__spstr
+
+    s = spstr = property(get_spstr)
+
+    def get_nlstr(self):
+        try:
+            return self.__nlstr
+        except AttributeError:
+            self.__nlstr = '\n'.join(self)
+            return self.__nlstr
+
+    n = nlstr = property(get_nlstr)
+
+    def get_paths(self):
+        try:
+            return self.__paths
+        except AttributeError:
+            self.__paths = [path(p) for p in self if os.path.exists(p)]
+            return self.__paths
+
+    p = paths = property(get_paths)
+
+    def grep(self, pattern, prune = False, field = None):
+        """ Return all strings matching 'pattern' (a regex or callable)
+
+        This is case-insensitive. If prune is true, return all items
+        NOT matching the pattern.
+
+        If field is specified, the match must occur in the specified
+        whitespace-separated field.
+
+        Examples::
+
+            a.grep( lambda x: x.startswith('C') )
+            a.grep('Cha.*log', prune=1)
+            a.grep('chm', field=-1)
+        """
+
+        def match_target(s):
+            if field is None:
+                return s
+            parts = s.split()
+            try:
+                tgt = parts[field]
+                return tgt
+            except IndexError:
+                return ""
+
+        if isinstance(pattern, basestring):
+            pred = lambda x : re.search(pattern, x, re.IGNORECASE)
+        else:
+            pred = pattern
+        if not prune:
+            return SList([el for el in self if pred(match_target(el))])
+        else:
+            return SList([el for el in self if not pred(match_target(el))])
+
+    def fields(self, *fields):
+        """ Collect whitespace-separated fields from string list
+
+        Allows quick awk-like usage of string lists.
+
+        Example data (in var a, created by 'a = !ls -l')::
+            -rwxrwxrwx  1 ville None      18 Dec 14  2006 ChangeLog
+            drwxrwxrwx+ 6 ville None       0 Oct 24 18:05 IPython
+
+        a.fields(0) is ['-rwxrwxrwx', 'drwxrwxrwx+']
+        a.fields(1,0) is ['1 -rwxrwxrwx', '6 drwxrwxrwx+']
+        (note the joining by space).
+        a.fields(-1) is ['ChangeLog', 'IPython']
+
+        IndexErrors are ignored.
+
+        Without args, fields() just split()'s the strings.
+        """
+        if len(fields) == 0:
+            return [el.split() for el in self]
+
+        res = SList()
+        for el in [f.split() for f in self]:
+            lineparts = []
+
+            for fd in fields:
+                try:
+                    lineparts.append(el[fd])
+                except IndexError:
+                    pass
+            if lineparts:
+                res.append(" ".join(lineparts))
+
+        return res
+
+    def sort(self,field= None,  nums = False):
+        """ sort by specified fields (see fields())
+
+        Example::
+            a.sort(1, nums = True)
+
+        Sorts a by second field, in numerical order (so that 21 > 3)
+
+        """
+
+        #decorate, sort, undecorate
+        if field is not None:
+            dsu = [[SList([line]).fields(field),  line] for line in self]
+        else:
+            dsu = [[line,  line] for line in self]
+        if nums:
+            for i in range(len(dsu)):
+                numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
+                try:
+                    n = int(numstr)
+                except ValueError:
+                    n = 0;
+                dsu[i][0] = n
+
+
+        dsu.sort()
+        return SList([t[1] for t in dsu])
+
+
+def print_slist(arg):
+    """ Prettier (non-repr-like) and more informative printer for SList """
+    print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
+    if hasattr(arg,  'hideonce') and arg.hideonce:
+        arg.hideonce = False
+        return
+
+    nlprint(arg)
+
+
+print_slist = result_display.when_type(SList)(print_slist)
+
+
+def esc_quotes(strng):
+    """Return the input string with single and double quotes escaped out"""
+
+    return strng.replace('"','\\"').replace("'","\\'")
+
+
+def make_quoted_expr(s):
+    """Return string s in appropriate quotes, using raw string if possible.
+
+    XXX - example removed because it caused encoding errors in documentation
+    generation.  We need a new example that doesn't contain invalid chars.
+
+    Note the use of raw string and padding at the end to allow trailing
+    backslash.
+    """
+
+    tail = ''
+    tailpadding = ''
+    raw  = ''
+    if "\\" in s:
+        raw = 'r'
+        if s.endswith('\\'):
+            tail = '[:-1]'
+            tailpadding = '_'
+    if '"' not in s:
+        quote = '"'
+    elif "'" not in s:
+        quote = "'"
+    elif '"""' not in s and not s.endswith('"'):
+        quote = '"""'
+    elif "'''" not in s and not s.endswith("'"):
+        quote = "'''"
+    else:
+        # give up, backslash-escaped string will do
+        return '"%s"' % esc_quotes(s)
+    res = raw + quote + s + tailpadding + quote + tail
+    return res
+
+
+def qw(words,flat=0,sep=None,maxsplit=-1):
+    """Similar to Perl's qw() operator, but with some more options.
+
+    qw(words,flat=0,sep=' ',maxsplit=-1) -> words.split(sep,maxsplit)
+
+    words can also be a list itself, and with flat=1, the output will be
+    recursively flattened.
+
+    Examples:
+
+    >>> qw('1 2')
+    ['1', '2']
+
+    >>> qw(['a b','1 2',['m n','p q']])
+    [['a', 'b'], ['1', '2'], [['m', 'n'], ['p', 'q']]]
+
+    >>> qw(['a b','1 2',['m n','p q']],flat=1)
+    ['a', 'b', '1', '2', 'm', 'n', 'p', 'q']
+    """
+
+    if type(words) in StringTypes:
+        return [word.strip() for word in words.split(sep,maxsplit)
+                if word and not word.isspace() ]
+    if flat:
+        return flatten(map(qw,words,[1]*len(words)))
+    return map(qw,words)
+
+
+def qwflat(words,sep=None,maxsplit=-1):
+    """Calls qw(words) in flat mode. It's just a convenient shorthand."""
+    return qw(words,1,sep,maxsplit)
+
+
+def qw_lol(indata):
+    """qw_lol('a b') -> [['a','b']],
+    otherwise it's just a call to qw().
+
+    We need this to make sure the modules_some keys *always* end up as a
+    list of lists."""
+
+    if type(indata) in StringTypes:
+        return [qw(indata)]
+    else:
+        return qw(indata)
+
+
+def grep(pat,list,case=1):
+    """Simple minded grep-like function.
+    grep(pat,list) returns occurrences of pat in list, None on failure.
+
+    It only does simple string matching, with no support for regexps. Use the
+    option case=0 for case-insensitive matching."""
+
+    # This is pretty crude. At least it should implement copying only references
+    # to the original data in case it's big. Now it copies the data for output.
+    out=[]
+    if case:
+        for term in list:
+            if term.find(pat)>-1: out.append(term)
+    else:
+        lpat=pat.lower()
+        for term in list:
+            if term.lower().find(lpat)>-1: out.append(term)
+
+    if len(out): return out
+    else: return None
+
+
+def dgrep(pat,*opts):
+    """Return grep() on dir()+dir(__builtins__).
+
+    A very common use of grep() when working interactively."""
+
+    return grep(pat,dir(__main__)+dir(__main__.__builtins__),*opts)
+
+
+def idgrep(pat):
+    """Case-insensitive dgrep()"""
+
+    return dgrep(pat,0)
+
+
+def igrep(pat,list):
+    """Synonym for case-insensitive grep."""
+
+    return grep(pat,list,case=0)
+
+
+def indent(str,nspaces=4,ntabs=0):
+    """Indent a string a given number of spaces or tabstops.
+
+    indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
+    """
+    if str is None:
+        return
+    ind = '\t'*ntabs+' '*nspaces
+    outstr = '%s%s' % (ind,str.replace(os.linesep,os.linesep+ind))
+    if outstr.endswith(os.linesep+ind):
+        return outstr[:-len(ind)]
+    else:
+        return outstr
+
+def native_line_ends(filename,backup=1):
+    """Convert (in-place) a file to line-ends native to the current OS.
+
+    If the optional backup argument is given as false, no backup of the
+    original file is left.  """
+
+    backup_suffixes = {'posix':'~','dos':'.bak','nt':'.bak','mac':'.bak'}
+
+    bak_filename = filename + backup_suffixes[os.name]
+
+    original = open(filename).read()
+    shutil.copy2(filename,bak_filename)
+    try:
+        new = open(filename,'wb')
+        new.write(os.linesep.join(original.splitlines()))
+        new.write(os.linesep) # ALWAYS put an eol at the end of the file
+        new.close()
+    except:
+        os.rename(bak_filename,filename)
+    if not backup:
+        try:
+            os.remove(bak_filename)
+        except:
+            pass
+
+
+def list_strings(arg):
+    """Always return a list of strings, given a string or list of strings
+    as input.
+
+    :Examples:
+
+        In [7]: list_strings('A single string')
+        Out[7]: ['A single string']
+
+        In [8]: list_strings(['A single string in a list'])
+        Out[8]: ['A single string in a list']
+
+        In [9]: list_strings(['A','list','of','strings'])
+        Out[9]: ['A', 'list', 'of', 'strings']
+    """
+
+    if isinstance(arg,basestring): return [arg]
+    else: return arg
+
+
+def marquee(txt='',width=78,mark='*'):
+    """Return the input string centered in a 'marquee'.
+
+    :Examples:
+
+        In [16]: marquee('A test',40)
+        Out[16]: '**************** A test ****************'
+
+        In [17]: marquee('A test',40,'-')
+        Out[17]: '---------------- A test ----------------'
+
+        In [18]: marquee('A test',40,' ')
+        Out[18]: '                 A test                 '
+
+    """
+    if not txt:
+        return (mark*width)[:width]
+    nmark = (width-len(txt)-2)/len(mark)/2
+    if nmark < 0: nmark =0
+    marks = mark*nmark
+    return '%s %s %s' % (marks,txt,marks)
+
+

IPython/utils/timing.py

@@ -0,0 +1,116 b''
+# encoding: utf-8
+"""
+Utilities for timing code execution.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import time
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+# If possible (Unix), use the resource module instead of time.clock()
+try:
+    import resource
+    def clocku():
+        """clocku() -> floating point number
+
+        Return the *USER* CPU time in seconds since the start of the process.
+        This is done via a call to resource.getrusage, so it avoids the
+        wraparound problems in time.clock()."""
+
+        return resource.getrusage(resource.RUSAGE_SELF)[0]
+
+    def clocks():
+        """clocks() -> floating point number
+
+        Return the *SYSTEM* CPU time in seconds since the start of the process.
+        This is done via a call to resource.getrusage, so it avoids the
+        wraparound problems in time.clock()."""
+
+        return resource.getrusage(resource.RUSAGE_SELF)[1]
+
+    def clock():
+        """clock() -> floating point number
+
+        Return the *TOTAL USER+SYSTEM* CPU time in seconds since the start of
+        the process.  This is done via a call to resource.getrusage, so it
+        avoids the wraparound problems in time.clock()."""
+
+        u,s = resource.getrusage(resource.RUSAGE_SELF)[:2]
+        return u+s
+
+    def clock2():
+        """clock2() -> (t_user,t_system)
+
+        Similar to clock(), but return a tuple of user/system times."""
+        return resource.getrusage(resource.RUSAGE_SELF)[:2]
+except ImportError:
+    # There is no distinction of user/system time under windows, so we just use
+    # time.clock() for everything...
+    clocku = clocks = clock = time.clock
+    def clock2():
+        """Under windows, system CPU time can't be measured.
+
+        This just returns clock() and zero."""
+        return time.clock(),0.0
+
+    
+def timings_out(reps,func,*args,**kw):
+    """timings_out(reps,func,*args,**kw) -> (t_total,t_per_call,output)
+
+    Execute a function reps times, return a tuple with the elapsed total
+    CPU time in seconds, the time per call and the function's output.
+
+    Under Unix, the return value is the sum of user+system time consumed by
+    the process, computed via the resource module.  This prevents problems
+    related to the wraparound effect which the time.clock() function has.
+
+    Under Windows the return value is in wall clock seconds. See the
+    documentation for the time module for more details."""
+
+    reps = int(reps)
+    assert reps >=1, 'reps must be >= 1'
+    if reps==1:
+        start = clock()
+        out = func(*args,**kw)
+        tot_time = clock()-start
+    else:
+        rng = xrange(reps-1) # the last time is executed separately to store output
+        start = clock()
+        for dummy in rng: func(*args,**kw)
+        out = func(*args,**kw)  # one last time
+        tot_time = clock()-start
+    av_time = tot_time / reps
+    return tot_time,av_time,out
+
+
+def timings(reps,func,*args,**kw):
+    """timings(reps,func,*args,**kw) -> (t_total,t_per_call)
+
+    Execute a function reps times, return a tuple with the elapsed total CPU
+    time in seconds and the time per call. These are just the first two values
+    in timings_out()."""
+
+    return timings_out(reps,func,*args,**kw)[0:2]
+
+
+def timing(func,*args,**kw):
+    """timing(func,*args,**kw) -> t_total
+
+    Execute a function once, return the elapsed total CPU time in
+    seconds. This is just the first value in timings_out()."""
+
+    return timings_out(1,func,*args,**kw)[0]
+

IPython/utils/warn.py

@@ -0,0 +1,66 b''
+# encoding: utf-8
+"""
+Utilities for warnings.  Shoudn't we just use the built in warnings module.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+
+from IPython.utils.io import Term
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def warn(msg,level=2,exit_val=1):
+    """Standard warning printer. Gives formatting consistency.
+
+    Output is sent to Term.cerr (sys.stderr by default).
+
+    Options:
+
+    -level(2): allows finer control:
+      0 -> Do nothing, dummy function.
+      1 -> Print message.
+      2 -> Print 'WARNING:' + message. (Default level).
+      3 -> Print 'ERROR:' + message.
+      4 -> Print 'FATAL ERROR:' + message and trigger a sys.exit(exit_val).
+
+    -exit_val (1): exit value returned by sys.exit() for a level 4
+    warning. Ignored for all other levels."""
+
+    if level>0:
+        header = ['','','WARNING: ','ERROR: ','FATAL ERROR: ']
+        print >> Term.cerr, '%s%s' % (header[level],msg)
+        if level == 4:
+            print >> Term.cerr,'Exiting.\n'
+            sys.exit(exit_val)
+
+            
+def info(msg):
+    """Equivalent to warn(msg,level=1)."""
+
+    warn(msg,level=1)
+
+    
+def error(msg):
+    """Equivalent to warn(msg,level=3)."""
+
+    warn(msg,level=3)
+
+    
+def fatal(msg,exit_val=1):
+    """Equivalent to warn(msg,exit_val=exit_val,level=4)."""
+
+    warn(msg,exit_val=exit_val,level=4)
+

docs/source/development/core_design.txt

@@ -0,0 +1,286 b''
+========================================
+ Design proposal for mod:`IPython.core`
+========================================
+
+Currently mod:`IPython.core` is not well suited for use in GUI applications.
+The purpose of this document is to describe a design that will resolve this
+limitation.
+
+Process and thread model
+========================
+
+The design described here is based on a two process model. These two processes
+are:
+
+1. The IPython engine/kernel. This process contains the user's namespace and 
+   is responsible for executing user code. If user code uses
+   :mod:`enthought.traits` or uses a GUI toolkit to perform plotting, the GUI
+   event loop will run in this process.
+
+2. The GUI application. The user facing GUI application will run in a second
+   process that communicates directly with the IPython engine using
+   asynchronous messaging. The GUI application will not execute any user code.
+   The canonical example of a GUI application that talks to the IPython 
+   engine, would be a GUI based IPython terminal. However, the GUI application 
+   could provide a more sophisticated interface such as a notebook.
+
+We now describe the threading model of the IPython engine. Two threads will be
+used to implement the IPython engine: a main thread that executes user code
+and a networking thread that communicates with the outside world. This
+specific design is required by a number of different factors.
+
+First, The IPython engine must run the GUI event loop if the user wants to
+perform interactive plotting. Because of the design of most GUIs, this means
+that the user code (which will make GUI calls) must live in the main thread.
+
+Second, networking code in the engine (Twisted or otherwise) must be able to
+communicate with the outside world while user code runs. An example would be
+if user code does the following::
+
+    import time
+    for i in range(10):
+        print i
+        time.sleep(2)
+
+We would like to result of each ``print i`` to be seen by the GUI application
+before the entire code block completes. We call this asynchronous printing.
+For this to be possible, the networking code has to be able to be able to
+communicate the current value of ``sys.stdout`` to the GUI application while
+user code is run. Another example is using :mod:`IPython.kernel.client` in
+user code to perform a parallel computation by talking to an IPython
+controller and a set of engines (these engines are separate from the one we
+are discussing here). This module requires the Twisted event loop to be run in
+a different thread than user code.
+
+For the GUI application, threads are optional. However, the GUI application
+does need to be able to perform network communications asynchronously (without
+blocking the GUI itself). With this in mind, there are two options:
+
+* Use Twisted (or another non-blocking socket library) in the same thread as
+  the GUI event loop.
+
+* Don't use Twisted, but instead run networking code in the GUI application
+  using blocking sockets in threads. This would require the usage of polling
+  and queues to manage the networking in the GUI application.
+
+Thus, for the GUI application, there is a choice between non-blocking sockets
+(Twisted) or threads.
+
+Asynchronous messaging
+======================
+
+The GUI application will use asynchronous message queues to communicate with
+the networking thread of the engine. Because this communication will typically
+happen over localhost, a simple, one way, network protocol like XML-RPC or
+JSON-RPC can be used to implement this messaging. These options will also make
+it easy to implement the required networking in the GUI application using the
+standard library. In applications where secure communications are required,
+Twisted and Foolscap will probably be the best way to go for now, but HTTP is
+also an option.
+
+There is some flexibility as to where the message queues are located. One
+option is that we could create a third process (like the IPython controller)
+that only manages the message queues. This is attractive, but does require
+an additional process.
+
+Using this communication channel, the GUI application and kernel/engine will
+be able to send messages back and forth. For the most part, these messages
+will have a request/reply form, but it will be possible for the kernel/engine
+to send multiple replies for a single request.
+
+The GUI application will use these messages to control the engine/kernel.
+Examples of the types of things that will be possible are:
+
+* Pass code (as a string) to be executed by the engine in the user's namespace
+  as a string.
+
+* Get the current value of stdout and stderr.
+
+* Get the ``repr`` of an object returned (Out []:).
+
+* Pass a string to the engine to be completed when the GUI application
+  receives a tab completion event.
+
+* Get a list of all variable names in the user's namespace.
+
+The in memory format of a message should be a Python dictionary, as this
+will be easy to serialize using virtually any network protocol. The 
+message dict should only contain basic types, such as strings, floats,
+ints, lists, tuples and other dicts.
+
+Each message will have a unique id and will probably be determined by the
+messaging system and returned when something is queued in the message
+system. This unique id will be used to pair replies with requests.
+
+Each message should have a header of key value pairs that can be introspected
+by the message system and a body, or payload, that is opaque. The queues
+themselves will be purpose agnostic, so the purpose of the message will have
+to be encoded in the message itself. While we are getting started, we
+probably don't need to distinguish between the header and body.
+
+Here are some examples::
+
+    m1 = dict(
+        method='execute',
+        id=24,  # added by the message system
+        parent=None # not a reply,
+        source_code='a=my_func()'
+    )
+
+This single message could generate a number of reply messages::
+    
+    m2 = dict(
+        method='stdout'
+        id=25,  # my id, added by the message system
+        parent_id=24,  # The message id of the request
+        value='This was printed by my_func()'
+    )
+
+    m3 = dict(
+        method='stdout'
+        id=26,  # my id, added by the message system
+        parent_id=24,  # The message id of the request
+        value='This too was printed by my_func() at a later time.'
+    )
+
+    m4 = dict(
+        method='execute_finished',
+        id=27,
+        parent_id=24
+        # not sure what else should come back with this message,
+        # but we will need a way for the GUI app to tell that an execute
+        # is done.
+    )
+
+We should probably use flags for the method and other purposes:
+
+EXECUTE='0'
+EXECUTE_REPLY='1'
+
+This will keep out network traffic down and enable us to easily change the
+actual value that is sent.
+
+Engine details
+==============
+
+As discussed above, the engine will consist of two threads: a main thread and
+a networking thread. These two threads will communicate using a pair of
+queues: one for data and requests passing to the main thread (the main
+thread's "input queue") and another for data and requests passing out of the
+main thread (the main thread's "output queue"). Both threads will have an
+event loop that will enqueue elements on one queue and dequeue elements on the
+other queue.
+
+The event loop of the main thread will be of a different nature depending on
+if the user wants to perform interactive plotting. If they do want to perform
+interactive plotting, the main threads event loop will simply be the GUI event
+loop. In that case, GUI timers will be used to monitor the main threads input
+queue. When elements appear on that queue, the main thread will respond
+appropriately. For example, if the queue contains an element that consists of
+user code to execute, the main thread will call the appropriate method of its
+IPython instance. If the user does not want to perform interactive plotting,
+the main thread will have a simpler event loop that will simply block on the
+input queue. When something appears on that queue, the main thread will awake
+and handle the request.
+
+The event loop of the networking thread will typically be the Twisted event
+loop. While it is possible to implement the engine's networking without using
+Twisted, at this point, Twisted provides the best solution. Note that the GUI
+application does not need to use Twisted in this case. The Twisted event loop
+will contain an XML-RPC or JSON-RPC server that takes requests over the
+network and handles those requests by enqueing elements on the main thread's
+input queue or dequeing elements on the main thread's output queue.
+
+Because of the asynchronous nature of the network communication, a single
+input and output queue will be used to handle the interaction with the main
+thread. It is also possible to use multiple queues to isolate the different
+types of requests, but our feeling is that this is more complicated than it
+needs to be.
+
+One of the main issues is how stdout/stderr will be handled. Our idea is to
+replace sys.stdout/sys.stderr by custom classes that will immediately write
+data to the main thread's output queue when user code writes to these streams
+(by doing print). Once on the main thread's output queue, the networking
+thread will make the data available to the GUI application over the network.
+
+One unavoidable limitation in this design is that if user code does a print
+and then enters non-GIL-releasing extension code, the networking thread will
+go silent until the GIL is again released. During this time, the networking
+thread will not be able to process the GUI application's requests of the
+engine. Thus, the values of stdout/stderr will be unavailable during this
+time. This goes beyond stdout/stderr, however. Anytime the main thread is
+holding the GIL, the networking thread will go silent and be unable to handle
+requests.
+
+GUI Application details
+=======================
+
+The GUI application will also have two threads. While this is not a strict
+requirement, it probably makes sense and is a good place to start. The main
+thread will be the GUI tread. The other thread will be a networking thread and
+will handle the messages that are sent to and from the engine process.
+
+Like the engine, we will use two queues to control the flow of messages
+between the main thread and networking thread. One of these queues will be
+used for messages sent from the GUI application to the engine. When the GUI
+application needs to send a message to the engine, it will simply enque the
+appropriate message on this queue. The networking thread will watch this queue
+and forward messages to the engine using an appropriate network protocol.
+
+The other queue will be used for incoming messages from the engine. The
+networking thread will poll for incoming messages from the engine. When it
+receives any message, it will simply put that message on this other queue. The
+GUI application will periodically see if there are any messages on this queue
+and if there are it will handle them.
+
+The GUI application must be prepared to handle any incoming message at any
+time. Due to a variety of reasons, the one or more reply messages associated
+with a request, may appear at any time in the future and possible in different
+orders. It is also possible that a reply might not appear. An example of this
+would be a request for a tab completion event. If the engine is busy, it won't
+be possible to fulfill the request for a while. While the tab completion
+request will eventually be handled, the GUI application has to be prepared to
+abandon waiting for the reply if the user moves on or a certain timeout
+expires.
+
+Prototype details
+=================
+
+With this design, it should be possible to develop a relatively complete GUI
+application, while using a mock engine. This prototype should use the two
+process design described above, but instead of making actual network calls,
+the network thread of the GUI application should have an object that fakes the
+network traffic. This mock object will consume messages off of one queue,
+pause for a short while (to model network and other latencies) and then place
+reply messages on the other queue.
+
+This simple design will allow us to determine exactly what the message types
+and formats should be as well as how the GUI application should interact with
+the two message queues. Note, it is not required that the mock object actually
+be able to execute Python code or actually complete strings in the users
+namespace. All of these things can simply be faked. This will also help us to
+understand what the interface needs to look like that handles the network
+traffic. This will also help us to understand the design of the engine better.
+
+The GUI application should be developed using IPython's component, application
+and configuration system. It may take some work to see what the best way of
+integrating these things with PyQt are.
+
+After this stage is done, we can move onto creating a real IPython engine for
+the GUI application to communicate with. This will likely be more work that
+the GUI application itself, but having a working GUI application will make it
+*much* easier to design and implement the engine.
+
+We also might want to introduce a third process into the mix. Basically, this
+would be a central messaging hub that both the engine and GUI application
+would use to send and retrieve messages. This is not required, but it might be
+a really good idea.
+
+Also, I have some ideas on the best way to handle notebook saving and
+persistence.
+
+Refactoring of IPython.core
+===========================
+
+We need to go through IPython.core and describe what specifically needs to be
+done.

docs/source/development/inputhook_app.txt

@@ -0,0 +1,55 b''
+=========================
+IPython GUI Support Notes
+=========================
+
+IPython allows GUI event loops to be run in an interactive IPython session.
+This is done using Python's PyOS_InputHook hook which Python calls
+when the :func:`raw_input` function is called and waiting for user input.
+IPython has versions of this hook for wx, pyqt4 and pygtk.
+
+When a GUI program is used interactively within IPython, the event loop of
+the GUI should *not* be started. This is because, the PyOS_Inputhook itself
+is responsible for iterating the GUI event loop.
+
+IPython has facilities for installing the needed input hook for each GUI 
+toolkit and for creating the needed main GUI application object. Usually,
+these main application objects should be created only once and for some
+GUI toolkits, special options have to be passed to the application object
+to enable it to function properly in IPython.
+
+We need to answer the following questions:
+
+* Who is responsible for creating the main GUI application object, IPython
+  or third parties (matplotlib, enthought.traits, etc.)?
+
+* What is the proper way for third party code to detect if a GUI application
+  object has already been created?  If one has been created, how should
+  the existing instance be retrieved?
+
+* In a GUI application object has been created, how should third party code
+  detect if the GUI event loop is running. It is not sufficient to call the
+  relevant function methods in the GUI toolkits (like ``IsMainLoopRunning``)
+  because those don't know if the GUI event loop is running through the
+  input hook.
+
+* We might need a way for third party code to determine if it is running
+  in IPython or not.  Currently, the only way of running GUI code in IPython
+  is by using the input hook, but eventually, GUI based versions of IPython
+  will allow the GUI event loop in the more traditional manner. We will need
+  a way for third party code to distinguish between these two cases.
+
+Here is some sample code I have been using to debug this issue::
+
+    from matplotlib import pyplot as plt
+
+    from enthought.traits import api as traits
+
+    class Foo(traits.HasTraits):
+        a = traits.Float()
+
+    f = Foo()
+    f.configure_traits()
+
+    plt.plot(range(10))
+
+

docs/source/development/session_format.txt

@@ -0,0 +1,111 b''
+===============================
+ IPython session storage notes
+===============================
+
+This document serves as a sample/template for ideas on how to store session
+data on disk.  This stems from discussions we had on various mailing lists, and
+should be considered a pure work in progress.  We haven't settled these ideas
+completely yet, and there's a lot to discuss; this document should just serve
+as a reference of the distilled points from various conversations on multiple
+mailing lists, and will congeal over time on a specific design we implement.
+
+The frontend would store, for now, 5 types of data:
+
+#. Input: this is python/ipython code to be executed.
+
+#. Output (python): result of executing Inputs.
+
+#. Standard output: from subprocesses.
+
+#. Standard error: from subprocesses.
+
+#. Text: arbitrary text.  For now, we'll just store plain text and will defer
+   to the user on how to format it, though it should be valid reST if it is
+   later to be converted into html/pdf.
+
+The non-text cells would be stored on-disk as follows::
+
+    .. input-cell::
+      :id: 1
+
+      3+3
+
+    .. output-cell::
+       :id: 1
+
+       6
+
+    .. input-cell::
+       :id: 2
+
+       ls
+
+    .. stdout-cell::
+       :id: 2
+
+       a.py b.py
+
+    .. input-cell::
+       :id: 3
+
+      !askdfj
+
+    .. stderr-cell::
+       :id: 3
+
+       sh: askdfj: command not found
+
+Brian made some interesting points on the mailing list inspired by the
+Mathematica format, reproduced here for reference:
+
+The Mathematica notebook format is a plain text file that itself is *valid
+Mathematica code*.  This id documented here: 
+
+http://reference.wolfram.com/mathematica/guide/LowLevelNotebookProgramming.html
+
+For examples a simple notebook with one text cell is just::
+
+  Notebook[{Cell['Here is my text', 'Text']}]
+
+Everything - input cells, output cells, static images and all are represented
+in this way and embedded in the plain text notebook file.  The Python
+generalization of this would be the following: 
+
+* A Python notebook is plain text, importable Python code.
+
+* That code is simply a tree of objects that declare the relevant parts of the
+  notebook. 
+
+This has a number of advantages:
+
+* A notebook can be imported, manipulated and run by anyone who has the support
+  code (the notebook module that defines the relevant classes).
+  
+* A notebook doesn't need to be parsed.  It is valid Python and can be imported
+  or exec'd.  Once that is done, you have the full notebook in memory.  You can
+  immediately do anything you want with it.
+  
+* The various Notebook, Cell, Image, etc. classes can know about how to output
+  to various formats, latex, html, reST, XML, etc::
+
+  import mynotebook
+  mynotebook.notebook.export('rest')
+
+* Each individual format (HTML, reST, latex) has weaknesses.  If you pick any
+  one to be *the* notebook format, you are building those weaknesses into your
+  design.  A pure python based notebook format won't suffer from that syndrome.
+  
+* It is a clean separation of the model (Notebook, Cell, Image, etc.) and the
+  view (HTML, reST, etc.).  Picking HTML or reST for the notebook format
+  confuses (at some level) the model and view...
+  
+* Third party code can define new Notebook elements that specify how they can
+  be rendered in different contexts.  For example, matplotlib could ship a
+  Figure element that knows how to render itself as a native PyQt GUI, a static
+  image, a web page, etc.
+  
+* A notebook remains a single plain text file that anyone can edit - even if it
+  has embedded images.  Neither HTML nor reST have the ability to inline
+  graphics in plain text files.  While I love reST, it is a pain that I need an
+  entire directory of files to render a single Sphinx doc.
+  
  No newline at end of file

IPython/__init__.py

@@ -30,14 +30,14 b" if sys.version[0:3] < '2.5':"
 
 
 # Make it easy to import extensions - they are always directly on pythonpath.
-# Therefore, non-IPython modules can be added to extensions directory
+# Therefore, non-IPython modules can be added to extensions directory.
+# This should probably be in ipapp.py.
 sys.path.append(os.path.join(os.path.dirname(__file__), "extensions"))
 
 #-----------------------------------------------------------------------------
 # Setup the top level names
 #-----------------------------------------------------------------------------
 
-# In some cases, these are causing circular imports.
 from .config.loader import Config
 from .core import release
 from .core.application import Application

IPython/config/loader.py

@@ -23,7 +23,7 b' import os'
 import sys
 
 from IPython.external import argparse
-from IPython.utils.genutils import filefind
+from IPython.utils.path import filefind
 
 #-----------------------------------------------------------------------------
 # Exceptions
@@ -40,6 +40,7 b' class ConfigLoaderError(ConfigError):'
 #-----------------------------------------------------------------------------
 # Argparse fix
 #-----------------------------------------------------------------------------
+
 # Unfortunately argparse by default prints help messages to stderr instead of
 # stdout.  This makes it annoying to capture long help screens at the command
 # line, since one must know how to pipe stderr, which many users don't know how
@@ -200,10 +201,13 b' class ConfigLoader(object):'
         self.config = Config()
 
     def load_config(self):
-        """Load a config from somewhere, return a Struct.
+        """Load a config from somewhere, return a :class:`Config` instance.
         
         Usually, this will cause self.config to be set and then returned.
+        However, in most cases, :meth:`ConfigLoader.clear` should be called
+        to erase any previous state.
         """
+        self.clear()
         return self.config
 
 
@@ -242,6 +246,7 b' class PyFileConfigLoader(FileConfigLoader):'
 
     def load_config(self):
         """Load the config from a file and return it as a Struct."""
+        self.clear()
         self._find_file()
         self._read_file_as_dict()
         self._convert_to_config()
@@ -292,21 +297,11 b' class CommandLineConfigLoader(ConfigLoader):'
     """
 
 
-class __NoConfigDefault(object): pass
-NoConfigDefault = __NoConfigDefault()
-
-
 class ArgParseConfigLoader(CommandLineConfigLoader):
-    #: Global default for arguments (see argparse docs for details)
-    argument_default = NoConfigDefault
     
-    def __init__(self, argv=None, arguments=(), *args, **kw):
+    def __init__(self, argv=None, *parser_args, **parser_kw):
         """Create a config loader for use with argparse.
 
-        With the exception of ``argv`` and ``arguments``, other args and kwargs
-        arguments here are passed onto the constructor of
-        :class:`argparse.ArgumentParser`.
-
         Parameters
         ----------
 
@@ -314,19 +309,22 b' class ArgParseConfigLoader(CommandLineConfigLoader):'
           If given, used to read command-line arguments from, otherwise
           sys.argv[1:] is used.
 
-        arguments : optional, tuple
-          Description of valid command-line arguments, to be called in sequence
-          with parser.add_argument() to configure the parser.
+        parser_args : tuple
+          A tuple of positional arguments that will be passed to the
+          constructor of :class:`argparse.ArgumentParser`.
+
+        parser_kw : dict
+          A tuple of keyword arguments that will be passed to the
+          constructor of :class:`argparse.ArgumentParser`.
         """
         super(CommandLineConfigLoader, self).__init__()
         if argv == None:
             argv = sys.argv[1:]
         self.argv = argv
-        self.arguments = arguments
-        self.args = args
-        kwargs = dict(argument_default=self.argument_default)
-        kwargs.update(kw)
-        self.kw = kwargs
+        self.parser_args = parser_args
+        kwargs = dict(argument_default=argparse.SUPPRESS)
+        kwargs.update(parser_kw)
+        self.parser_kw = kwargs
 
     def load_config(self, args=None):
         """Parse command line arguments and return as a Struct.
@@ -335,10 +333,10 b' class ArgParseConfigLoader(CommandLineConfigLoader):'
         ----------
 
         args : optional, list
-          If given, a list with the structure of sys.argv[1:] to parse arguments
-          from.  If not given, the instance's self.argv attribute (given at
-          construction time) is used."""
-        
+          If given, a list with the structure of sys.argv[1:] to parse
+          arguments from. If not given, the instance's self.argv attribute
+          (given at construction time) is used."""
+        self.clear()
         if args is None:
             args = self.argv
         self._create_parser()
@@ -353,17 +351,11 b' class ArgParseConfigLoader(CommandLineConfigLoader):'
             return []
 
     def _create_parser(self):
-        self.parser = ArgumentParser(*self.args, **self.kw)
+        self.parser = ArgumentParser(*self.parser_args, **self.parser_kw)
         self._add_arguments()
-        self._add_other_arguments()
 
     def _add_arguments(self):
-        for argument in self.arguments:
-            self.parser.add_argument(*argument[0],**argument[1])
-
-    def _add_other_arguments(self):
-        """Meant for subclasses to add their own arguments."""
-        pass
+        raise NotImplementedError("subclasses must implement _add_arguments")
 
     def _parse_args(self, args):
         """self.parser->self.parsed_data""" 
@@ -372,6 +364,7 b' class ArgParseConfigLoader(CommandLineConfigLoader):'
     def _convert_to_config(self):
         """self.parsed_data->self.config"""
         for k, v in vars(self.parsed_data).items():
-            if v is not NoConfigDefault:
                 exec_str = 'self.config.' + k + '= v'
                 exec exec_str in locals(), globals()
+
+

IPython/config/tests/test_loader.py

@@ -61,35 +61,38 b' class TestPyFileCL(TestCase):'
         self.assertEquals(config.Foo.Bam.value, range(10))
         self.assertEquals(config.D.C.value, 'hi there')
 
+class MyLoader1(ArgParseConfigLoader):
+    def _add_arguments(self):
+        p = self.parser
+        p.add_argument('-f', '--foo', dest='Global.foo', type=str)
+        p.add_argument('-b', dest='MyClass.bar', type=int)
+        p.add_argument('-n', dest='n', action='store_true')
+        p.add_argument('Global.bam', type=str)
+
+class MyLoader2(ArgParseConfigLoader):
+    def _add_arguments(self):
+        subparsers = self.parser.add_subparsers(dest='subparser_name')
+        subparser1 = subparsers.add_parser('1')
+        subparser1.add_argument('-x',dest='Global.x')
+        subparser2 = subparsers.add_parser('2')
+        subparser2.add_argument('y')
 
 class TestArgParseCL(TestCase):
 
     def test_basic(self):
-
-        arguments = (
-                (('-f','--foo'), dict(dest='Global.foo', type=str)),
-                (('-b',), dict(dest='MyClass.bar', type=int)),
-                (('-n',), dict(dest='n', action='store_true')),
-                (('Global.bam',), dict(type=str))
-            )
-        cl = ArgParseConfigLoader(arguments=arguments)
+        cl = MyLoader1()
         config = cl.load_config('-f hi -b 10 -n wow'.split())
         self.assertEquals(config.Global.foo, 'hi')
         self.assertEquals(config.MyClass.bar, 10)
         self.assertEquals(config.n, True)
         self.assertEquals(config.Global.bam, 'wow')
+        config = cl.load_config(['wow'])
+        self.assertEquals(config.keys(), ['Global'])
+        self.assertEquals(config.Global.keys(), ['bam'])
+        self.assertEquals(config.Global.bam, 'wow')
 
     def test_add_arguments(self):
-
-        class MyLoader(ArgParseConfigLoader):
-            def _add_arguments(self):
-                subparsers = self.parser.add_subparsers(dest='subparser_name')
-                subparser1 = subparsers.add_parser('1')
-                subparser1.add_argument('-x',dest='Global.x')
-                subparser2 = subparsers.add_parser('2')
-                subparser2.add_argument('y')
-
-        cl = MyLoader()
+        cl = MyLoader2()
         config = cl.load_config('2 frobble'.split())
         self.assertEquals(config.subparser_name, '2')
         self.assertEquals(config.y, 'frobble')
@@ -97,6 +100,15 b' class TestArgParseCL(TestCase):'
         self.assertEquals(config.subparser_name, '1')
         self.assertEquals(config.Global.x, 'frobble')
 
+    def test_argv(self):
+        cl = MyLoader1(argv='-f hi -b 10 -n wow'.split())
+        config = cl.load_config()
+        self.assertEquals(config.Global.foo, 'hi')
+        self.assertEquals(config.MyClass.bar, 10)
+        self.assertEquals(config.n, True)
+        self.assertEquals(config.Global.bam, 'wow')
+
+
 class TestConfig(TestCase):
 
     def test_setget(self):

IPython/core/alias.py

@@ -28,9 +28,9 b' import sys'
 from IPython.core.component import Component
 from IPython.core.splitinput import split_user_input
 
-from IPython.utils.traitlets import CBool, List, Instance
-from IPython.utils.genutils import error
+from IPython.utils.traitlets import List
 from IPython.utils.autoattr import auto_attr
+from IPython.utils.warn import warn, error
 
 #-----------------------------------------------------------------------------
 # Utilities

IPython/core/application.py

@@ -33,7 +33,7 b' import os'
 import sys
 
 from IPython.core import release, crashhandler
-from IPython.utils.genutils import get_ipython_dir, get_ipython_package_dir
+from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
 from IPython.config.loader import (
     PyFileConfigLoader,
     ArgParseConfigLoader,
@@ -48,108 +48,86 b' class ApplicationError(Exception):'
     pass
 
 
-app_cl_args = (
-    (('--ipython-dir', ), dict(
+class BaseAppConfigLoader(ArgParseConfigLoader):
+    """Default command line options for IPython based applications."""
+
+    def _add_ipython_dir(self, parser):
+        """Add the --ipython-dir option to the parser."""
+        paa = parser.add_argument
+        paa('--ipython-dir', 
         dest='Global.ipython_dir',type=unicode,
         help=
         """Set to override default location of the IPython directory
-        IPYTHON_DIR, stored as Global.ipython_dir.  This can also be specified
-        through the environment variable IPYTHON_DIR.""",
-        metavar='Global.ipython_dir') ),
-    (('-p', '--profile',), dict(
-        dest='Global.profile',type=unicode,
-        help=
-        """The string name of the ipython profile to be used.  Assume that your
-        config file is ipython_config-<name>.py (looks in current dir first,
-        then in IPYTHON_DIR). This is a quick way to keep and load multiple
-        config files for different tasks, especially if include your basic one
-        in your more specialized ones.  You can keep a basic
-        IPYTHON_DIR/ipython_config.py file and then have other 'profiles' which
-        include this one and load extra things for particular tasks.""",
-        metavar='Global.profile') ),
-    (('--log-level',), dict(
+            IPYTHON_DIR, stored as Global.ipython_dir.  This can also be 
+            specified through the environment variable IPYTHON_DIR.""",
+            metavar='Global.ipython_dir')
+
+    def _add_log_level(self, parser):
+        """Add the --log-level option to the parser."""
+        paa = parser.add_argument
+        paa('--log-level',
         dest="Global.log_level",type=int,
         help='Set the log level (0,10,20,30,40,50).  Default is 30.',
-        metavar='Global.log_level')),
-    (('--config-file',), dict(
-        dest='Global.config_file',type=unicode,
-        help=
-        """Set the config file name to override default.  Normally IPython
-        loads ipython_config.py (from current directory) or
-        IPYTHON_DIR/ipython_config.py.  If the loading of your config file
-        fails, IPython starts with a bare bones configuration (no modules
-        loaded at all).""",
-        metavar='Global.config_file')),
-    )
+            metavar='Global.log_level')
+
+    def _add_arguments(self):
+        self._add_ipython_dir(self.parser)
+        self._add_log_level(self.parser)
+
 
 class Application(object):
     """Load a config, construct components and set them running.
 
-    The configuration of an application can be done via four different Config
-    objects, which are loaded and ultimately merged into a single one used from
-    that point on by the app.  These are:
+    The configuration of an application can be done via three different Config
+    objects, which are loaded and ultimately merged into a single one used
+    from that point on by the app. These are:
 
     1. default_config: internal defaults, implemented in code.
     2. file_config: read from the filesystem.
     3. command_line_config: read from the system's command line flags.
-    4. constructor_config: passed parametrically to the constructor.
 
     During initialization, 3 is actually read before 2, since at the
     command-line one may override the location of the file to be read.  But the
     above is the order in which the merge is made.
-
-    There is a final config object can be created and passed to the
-    constructor: override_config.  If it exists, this completely overrides the
-    configs 2-4 above (the default is still used to ensure that all needed
-    fields at least are created).  This makes it easier to create
-    parametrically (e.g. in testing or sphinx plugins) objects with a known
-    configuration, that are unaffected by whatever arguments may be present in
-    sys.argv or files in the user's various directories.
     """
 
     name = u'ipython'
     description = 'IPython: an enhanced interactive Python shell.'
-    #: usage message printed by argparse. If None, auto-generate
+    #: Usage message printed by argparse. If None, auto-generate
     usage = None
-    config_file_name = u'ipython_config.py'
-    #: Track the default and actual separately because some messages are
-    #: only printed if we aren't using the default.
-    default_config_file_name = config_file_name
+    #: The command line config loader.  Subclass of ArgParseConfigLoader.
+    command_line_loader = BaseAppConfigLoader
+    #: The name of the config file to load, determined at runtime
+    config_file_name = None
+    #: The name of the default config file. Track separately from the actual
+    #: name because some logic happens only if we aren't using the default.
+    default_config_file_name = u'ipython_config.py'
     default_log_level = logging.WARN
     #: Set by --profile option
     profile_name = None
     #: User's ipython directory, typically ~/.ipython/
     ipython_dir = None
-    #: internal defaults, implemented in code.
+    #: Internal defaults, implemented in code.
     default_config = None
-    #: read from the filesystem
+    #: Read from the filesystem.
     file_config = None
-    #: read from the system's command line flags
+    #: Read from the system's command line flags.
     command_line_config = None
-    #: passed parametrically to the constructor.
-    constructor_config = None
-    #: final override, if given supercedes file/command/constructor configs
-    override_config = None
+    #: The final config that will be passed to the component.
+    master_config = None
     #: A reference to the argv to be used (typically ends up being sys.argv[1:])
     argv = None
-    #: Default command line arguments.  Subclasses should create a new tuple
-    #: that *includes* these.
-    cl_arguments = app_cl_args
-
     #: extra arguments computed by the command-line loader
     extra_args = None
+    #: The class to use as the crash handler.
+    crash_handler_class = crashhandler.CrashHandler
 
     # Private attributes
     _exiting = False
     _initialized = False
 
-    # Class choices for things that will be instantiated at runtime.
-    _CrashHandler = crashhandler.CrashHandler
-
-    def __init__(self, argv=None, constructor_config=None, override_config=None):
+    def __init__(self, argv=None):
         self.argv = sys.argv[1:] if argv is None else argv
-        self.constructor_config = constructor_config
-        self.override_config = override_config
         self.init_logger()
 
     def init_logger(self):
@@ -194,7 +172,6 b' class Application(object):'
         self.log_default_config()
         self.set_default_config_log_level()
 
-        if self.override_config is None:
             # Command-line config
             self.pre_load_command_line_config()
             self.load_command_line_config()
@@ -209,7 +186,6 b' class Application(object):'
         self.find_config_file_name()
         self.find_config_file_paths()
 
-        if self.override_config is None:
             # File-based config
             self.pre_load_file_config()
             self.load_file_config()
@@ -240,7 +216,7 b' class Application(object):'
 
     def create_crash_handler(self):
         """Create a crash handler, typically setting sys.excepthook to it."""
-        self.crash_handler = self._CrashHandler(self, self.name)
+        self.crash_handler = self.crash_handler_class(self)
         sys.excepthook = self.crash_handler
 
     def create_default_config(self):
@@ -270,10 +246,11 b' class Application(object):'
 
     def create_command_line_config(self):
         """Create and return a command line config loader."""
-        return ArgParseConfigLoader(self.argv, self.cl_arguments,
+        return self.command_line_loader(
+            self.argv,
                                     description=self.description, 
                                     version=release.version,
-                                    usage=self.usage,
+            usage=self.usage
                                     )
 
     def pre_load_command_line_config(self):
@@ -338,18 +315,22 b' class Application(object):'
 
         If a profile has been set at the command line, this will resolve it.
         """
-
         try:
             self.config_file_name = self.command_line_config.Global.config_file
         except AttributeError:
             pass
+        else:
+            return
 
         try:
             self.profile_name = self.command_line_config.Global.profile
         except AttributeError:
-            pass
+            # Just use the default as there is no profile
+            self.config_file_name = self.default_config_file_name
         else:
-            name_parts = self.config_file_name.split('.')
+            # Use the default config file name and profile name if set
+            # to determine the used config file name.
+            name_parts = self.default_config_file_name.split('.')
             name_parts.insert(1, u'_' + self.profile_name + u'.')
             self.config_file_name = ''.join(name_parts)
 
@@ -418,13 +399,9 b' class Application(object):'
         """Merge the default, command line and file config objects."""
         config = Config()
         config._merge(self.default_config)
-        if self.override_config is None:
             config._merge(self.file_config)
             config._merge(self.command_line_config)
-            if self.constructor_config is not None:
-                config._merge(self.constructor_config)
-        else:
-            config._merge(self.override_config)
+
         # XXX fperez - propose to Brian we rename master_config to simply
         # config, I think this is going to be heavily used in examples and
         # application code and the name is shorter/easier to find/remember.
@@ -456,15 +433,6 b' class Application(object):'
     # Utility methods
     #-------------------------------------------------------------------------
 
-    def abort(self):
-        """Abort the starting of the application."""
-        if self._exiting:
-            pass
-        else:
-            self.log.critical("Aborting application: %s" % self.name, exc_info=True)
-            self._exiting = True
-            sys.exit(1)
-
     def exit(self, exit_status=0):
         if self._exiting:
             pass
@@ -473,17 +441,13 b' class Application(object):'
             self._exiting = True
             sys.exit(exit_status)
 
-    def attempt(self, func, action='abort'):
+    def attempt(self, func):
         try:
             func()
         except SystemExit:
             raise
         except:
-            if action == 'abort':
                 self.log.critical("Aborting application: %s" % self.name,
                                   exc_info=True)
-                self.abort()
-                raise
-            elif action == 'exit':
                 self.exit(0)
 

IPython/core/completer.py

@@ -69,19 +69,20 b' used, and this module (and the readline module) are silently inactive.'
 import __builtin__
 import __main__
 import glob
+import inspect
 import itertools
 import keyword
 import os
 import re
 import shlex
 import sys
-import types
 
-import IPython.utils.rlineimpl as readline
 from IPython.core.error import TryNext
 from IPython.core.prefilter import ESC_MAGIC
 from IPython.utils import generics
-from IPython.utils.genutils import debugx, dir2
+from IPython.utils.frame import debugx
+from IPython.utils.dir2 import dir2
+import IPython.utils.rlineimpl as readline
 
 #-----------------------------------------------------------------------------
 # Globals
@@ -216,7 +217,6 b' class Completer:'
         with a __getattr__ hook is evaluated.
 
         """
-        import re
 
         #print 'Completer->attr_matches, txt=%r' % text # dbg
         # Another option, seems to work great. Catches things like ''.<tab>

IPython/core/component.py

@@ -27,7 +27,7 b' from weakref import WeakValueDictionary'
 from IPython.utils.importstring import import_item
 from IPython.config.loader import Config
 from IPython.utils.traitlets import (
-    HasTraitlets, TraitletError, MetaHasTraitlets, Instance, This
+    HasTraitlets, MetaHasTraitlets, Instance, This
 )
 
 

IPython/core/crashhandler.py

@@ -1,74 +1,94 b''
-# -*- coding: utf-8 -*-
+# encoding: utf-8
 """sys.excepthook for IPython itself, leaves a detailed report on disk.
 
+Authors:
 
-Authors
--------
-- Fernando Perez <[e-mail]>
+* Fernando Perez
+* Brian E. Granger
 """
 
-#*****************************************************************************
-#       Copyright (C) 2008-2009 The IPython Development Team
+#-----------------------------------------------------------------------------
 #       Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
+#  Copyright (C) 2008-2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#*****************************************************************************
+#-----------------------------------------------------------------------------
 
-#****************************************************************************
-# Required modules
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
 
-# From the standard library
 import os
 import sys
 from pprint import pformat
 
-# Our own
-from IPython.core import release
 from IPython.core import ultratb
-from IPython.utils.genutils import sys_info
-
 from IPython.external.Itpl import itpl
+from IPython.utils.sysinfo import sys_info
 
-#****************************************************************************
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
 
-class CrashHandler(object):
-    """Customizable crash handlers for IPython-based systems.
+# Template for the user message.
+_default_message_template = """\
+Oops, $self.app_name crashed. We do our best to make it stable, but...
 
-    Instances of this class provide a __call__ method which can be used as a
-    sys.excepthook, i.e., the __call__ signature is:
+A crash report was automatically generated with the following information:
+  - A verbatim copy of the crash traceback.
+  - A copy of your input history during this session.
+  - Data on your current $self.app_name configuration.
 
-        def __call__(self,etype, evalue, etb)
+It was left in the file named:
+\t'$self.crash_report_fname'
+If you can email this file to the developers, the information in it will help
+them in understanding and correcting the problem.
 
+You can mail it to: $self.contact_name at $self.contact_email
+with the subject '$self.app_name Crash Report'.
+
+If you want to do it now, the following command will work (under Unix):
+mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
+
+To ensure accurate tracking of this issue, please file a report about it at:
+$self.bug_tracker
     """
 
-    def __init__(self,app, app_name, contact_name=None, contact_email=None, 
-                 bug_tracker=None, crash_report_fname='CrashReport.txt', 
-                 show_crash_traceback=True, call_pdb=False):
-        """New crash handler.
 
-        Inputs:
+class CrashHandler(object):
+    """Customizable crash handlers for IPython applications.
 
-        - app: a running application instance, which will be queried at crash
-        time for internal information.
+    Instances of this class provide a :meth:`__call__` method which can be
+    used as a ``sys.excepthook``.  The :meth:`__call__` signature is::
 
-        - app_name: a string containing the name of your application.
+        def __call__(self, etype, evalue, etb)
+    """
 
-        - contact_name: a string with the name of the person to contact.
+    message_template = _default_message_template
 
-        - contact_email: a string with the email address of the contact.
+    def __init__(self, app, contact_name=None, contact_email=None, 
+                 bug_tracker=None, show_crash_traceback=True, call_pdb=False):
+        """Create a new crash handler
 
-        - bug_tracker: a string with the URL for your project's bug tracker.
+        Parameters
+        ----------
+        app :  Application
+            A running :class:`Application` instance, which will be queried at 
+            crash time for internal information.
 
-        - crash_report_fname: a string with the filename for the crash report
-        to be saved in.  These reports are left in the ipython user directory
-        as determined by the running IPython instance.
+        contact_name : str
+            A string with the name of the person to contact.
 
-        Optional inputs:
+        contact_email : str
+            A string with the email address of the contact.
         
-        - show_crash_traceback(True): if false, don't print the crash
-        traceback on stderr, only generate the on-disk report
+        bug_tracker : str
+            A string with the URL for your project's bug tracker.
 
+        show_crash_traceback : bool
+            If false, don't print the crash traceback on stderr, only generate
+            the on-disk report
 
         Non-argument instance attributes:
 
@@ -76,48 +96,17 b' class CrashHandler(object):'
         further customization of the crash handler's behavior.  Please see the
         source for further details.
         """
-
-        # apply args into instance
         self.app = app
-        self.app_name = app_name
+        self.app_name = self.app.name
         self.contact_name = contact_name
         self.contact_email = contact_email
         self.bug_tracker = bug_tracker
-        self.crash_report_fname = crash_report_fname
+        self.crash_report_fname = "Crash_report_%s.txt" % self.app_name
         self.show_crash_traceback = show_crash_traceback
         self.section_sep = '\n\n'+'*'*75+'\n\n'
         self.call_pdb = call_pdb
         #self.call_pdb = True # dbg
         
-        # Hardcoded defaults, which can be overridden either by subclasses or
-        # at runtime for the instance.
-
-        # Template for the user message.  Subclasses which completely override
-        # this, or user apps, can modify it to suit their tastes.  It gets
-        # expanded using itpl, so calls of the kind $self.foo are valid.
-        self.user_message_template = """
-Oops, $self.app_name crashed. We do our best to make it stable, but...
-
-A crash report was automatically generated with the following information:
-  - A verbatim copy of the crash traceback.
-  - A copy of your input history during this session.
-  - Data on your current $self.app_name configuration.
-
-It was left in the file named:
-\t'$self.crash_report_fname'
-If you can email this file to the developers, the information in it will help
-them in understanding and correcting the problem.
-
-You can mail it to: $self.contact_name at $self.contact_email
-with the subject '$self.app_name Crash Report'.
-
-If you want to do it now, the following command will work (under Unix):
-mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
-
-To ensure accurate tracking of this issue, please file a report about it at:
-$self.bug_tracker
-"""
-
     def __call__(self,etype, evalue, etb):
         """Handle an exception, call for compatible with sys.excepthook"""
 
@@ -137,7 +126,8 b' $self.bug_tracker'
         # write the report filename into the instance dict so it can get
         # properly expanded out in the user message template
         self.crash_report_fname = report_name
-        TBhandler = ultratb.VerboseTB(color_scheme=color_scheme,
+        TBhandler = ultratb.VerboseTB(
+            color_scheme=color_scheme,
                                       long_header=1,
                                       call_pdb=self.call_pdb,
                                       )
@@ -159,7 +149,7 b' $self.bug_tracker'
             return
 
         # Inform user on stderr of what happened
-        msg = itpl('\n'+'*'*70+'\n'+self.user_message_template)
+        msg = itpl('\n'+'*'*70+'\n'+self.message_template)
         print >> sys.stderr, msg
 
         # Construct report on disk
@@ -178,7 +168,9 b' $self.bug_tracker'
         
         try:
             config = pformat(self.app.config)
-            rpt_add(sec_sep+'Current user configuration structure:\n\n')
+            rpt_add(sec_sep)
+            rpt_add('Application name: %s\n\n' % self.app_name)
+            rpt_add('Current user configuration structure:\n\n')
             rpt_add(config)
         except:
             pass
@@ -186,38 +178,3 b' $self.bug_tracker'
 
         return ''.join(report)
 
-
-class IPythonCrashHandler(CrashHandler):
-    """sys.excepthook for IPython itself, leaves a detailed report on disk."""
-    
-    def __init__(self, app, app_name='IPython'):
-
-        # Set here which of the IPython authors should be listed as contact
-        AUTHOR_CONTACT = 'Fernando'
-        
-        # Set argument defaults
-        bug_tracker = 'https://bugs.launchpad.net/ipython/+filebug'
-        contact_name,contact_email = release.authors[AUTHOR_CONTACT][:2]
-        crash_report_fname = 'IPython_crash_report.txt'
-        # Call parent constructor
-        CrashHandler.__init__(self,app,app_name,contact_name,contact_email,
-                              bug_tracker,crash_report_fname)
-
-    def make_report(self,traceback):
-        """Return a string containing a crash report."""
-
-        sec_sep = self.section_sep
-        # Start with parent report
-        report = [super(IPythonCrashHandler, self).make_report(traceback)]
-        # Add interactive-specific info we may have
-        rpt_add = report.append
-        try:
-            rpt_add(sec_sep+"History of session input:")
-            for line in self.app.shell.user_ns['_ih']:
-                rpt_add(line)
-            rpt_add('\n*** Last line of input (may not be in above history):\n')
-            rpt_add(self.app.shell._last_input_line+'\n')
-        except:
-            pass
-
-        return ''.join(report)

IPython/core/debugger.py

@@ -26,15 +26,13 b' http://www.python.org/2.2.3/license.html"""'
 #*****************************************************************************
 
 import bdb
-import cmd
 import linecache
-import os
 import sys
 
 from IPython.utils import PyColorize
 from IPython.core import ipapi
 from IPython.utils import coloransi
-from IPython.utils.genutils import Term
+from IPython.utils.io import Term
 from IPython.core.excolors import exception_colors
 
 # See if we can use pydb.

IPython/core/display_trap.py

@@ -24,8 +24,6 b' import sys'
 
 from IPython.core.component import Component
 
-from IPython.utils.autoattr import auto_attr
-
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------

IPython/core/embed.py

@@ -24,6 +24,7 b' Notes'
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
+import __main__
 
 import sys
 from contextlib import nested
@@ -33,7 +34,7 b' from IPython.core.iplib import InteractiveShell'
 from IPython.core.ipapp import load_default_config
 
 from IPython.utils.traitlets import Bool, Str, CBool
-from IPython.utils.genutils import ask_yes_no
+from IPython.utils.io import ask_yes_no
 
 
 #-----------------------------------------------------------------------------

IPython/core/excolors.py

@@ -10,8 +10,6 b' Color schemes for exception handling code in IPython.'
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
-#****************************************************************************
-# Required modules
 from IPython.utils.coloransi import ColorSchemeTable, TermColors, ColorScheme
 
 def exception_colors():

IPython/core/history.py

@@ -5,7 +5,8 b''
 import fnmatch
 import os
 
-from IPython.utils.genutils import Term, ask_yes_no, warn
+from IPython.utils.io import Term, ask_yes_no
+from IPython.utils.warn import warn
 from IPython.core import ipapi
 
 def magic_history(self, parameter_s = ''):

IPython/core/hooks.py

@@ -43,9 +43,12 b' somewhere in your configuration files or ipython command line.'
 
 import os, bisect
 import sys
-from IPython.utils.genutils import Term, shell
+
 from pprint import PrettyPrinter
 
+from IPython.utils.io import Term
+from IPython.utils.process import shell
+
 from IPython.core.error import TryNext
 
 # List here all the default hooks.  For now it's just the editor functions

IPython/core/ipapi.py

@@ -18,8 +18,6 b' has been made into a component, this module will be sent to deathrow.'
 # Imports
 #-----------------------------------------------------------------------------
 
-from IPython.core.error import TryNext, UsageError, IPythonCoreError
-
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------

IPython/core/ipapp.py

@@ -21,33 +21,60 b' Authors'
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
+
 from __future__ import absolute_import
 
 import logging
 import os
 import sys
 
-from IPython.core import crashhandler
-from IPython.core.application import Application
+from IPython.core import release
+from IPython.core.crashhandler import CrashHandler
+from IPython.core.application import Application, BaseAppConfigLoader
 from IPython.core.iplib import InteractiveShell
 from IPython.config.loader import (
     Config,
-    PyFileConfigLoader,
-#    NoConfigDefault,
+    PyFileConfigLoader
 )
 from IPython.lib import inputhook
-from IPython.utils.genutils import filefind, get_ipython_dir
+from IPython.utils.path import filefind, get_ipython_dir
 from . import usage
 
 #-----------------------------------------------------------------------------
 # Globals, utilities and helpers
 #-----------------------------------------------------------------------------
 
+#: The default config file name for this application.
 default_config_file_name = u'ipython_config.py'
 
-cl_args = (
-    (('--autocall',), dict(
-        type=int, dest='InteractiveShell.autocall',
+
+class IPAppConfigLoader(BaseAppConfigLoader):
+
+    def _add_arguments(self):
+        super(IPAppConfigLoader, self)._add_arguments()
+        paa = self.parser.add_argument
+        paa('-p', 
+            '--profile', dest='Global.profile', type=unicode,
+            help=
+            """The string name of the ipython profile to be used.  Assume that your
+            config file is ipython_config-<name>.py (looks in current dir first,
+            then in IPYTHON_DIR). This is a quick way to keep and load multiple
+            config files for different tasks, especially if include your basic one
+            in your more specialized ones.  You can keep a basic
+            IPYTHON_DIR/ipython_config.py file and then have other 'profiles' which
+            include this one and load extra things for particular tasks.""",
+            metavar='Global.profile')
+        paa('--config-file', 
+            dest='Global.config_file', type=unicode,
+            help=
+            """Set the config file name to override default.  Normally IPython
+            loads ipython_config.py (from current directory) or
+            IPYTHON_DIR/ipython_config.py.  If the loading of your config file
+            fails, IPython starts with a bare bones configuration (no modules
+            loaded at all).""",
+            metavar='Global.config_file')
+        paa('--autocall', 
+            dest='InteractiveShell.autocall', type=int, 
         help=
         """Make IPython automatically call any callable object even if you
         didn't type explicit parentheses. For example, 'str 43' becomes
@@ -57,41 +84,33 b' cl_args = ('
         objects are automatically called (even if no arguments are present).
         The default is '1'.""",
         metavar='InteractiveShell.autocall')
-    ),
-    (('--autoindent',), dict(
+        paa('--autoindent',
         action='store_true', dest='InteractiveShell.autoindent',
         help='Turn on autoindenting.')
-    ),
-    (('--no-autoindent',), dict(
+        paa('--no-autoindent',
         action='store_false', dest='InteractiveShell.autoindent',
         help='Turn off autoindenting.')
-    ),
-    (('--automagic',), dict(
+        paa('--automagic',
         action='store_true', dest='InteractiveShell.automagic',
-        help='Turn on the auto calling of magic commands.'
-        'Type %%magic at  the  IPython  prompt  for  more information.')
-     ),
-    (('--no-automagic',), dict(
+            help=
+            """Turn on the auto calling of magic commands. Type %%magic at the
+            IPython  prompt  for  more information.""")
+        paa('--no-automagic',
         action='store_false', dest='InteractiveShell.automagic',
         help='Turn off the auto calling of magic commands.')
-    ),
-    (('--autoedit-syntax',), dict(
+        paa('--autoedit-syntax',
         action='store_true', dest='InteractiveShell.autoedit_syntax',
         help='Turn on auto editing of files with syntax errors.')
-    ),
-    (('--no-autoedit-syntax',), dict(
+        paa('--no-autoedit-syntax',
         action='store_false', dest='InteractiveShell.autoedit_syntax',
         help='Turn off auto editing of files with syntax errors.')
-    ),
-    (('--banner',), dict(
+        paa('--banner',
         action='store_true', dest='Global.display_banner',
         help='Display a banner upon starting IPython.')
-    ),
-    (('--no-banner',), dict(
+        paa('--no-banner',
         action='store_false', dest='Global.display_banner',
         help="Don't display a banner upon starting IPython.")
-    ),
-    (('--cache-size',), dict(
+        paa('--cache-size',
         type=int, dest='InteractiveShell.cache_size',
         help=
         """Set the size of the output cache.  The default is 1000, you can
@@ -99,20 +118,16 b' cl_args = ('
         disables the caching system, and the minimum value accepted is 20 (if
         you provide a value less than 20, it is reset to 0 and a warning is
         issued).  This limit is defined because otherwise you'll spend more
-        time re-flushing a too small cache than working.
-        """,
+            time re-flushing a too small cache than working""",
         metavar='InteractiveShell.cache_size')
-    ),
-    (('--classic',), dict(
+        paa('--classic',
         action='store_true', dest='Global.classic',
         help="Gives IPython a similar feel to the classic Python prompt.")
-    ),
-    (('--colors',), dict(
+        paa('--colors',
         type=str, dest='InteractiveShell.colors',
         help="Set the color scheme (NoColor, Linux, and LightBG).",
         metavar='InteractiveShell.colors')
-    ),
-    (('--color-info',), dict(
+        paa('--color-info',
         action='store_true', dest='InteractiveShell.color_info',
         help=
         """IPython can display information about objects via a set of func-
@@ -123,27 +138,20 b' cl_args = ('
         it and turn it on permanently in your ipython_config.py file if it
         works for you.  Test it and turn it on permanently if it works with
         your system.  The magic function %%color_info allows you to toggle this
-        inter- actively for testing."""
-        )
-    ),
-    (('--no-color-info',), dict(
+            inter- actively for testing.""")
+        paa('--no-color-info',
         action='store_false', dest='InteractiveShell.color_info',
         help="Disable using colors for info related things.")
-    ),
-    (('--confirm-exit',), dict(
+        paa('--confirm-exit',
         action='store_true', dest='InteractiveShell.confirm_exit',
         help=
         """Set to confirm when you try to exit IPython with an EOF (Control-D
         in Unix, Control-Z/Enter in Windows). By typing 'exit', 'quit' or
-        '%%Exit', you can force a direct exit without any confirmation.
-        """
-        )
-    ),
-    (('--no-confirm-exit',), dict(
+            '%%Exit', you can force a direct exit without any confirmation.""")
+        paa('--no-confirm-exit',
         action='store_false', dest='InteractiveShell.confirm_exit',
         help="Don't prompt the user when exiting.")
-    ),
-    (('--deep-reload',), dict(
+        paa('--deep-reload',
         action='store_true', dest='InteractiveShell.deep_reload',
         help=
         """Enable deep (recursive) reloading by default. IPython can use the
@@ -155,47 +163,37 b' cl_args = ('
         deep_reload will still be available as dreload(). This fea- ture is off
         by default [which means that you have both normal reload() and
         dreload()].""")
-    ),
-    (('--no-deep-reload',), dict(
+        paa('--no-deep-reload',
         action='store_false', dest='InteractiveShell.deep_reload',
         help="Disable deep (recursive) reloading by default.")
-    ),
-    (('--editor',), dict(
+        paa('--editor',
         type=str, dest='InteractiveShell.editor',
         help="Set the editor used by IPython (default to $EDITOR/vi/notepad).",
         metavar='InteractiveShell.editor')
-    ),
-    (('--log','-l'), dict(
+        paa('--log','-l', 
         action='store_true', dest='InteractiveShell.logstart',
         help="Start logging to the default log file (./ipython_log.py).")
-    ),
-    (('--logfile','-lf'), dict(
+        paa('--logfile','-lf',
         type=unicode, dest='InteractiveShell.logfile',
         help="Start logging to logfile with this name.",
         metavar='InteractiveShell.logfile')
-    ),
-    (('--log-append','-la'), dict(
+        paa('--log-append','-la',
         type=unicode, dest='InteractiveShell.logappend',
         help="Start logging to the given file in append mode.",
         metavar='InteractiveShell.logfile')
-    ),
-    (('--pdb',), dict(
+        paa('--pdb',
         action='store_true', dest='InteractiveShell.pdb',
         help="Enable auto calling the pdb debugger after every exception.")
-    ),
-    (('--no-pdb',), dict(
+        paa('--no-pdb',
         action='store_false', dest='InteractiveShell.pdb',
         help="Disable auto calling the pdb debugger after every exception.")
-    ),
-    (('--pprint',), dict(
+        paa('--pprint',
         action='store_true', dest='InteractiveShell.pprint',
         help="Enable auto pretty printing of results.")
-    ),
-    (('--no-pprint',), dict(
+        paa('--no-pprint',
         action='store_false', dest='InteractiveShell.pprint',
         help="Disable auto auto pretty printing of results.")
-    ),
-    (('--prompt-in1','-pi1'), dict(
+        paa('--prompt-in1','-pi1',
         type=str, dest='InteractiveShell.prompt_in1',
         help=
         """Set the main input prompt ('In [\#]: ').  Note that if you are using
@@ -206,8 +204,7 b' cl_args = ('
         prompt escapes are described in detail in the Customization section of
         the IPython manual.""",
         metavar='InteractiveShell.prompt_in1')
-    ),
-    (('--prompt-in2','-pi2'), dict(
+        paa('--prompt-in2','-pi2',
         type=str, dest='InteractiveShell.prompt_in2',
         help=
         """Set the secondary input prompt (' .\D.: ').  Similar to the previous
@@ -217,25 +214,20 b' cl_args = ('
         Default: ' .\D.: ' (note three spaces at the start for alignment with
         'In [\#]')""",
         metavar='InteractiveShell.prompt_in2')
-    ),
-    (('--prompt-out','-po'), dict(
+        paa('--prompt-out','-po',
         type=str, dest='InteractiveShell.prompt_out',
         help="Set the output prompt ('Out[\#]:')",
         metavar='InteractiveShell.prompt_out')
-    ),
-    (('--quick',), dict(
+        paa('--quick',
         action='store_true', dest='Global.quick',
         help="Enable quick startup with no config files.")
-    ),
-    (('--readline',), dict(
+        paa('--readline',
         action='store_true', dest='InteractiveShell.readline_use',
         help="Enable readline for command line usage.")
-    ),
-    (('--no-readline',), dict(
+        paa('--no-readline',
         action='store_false', dest='InteractiveShell.readline_use',
         help="Disable readline for command line usage.")
-    ),
-    (('--screen-length','-sl'), dict(
+        paa('--screen-length','-sl',
         type=int, dest='InteractiveShell.screen_length',
         help=
         """Number of lines of your screen, used to control printing of very
@@ -248,35 +240,28 b' cl_args = ('
         curses support), specify it yourself. Otherwise don't change the
         default.""",
         metavar='InteractiveShell.screen_length')
-    ),
-    (('--separate-in','-si'), dict(
+        paa('--separate-in','-si',
         type=str, dest='InteractiveShell.separate_in',
         help="Separator before input prompts.  Default '\\n'.",
         metavar='InteractiveShell.separate_in')
-    ),
-    (('--separate-out','-so'), dict(
+        paa('--separate-out','-so',
         type=str, dest='InteractiveShell.separate_out',
         help="Separator before output prompts.  Default 0 (nothing).",
         metavar='InteractiveShell.separate_out')
-    ),
-    (('--separate-out2','-so2'), dict(
+        paa('--separate-out2','-so2', 
         type=str, dest='InteractiveShell.separate_out2',
         help="Separator after output prompts.  Default 0 (nonight).",
         metavar='InteractiveShell.separate_out2')
-    ),
-    (('-no-sep',), dict(
+        paa('--no-sep',
         action='store_true', dest='Global.nosep',
         help="Eliminate all spacing between prompts.")
-    ),
-    (('--term-title',), dict(
+        paa('--term-title',
         action='store_true', dest='InteractiveShell.term_title',
         help="Enable auto setting the terminal title.")
-    ),
-    (('--no-term-title',), dict(
+        paa('--no-term-title',
         action='store_false', dest='InteractiveShell.term_title',
         help="Disable auto setting the terminal title.")
-    ),
-    (('--xmode',), dict(
+        paa('--xmode',
         type=str, dest='InteractiveShell.xmode',
         help=
         """Exception reporting mode ('Plain','Context','Verbose').  Plain:
@@ -291,104 +276,121 b' cl_args = ('
         it more than once).
         """,
         metavar='InteractiveShell.xmode')
-    ),
-    (('--ext',), dict(
+        paa('--ext',
         type=str, dest='Global.extra_extension',
         help="The dotted module name of an IPython extension to load.",
         metavar='Global.extra_extension')
-    ),
-    (('-c',), dict(
+        paa('-c',
         type=str, dest='Global.code_to_run',
         help="Execute the given command string.",
         metavar='Global.code_to_run')
-    ),
-    (('-i',), dict(
+        paa('-i',
         action='store_true', dest='Global.force_interact',
         help=
-        "If running code from the command line, become interactive afterwards."
-        )
-    ),
+            "If running code from the command line, become interactive afterwards.")
 
     # Options to start with GUI control enabled from the beginning
-    (('--gui',), dict(
+        paa('--gui',
         type=str, dest='Global.gui',
         help="Enable GUI event loop integration ('qt', 'wx', 'gtk').",
         metavar='gui-mode')
-    ),
-
-    (('--pylab','-pylab'), dict(
+        paa('--pylab','-pylab',
         type=str, dest='Global.pylab',
         nargs='?', const='auto', metavar='gui-mode',
         help="Pre-load matplotlib and numpy for interactive use. "+
         "If no value is given, the gui backend is matplotlib's, else use "+
         "one of:  ['tk', 'qt', 'wx', 'gtk'].")
-    ),
     
     # Legacy GUI options.  Leave them in for backwards compatibility, but the
     # 'thread' names are really a misnomer now.
-    (('--wthread','-wthread'), dict(
+        paa('--wthread', '-wthread',
         action='store_true', dest='Global.wthread',
-        help="Enable wxPython event loop integration "+
-             "(DEPRECATED, use --gui wx)")
-    ),
-    (('--q4thread','--qthread','-q4thread','-qthread'), dict(
+            help=
+            """Enable wxPython event loop integration. (DEPRECATED, use --gui wx)""")
+        paa('--q4thread', '--qthread', '-q4thread', '-qthread',
         action='store_true', dest='Global.q4thread',
-        help="Enable Qt4 event loop integration. Qt3 is no longer supported. "+
-             "(DEPRECATED, use --gui qt)")
-    ),
-    (('--gthread','-gthread'), dict(
+            help=
+            """Enable Qt4 event loop integration. Qt3 is no longer supported.
+            (DEPRECATED, use --gui qt)""")
+        paa('--gthread', '-gthread',
         action='store_true', dest='Global.gthread',
-        help="Enable GTK event loop integration. "+
-             "(DEPRECATED, use --gui gtk)")
-    ),
-)
+            help=
+            """Enable GTK event loop integration. (DEPRECATED, use --gui gtk)""")
+
 
 #-----------------------------------------------------------------------------
-# Main classes and functions
+# Crash handler for this application
 #-----------------------------------------------------------------------------
 
-class IPythonApp(Application):
-    name = u'ipython'
-    #: argparse formats better the 'usage' than the 'description' field
-    description = None
-    #: usage message printed by argparse. If None, auto-generate
-    usage = usage.cl_usage
 
-    config_file_name = default_config_file_name
+_message_template = """\
+Oops, $self.app_name crashed. We do our best to make it stable, but...
 
-    cl_arguments = Application.cl_arguments + cl_args
+A crash report was automatically generated with the following information:
+  - A verbatim copy of the crash traceback.
+  - A copy of your input history during this session.
+  - Data on your current $self.app_name configuration.
 
-    # Private and configuration attributes
-    _CrashHandler = crashhandler.IPythonCrashHandler
+It was left in the file named:
+\t'$self.crash_report_fname'
+If you can email this file to the developers, the information in it will help
+them in understanding and correcting the problem.
     
-    def __init__(self, argv=None,
-                 constructor_config=None, override_config=None,
-                 **shell_params):
-        """Create a new IPythonApp.
+You can mail it to: $self.contact_name at $self.contact_email
+with the subject '$self.app_name Crash Report'.
 
-        See the parent class for details on how configuration is handled.
+If you want to do it now, the following command will work (under Unix):
+mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
 
-        Parameters
-        ----------
-        argv : optional, list
-          If given, used as the command-line argv environment to read arguments
-          from.
+To ensure accurate tracking of this issue, please file a report about it at:
+$self.bug_tracker
+"""
 
-        constructor_config : optional, Config
-          If given, additional config that is merged last, after internal
-          defaults, command-line and file-based configs.
+class IPAppCrashHandler(CrashHandler):
+    """sys.excepthook for IPython itself, leaves a detailed report on disk."""
 
-        override_config : optional, Config
-          If given, config that overrides all others unconditionally (except
-          for internal defaults, which ensure that all parameters exist).
+    message_template = _message_template
+
+    def __init__(self, app):
+        contact_name = release.authors['Fernando'][0]
+        contact_email = release.authors['Fernando'][1]
+        bug_tracker = 'https://bugs.launchpad.net/ipython/+filebug'
+        super(IPAppCrashHandler,self).__init__(
+            app, contact_name, contact_email, bug_tracker
+        )
+
+    def make_report(self,traceback):
+        """Return a string containing a crash report."""
+
+        sec_sep = self.section_sep
+        # Start with parent report
+        report = [super(IPAppCrashHandler, self).make_report(traceback)]
+        # Add interactive-specific info we may have
+        rpt_add = report.append
+        try:
+            rpt_add(sec_sep+"History of session input:")
+            for line in self.app.shell.user_ns['_ih']:
+                rpt_add(line)
+            rpt_add('\n*** Last line of input (may not be in above history):\n')
+            rpt_add(self.app.shell._last_input_line+'\n')
+        except:
+            pass
+
+        return ''.join(report)
 
-        shell_params : optional, dict
-          All other keywords are passed to the :class:`iplib.InteractiveShell`
-          constructor. 
-        """
-        super(IPythonApp, self).__init__(argv, constructor_config,
-                                         override_config)
-        self.shell_params = shell_params
+
+#-----------------------------------------------------------------------------
+# Main classes and functions
+#-----------------------------------------------------------------------------
+
+class IPythonApp(Application):
+    name = u'ipython'
+    #: argparse formats better the 'usage' than the 'description' field
+    description = None
+    usage = usage.cl_usage
+    command_line_loader = IPAppConfigLoader
+    default_config_file_name = default_config_file_name
+    crash_handler_class = IPAppCrashHandler
 
     def create_default_config(self):
         super(IPythonApp, self).create_default_config()
@@ -474,8 +476,7 b' class IPythonApp(Application):'
         sys.path.insert(0, '')
 
         # Create an InteractiveShell instance
-        self.shell = InteractiveShell(None, self.master_config,
-                                      **self.shell_params )
+        self.shell = InteractiveShell(None, self.master_config)
 
     def post_construct(self):
         """Do actions after construct, but before starting the app."""
@@ -485,7 +486,6 b' class IPythonApp(Application):'
         # based app, because we call shell.show_banner() by hand below
         # so the banner shows *before* all extension loading stuff.
         self.shell.display_banner = False
-
         if config.Global.display_banner and \
             config.Global.interact:
             self.shell.show_banner()
@@ -499,7 +499,6 b' class IPythonApp(Application):'
         self._run_exec_lines()
         self._run_exec_files()
         self._run_cmd_line_code()
-        self._configure_xmode()
 
     def _enable_gui_pylab(self):
         """Enable GUI event loop integration, taking pylab into account."""
@@ -624,11 +623,6 b' class IPythonApp(Application):'
                 self.log.warn("Error in executing file in user namespace: %s" % fname)
                 self.shell.showtraceback()
 
-    def _configure_xmode(self):
-        # XXX - shouldn't this be read from the config?  I'm still a little
-        # lost with all the details of handling the new config guys...
-        self.shell.InteractiveTB.set_mode(mode=self.shell.xmode)
-        
     def start_app(self):
         if self.master_config.Global.interact:
             self.log.debug("Starting IPython's mainloop...")
@@ -653,3 +647,7 b' def launch_new_instance():'
     """Create and run a full blown IPython instance"""
     app = IPythonApp()
     app.start()
+
+
+if __name__ == '__main__':
+    launch_new_instance()

IPython/core/iplib.py

@@ -20,7 +20,6 b' from __future__ import with_statement'
 from __future__ import absolute_import
 
 import __builtin__
-import StringIO
 import bdb
 import codeop
 import exceptions
@@ -47,29 +46,35 b' from IPython.core.logger import Logger'
 from IPython.core.magic import Magic
 from IPython.core.prefilter import PrefilterManager
 from IPython.core.prompts import CachedOutput
-from IPython.core.pylabtools import pylab_activate
 from IPython.core.usage import interactive_usage, default_banner
+import IPython.core.hooks
 from IPython.external.Itpl import ItplNS
 from IPython.lib.inputhook import enable_gui
 from IPython.lib.backgroundjobs import BackgroundJobManager
+from IPython.lib.pylabtools import pylab_activate
 from IPython.utils import PyColorize
 from IPython.utils import pickleshare
-from IPython.utils.genutils import get_ipython_dir
+from IPython.utils.doctestreload import doctest_reload
 from IPython.utils.ipstruct import Struct
-from IPython.utils.platutils import toggle_set_term_title, set_term_title
+from IPython.utils.io import Term, ask_yes_no
+from IPython.utils.path import get_home_dir, get_ipython_dir, HomeDirError
+from IPython.utils.process import (
+    abbrev_cwd,
+    getoutput,
+    getoutputerror
+)
+# import IPython.utils.rlineimpl as readline
 from IPython.utils.strdispatch import StrDispatch
 from IPython.utils.syspathcontext import prepended_to_syspath
-
-# XXX - need to clean up this import * line
-from IPython.utils.genutils import *
-
-# from IPython.utils import growl
-# growl.start("IPython")
-
+from IPython.utils.terminal import toggle_set_term_title, set_term_title
+from IPython.utils.warn import warn, error, fatal
 from IPython.utils.traitlets import (
     Int, Str, CBool, CaselessStrEnum, Enum, List, Unicode
 )
 
+# from IPython.utils import growl
+# growl.start("IPython")
+
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
@@ -186,64 +191,6 b' class SeparateStr(Str):'
         return super(SeparateStr, self).validate(obj, value)
 
 
-def make_user_namespaces(user_ns=None, user_global_ns=None):
-    """Return a valid local and global user interactive namespaces.
-
-    This builds a dict with the minimal information needed to operate as a
-    valid IPython user namespace, which you can pass to the various
-    embedding classes in ipython. The default implementation returns the
-    same dict for both the locals and the globals to allow functions to
-    refer to variables in the namespace. Customized implementations can
-    return different dicts. The locals dictionary can actually be anything
-    following the basic mapping protocol of a dict, but the globals dict
-    must be a true dict, not even a subclass. It is recommended that any
-    custom object for the locals namespace synchronize with the globals
-    dict somehow.
-
-    Raises TypeError if the provided globals namespace is not a true dict.
-
-    Parameters
-    ----------
-    user_ns : dict-like, optional
-        The current user namespace. The items in this namespace should
-        be included in the output. If None, an appropriate blank
-        namespace should be created.
-    user_global_ns : dict, optional
-        The current user global namespace. The items in this namespace
-        should be included in the output. If None, an appropriate
-        blank namespace should be created.
-
-    Returns
-    -------
-        A pair of dictionary-like object to be used as the local namespace
-        of the interpreter and a dict to be used as the global namespace.
-    """
-
-
-    # We must ensure that __builtin__ (without the final 's') is always
-    # available and pointing to the __builtin__ *module*.  For more details:
-    # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
-
-    if user_ns is None:
-        # Set __name__ to __main__ to better match the behavior of the
-        # normal interpreter.
-        user_ns = {'__name__'     :'__main__',
-                   '__builtin__' : __builtin__,
-                   '__builtins__' : __builtin__,
-                  }
-    else:
-        user_ns.setdefault('__name__','__main__')
-        user_ns.setdefault('__builtin__',__builtin__)
-        user_ns.setdefault('__builtins__',__builtin__)
-
-    if user_global_ns is None:
-        user_global_ns = user_ns
-    if type(user_global_ns) is not dict:
-        raise TypeError("user_global_ns must be a true dict; got %r"
-            % type(user_global_ns))
-
-    return user_ns, user_global_ns
-
 #-----------------------------------------------------------------------------
 # Main IPython class
 #-----------------------------------------------------------------------------
@@ -658,7 +605,6 b' class InteractiveShell(Component, Magic):'
         self.strdispatchers = {}
 
         # Set all default hooks, defined in the IPython.hooks module.
-        import IPython.core.hooks
         hooks = IPython.core.hooks
         for hook_name in hooks.__all__:
             # default hooks have priority 100, i.e. low; user hooks should have
@@ -876,7 +822,7 b' class InteractiveShell(Component, Magic):'
         # These routines return properly built dicts as needed by the rest of
         # the code, and can also be used by extension writers to generate
         # properly initialized namespaces.
-        user_ns, user_global_ns = make_user_namespaces(user_ns, user_global_ns)
+        user_ns, user_global_ns = self.make_user_namespaces(user_ns, user_global_ns)
 
         # Assign namespaces
         # This is the namespace where all normal user variables live
@@ -887,7 +833,7 b' class InteractiveShell(Component, Magic):'
         # loaded at startup, so we can list later only variables defined in
         # actual interactive use.  Since it is always a subset of user_ns, it
         # doesn't need to be separately tracked in the ns_table.
-        self.user_config_ns = {}
+        self.user_ns_hidden = {}
 
         # A namespace to keep track of internal data structures to prevent
         # them from cluttering user-visible stuff.  Will be updated later
@@ -933,9 +879,67 b' class InteractiveShell(Component, Magic):'
         # Similarly, track all namespaces where references can be held and that
         # we can safely clear (so it can NOT include builtin).  This one can be
         # a simple list.
-        self.ns_refs_table = [ user_ns, user_global_ns, self.user_config_ns,
+        self.ns_refs_table = [ user_ns, user_global_ns, self.user_ns_hidden,
                                self.internal_ns, self._main_ns_cache ]
 
+    def make_user_namespaces(self, user_ns=None, user_global_ns=None):
+        """Return a valid local and global user interactive namespaces.
+
+        This builds a dict with the minimal information needed to operate as a
+        valid IPython user namespace, which you can pass to the various
+        embedding classes in ipython. The default implementation returns the
+        same dict for both the locals and the globals to allow functions to
+        refer to variables in the namespace. Customized implementations can
+        return different dicts. The locals dictionary can actually be anything
+        following the basic mapping protocol of a dict, but the globals dict
+        must be a true dict, not even a subclass. It is recommended that any
+        custom object for the locals namespace synchronize with the globals
+        dict somehow.
+
+        Raises TypeError if the provided globals namespace is not a true dict.
+
+        Parameters
+        ----------
+        user_ns : dict-like, optional
+            The current user namespace. The items in this namespace should
+            be included in the output. If None, an appropriate blank
+            namespace should be created.
+        user_global_ns : dict, optional
+            The current user global namespace. The items in this namespace
+            should be included in the output. If None, an appropriate
+            blank namespace should be created.
+
+        Returns
+        -------
+            A pair of dictionary-like object to be used as the local namespace
+            of the interpreter and a dict to be used as the global namespace.
+        """
+
+
+        # We must ensure that __builtin__ (without the final 's') is always
+        # available and pointing to the __builtin__ *module*.  For more details:
+        # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
+
+        if user_ns is None:
+            # Set __name__ to __main__ to better match the behavior of the
+            # normal interpreter.
+            user_ns = {'__name__'     :'__main__',
+                       '__builtin__' : __builtin__,
+                       '__builtins__' : __builtin__,
+                      }
+        else:
+            user_ns.setdefault('__name__','__main__')
+            user_ns.setdefault('__builtin__',__builtin__)
+            user_ns.setdefault('__builtins__',__builtin__)
+
+        if user_global_ns is None:
+            user_global_ns = user_ns
+        if type(user_global_ns) is not dict:
+            raise TypeError("user_global_ns must be a true dict; got %r"
+                % type(user_global_ns))
+
+        return user_ns, user_global_ns
+
     def init_sys_modules(self):
         # We need to insert into sys.modules something that looks like a
         # module but which accesses the IPython namespace, for shelve and
@@ -974,7 +978,7 b' class InteractiveShell(Component, Magic):'
         therm.
         """
         # This function works in two parts: first we put a few things in
-        # user_ns, and we sync that contents into user_config_ns so that these
+        # user_ns, and we sync that contents into user_ns_hidden so that these
         # initial variables aren't shown by %who.  After the sync, we add the
         # rest of what we *do* want the user to see with %who even on a new
         # session (probably nothing, so theye really only see their own stuff)
@@ -1014,9 +1018,9 b' class InteractiveShell(Component, Magic):'
         # Store myself as the public api!!!
         ns['get_ipython'] = self.get_ipython
 
-        # Sync what we've added so far to user_config_ns so these aren't seen
+        # Sync what we've added so far to user_ns_hidden so these aren't seen
         # by %who
-        self.user_config_ns.update(ns)
+        self.user_ns_hidden.update(ns)
 
         # Anything put into ns now would show up in %who.  Think twice before
         # putting anything here, as we really want %who to show the user their
@@ -1089,7 +1093,7 b' class InteractiveShell(Component, Magic):'
         self.user_ns.update(vdict)
 
         # And configure interactive visibility
-        config_ns = self.user_config_ns
+        config_ns = self.user_ns_hidden
         if interactive:
             for name, val in vdict.iteritems():
                 config_ns.pop(name, None)
@@ -1164,7 +1168,9 b' class InteractiveShell(Component, Magic):'
         Convert func into callable that saves & restores
         history around the call """
 
-        if not self.has_readline:
+        if self.has_readline:
+            from IPython.utils import rlineimpl as readline
+        else:
             return func
 
         def wrapper():
@@ -1198,6 +1204,9 b' class InteractiveShell(Component, Magic):'
         # and add any custom exception handlers the user may have specified
         self.set_custom_exc(*custom_exceptions)
 
+        # Set the exception mode
+        self.InteractiveTB.set_mode(mode=self.xmode)
+
     def set_custom_exc(self,exc_tuple,handler):
         """set_custom_exc(exc_tuple,handler)
 
@@ -2351,6 +2360,9 b' class InteractiveShell(Component, Magic):'
         to make it easy to write extensions, you can also put your extensions
         in ``os.path.join(self.ipython_dir, 'extensions')``.  This directory
         is added to ``sys.path`` automatically.
+
+        If :func:`load_ipython_extension` returns anything, this function
+        will return that object.
         """
         from IPython.utils.syspathcontext import prepended_to_syspath
 
@@ -2495,11 +2507,11 b' class InteractiveShell(Component, Magic):'
         # We want to prevent the loading of pylab to pollute the user's
         # namespace as shown by the %who* magics, so we execute the activation
         # code in an empty namespace, and we update *both* user_ns and
-        # user_config_ns with this information.
+        # user_ns_hidden with this information.
         ns = {}
         gui = pylab_activate(ns, gui)
         self.user_ns.update(ns)
-        self.user_config_ns.update(ns)
+        self.user_ns_hidden.update(ns)
         # Now we must activate the gui pylab wants to use, and fix %run to take
         # plot updates into account
         enable_gui(gui)

IPython/core/macro.py

@@ -7,7 +7,7 b''
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
-from IPython.utils.genutils import Term
+from IPython.utils.io import Term
 from IPython.core.autocall import IPyAutocall
 
 class Macro(IPyAutocall):

IPython/core/magic.py

@@ -1,35 +1,33 b''
-# -*- coding: utf-8 -*-
+# encoding: utf-8
 """Magic functions for InteractiveShell.
 """
 
-#*****************************************************************************
+#-----------------------------------------------------------------------------
 #       Copyright (C) 2001 Janko Hauser <[e-mail]> and
-#       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
-#
+#  Copyright (C) 2001-2007 Fernando Perez <[e-mail]>
+#  Copyright (C) 2008-2009  The IPython Development Team
+
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#*****************************************************************************
+#-----------------------------------------------------------------------------
 
-#****************************************************************************
-# Modules and globals
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
 
-# Python standard modules
 import __builtin__
 import bdb
 import inspect
 import os
-import pdb
-import pydoc
 import sys
 import shutil
 import re
-import tempfile
 import time
-import cPickle as pickle
 import textwrap
+import types
 from cStringIO import StringIO
 from getopt import getopt,GetoptError
-from pprint import pprint, pformat
+from pprint import pformat
 
 # cProfile was added in Python2.5
 try:
@@ -42,10 +40,7 b' except ImportError:'
     except ImportError:
         profile = pstats = None
 
-# Homebrewed
 import IPython
-import IPython.utils.generics
-
 from IPython.core import debugger, oinspect
 from IPython.core.error import TryNext
 from IPython.core.error import UsageError
@@ -53,20 +48,24 b' from IPython.core.fakemodule import FakeModule'
 from IPython.core.macro import Macro
 from IPython.core.page import page
 from IPython.core.prefilter import ESC_MAGIC
-from IPython.core.pylabtools import mpl_runner
+from IPython.lib.pylabtools import mpl_runner
 from IPython.lib.inputhook import enable_gui
-from IPython.external.Itpl import Itpl, itpl, printpl,itplns
+from IPython.external.Itpl import itpl, printpl
 from IPython.testing import decorators as testdec
-from IPython.utils import platutils
-from IPython.utils import wildcard
-from IPython.utils.PyColorize import Parser
+from IPython.utils.io import Term, file_read, nlprint
+from IPython.utils.path import get_py_filename
+from IPython.utils.process import arg_split, abbrev_cwd
+from IPython.utils.terminal import set_term_title
+from IPython.utils.text import LSString, SList, StringTypes
+from IPython.utils.timing import clock, clock2
+from IPython.utils.warn import warn, error
 from IPython.utils.ipstruct import Struct
+import IPython.utils.generics
 
-# XXX - We need to switch to explicit imports here with genutils
-from IPython.utils.genutils import *
-
-#***************************************************************************
+#-----------------------------------------------------------------------------
 # Utility functions
+#-----------------------------------------------------------------------------
+
 def on_off(tag):
     """Return an ON/OFF string for a 1/0 input. Simple utility function."""
     return ['OFF','ON'][tag]
@@ -94,6 +93,9 b' def compress_dhist(dh):'
 # on construction of the main InteractiveShell object.  Something odd is going
 # on with super() calls, Component and the MRO... For now leave it as-is, but
 # eventually this needs to be clarified.
+# BG: This is because InteractiveShell inherits from this, but is itself a 
+# Component. This messes up the MRO in some way. The fix is that we need to
+# make Magic a component that InteractiveShell does not subclass.
 
 class Magic:
     """Magic functions for InteractiveShell.
@@ -277,7 +279,7 b' python-profiler package from non-free.""")'
     def arg_err(self,func):
         """Print docstring if incorrect arguments were passed"""
         print 'Error in arguments:'
-        print OInspect.getdoc(func)
+        print oinspect.getdoc(func)
 
     def format_latex(self,strng):
         """Format a string for latex inclusion."""
@@ -884,10 +886,10 b' Currently the magic system has the following functions:\\n"""'
 
         user_ns = self.shell.user_ns
         internal_ns = self.shell.internal_ns
-        user_config_ns = self.shell.user_config_ns
+        user_ns_hidden = self.shell.user_ns_hidden
         out = [ i for i in user_ns
                 if not i.startswith('_') \
-                and not (i in internal_ns or i in user_config_ns) ]
+                and not (i in internal_ns or i in user_ns_hidden) ]
 
         typelist = parameter_s.split()
         if typelist:
@@ -1170,7 +1172,7 b' Currently the magic system has the following functions:\\n"""'
             started  = logger.logstart(logfname,loghead,logmode,
                                        log_output,timestamp,log_raw_input)
         except:
-            rc.opts.logfile = old_logfile
+            self.shell.logfile = old_logfile
             warn("Couldn't start log: %s" % sys.exc_info()[1])
         else:
             # log input history up to this point, optionally interleaving
@@ -2811,7 +2813,7 b' Defaulting color scheme to \'NoColor\'"""'
             try:                
                 os.chdir(os.path.expanduser(ps))
                 if self.shell.term_title:
-                    platutils.set_term_title('IPython: ' + abbrev_cwd())
+                    set_term_title('IPython: ' + abbrev_cwd())
             except OSError:
                 print sys.exc_info()[1]
             else:
@@ -2824,7 +2826,7 b' Defaulting color scheme to \'NoColor\'"""'
         else:
             os.chdir(self.shell.home_dir)
             if self.shell.term_title:
-                platutils.set_term_title('IPython: ' + '~')
+                set_term_title('IPython: ' + '~')
             cwd = os.getcwd()
             dhist = self.shell.user_ns['_dh']
             

IPython/core/oinspect.py

@@ -27,10 +27,11 b' import sys'
 import types
 
 # IPython's own
-from IPython.utils import PyColorize
-from IPython.utils.genutils import indent, Term
 from IPython.core.page import page
 from IPython.external.Itpl import itpl
+from IPython.utils import PyColorize
+from IPython.utils.io import Term
+from IPython.utils.text import indent
 from IPython.utils.wildcard import list_namespace
 from IPython.utils.coloransi import *
 

IPython/core/page.py

@@ -30,15 +30,15 b' rid of that dependency, we could move it there.'
 import os
 import re
 import sys
+import tempfile
 
 from IPython.core import ipapi
 from IPython.core.error import TryNext
-from IPython.utils.genutils import (
-    chop, Term, USE_CURSES
-)
-
-if os.name == "nt":
-    from IPython.utils.winconsole import get_console_size
+from IPython.utils.cursesimport import use_curses
+from IPython.utils.data import chop
+from IPython.utils.io import Term
+from IPython.utils.process import xsys
+from IPython.utils.terminal import get_terminal_size
 
 
 #-----------------------------------------------------------------------------
@@ -69,7 +69,7 b' def page_dumb(strng,start=0,screen_lines=25):'
                 last_escape = esc_list[-1]
         print >>Term.cout, last_escape + os.linesep.join(screens[-1])
 
-#----------------------------------------------------------------------------
+
 def page(strng,start=0,screen_lines=0,pager_cmd = None):
     """Print a string, piping through a pager after a certain length.
 
@@ -120,19 +120,16 b' def page(strng,start=0,screen_lines=0,pager_cmd = None):'
     # terminals. If someone later feels like refining it, it's not hard.
     numlines = max(num_newlines,int(len_str/80)+1)
 
-    if os.name == "nt":
-        screen_lines_def = get_console_size(defaulty=25)[1]
-    else:
-        screen_lines_def = 25 # default value if we can't auto-determine
+    screen_lines_def = get_terminal_size()[1]
 
     # auto-determine screen size
     if screen_lines <= 0:
         if TERM=='xterm' or TERM=='xterm-color':
-            use_curses = USE_CURSES
+            local_use_curses = use_curses
         else:
             # curses causes problems on many terminals other than xterm.
-            use_curses = False
-        if use_curses:
+            local_use_curses = False
+        if local_use_curses:
             import termios
             import curses
             # There is a bug in curses, where *sometimes* it fails to properly
@@ -201,7 +198,7 b' def page(strng,start=0,screen_lines=0,pager_cmd = None):'
         if retval is not None:
             page_dumb(strng,screen_lines=screen_lines)
 
-#----------------------------------------------------------------------------
+
 def page_file(fname,start = 0, pager_cmd = None):
     """Page a file, using an optional pager command and starting line.
     """
@@ -221,12 +218,12 b' def page_file(fname,start = 0, pager_cmd = None):'
         except:
             print 'Unable to show file',`fname`
 
-#----------------------------------------------------------------------------
+
 def get_pager_cmd(pager_cmd = None):
     """Return a pager command.
 
-    Makes some attempts at finding an OS-correct one."""
-
+    Makes some attempts at finding an OS-correct one.
+    """
     if os.name == 'posix':
         default_pager_cmd = 'less -r'  # -r for color control sequences
     elif os.name in ['nt','dos']:
@@ -239,7 +236,7 b' def get_pager_cmd(pager_cmd = None):'
             pager_cmd = default_pager_cmd
     return pager_cmd
 
-#-----------------------------------------------------------------------------
+
 def get_pager_start(pager,start):
     """Return the string for paging files with an offset.
 
@@ -255,8 +252,8 b' def get_pager_start(pager,start):'
         start_string = ''
     return start_string
 
-#----------------------------------------------------------------------------
-# (X)emacs on W32 doesn't like to be bypassed with msvcrt.getch()
+
+# (X)emacs on win32 doesn't like to be bypassed with msvcrt.getch()
 if os.name == 'nt' and os.environ.get('TERM','dumb') != 'emacs':
     import msvcrt
     def page_more():
@@ -280,7 +277,7 b' else:'
         else:
             return True
 
-#----------------------------------------------------------------------------
+
 def snip_print(str,width = 75,print_full = 0,header = ''):
     """Print a string snipping the midsection to fit in width.
 
@@ -306,3 +303,4 b" def snip_print(str,width = 75,print_full = 0,header = ''):"
         if raw_input(header+' Snipped. View (y/n)? [N]').lower() == 'y':
             page(str)
     return snip
+

IPython/core/prefilter.py

@@ -27,10 +27,7 b' Authors:'
 
 import __builtin__
 import codeop
-import keyword
-import os
 import re
-import sys
 
 from IPython.core.alias import AliasManager
 from IPython.core.autocall import IPyAutocall
@@ -39,7 +36,8 b' from IPython.core.splitinput import split_user_input'
 from IPython.core.page import page
 
 from IPython.utils.traitlets import List, Int, Any, Str, CBool, Bool
-from IPython.utils.genutils import make_quoted_expr, Term
+from IPython.utils.io import Term
+from IPython.utils.text import make_quoted_expr
 from IPython.utils.autoattr import auto_attr
 
 #-----------------------------------------------------------------------------
@@ -158,6 +156,7 b' class LineInfo(object):'
         without worrying about *further* damaging state.
         """
         if not self._oinfo:
+            # ip.shell._ofind is actually on the Magic class!
             self._oinfo = ip.shell._ofind(self.ifun)
         return self._oinfo
 

IPython/core/prompts.py

@@ -12,23 +12,20 b' Classes for handling input/output prompts.'
 #*****************************************************************************
 
 #****************************************************************************
-# Required modules
+
 import __builtin__
 import os
+import re
 import socket
 import sys
-import time
 
-# IPython's own
-from IPython.utils import coloransi
 from IPython.core import release
 from IPython.external.Itpl import ItplNS
 from IPython.core.error import TryNext
-from IPython.utils.ipstruct import Struct
-from IPython.core.macro import Macro
+from IPython.utils import coloransi
 import IPython.utils.generics
-
-from IPython.utils.genutils import *
+from IPython.utils.warn import warn
+from IPython.utils.io import Term
 
 #****************************************************************************
 #Color schemes for Prompts.

IPython/core/quitter.py

@@ -19,7 +19,11 b' Authors'
 # Imports
 #-----------------------------------------------------------------------------
 
-import sys
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
 
 class Quitter(object):
     """Simple class to handle exit, similar to Python 2.5's.
@@ -40,4 +44,4 b' class Quitter(object):'
     # Repr MUST return a string, else display like pprint hooks get confused
     def __repr__(self):
         self.shell.ask_exit()
-        return 'Bye.'
+        return ''

IPython/core/release.py

@@ -21,9 +21,9 b" name = 'ipython'"
 # bdist_deb does not accept underscores (a Debian convention).
 
 development = True    # change this to False to do a release
-version_base = '0.11'
+version_base = '0.11.alpha1'
 branch = 'ipython'
-revision = '1346'
+revision = '1223'
 
 if development:
     if branch == 'ipython':

IPython/core/tests/test_iplib.py

@@ -30,9 +30,9 b' ip = get_ipython()'
 @dec.parametric
 def test_reset():
     """reset must clear most namespaces."""
-    # The number of variables in the private user_config_ns is not zero, but it
+    # The number of variables in the private user_ns_hidden is not zero, but it
     # should be constant regardless of what we do
-    nvars_config_ns = len(ip.user_config_ns)
+    nvars_config_ns = len(ip.user_ns_hidden)
 
     # Check that reset runs without error
     ip.reset()
@@ -51,7 +51,7 b' def test_reset():'
     for ns in ip.ns_refs_table:
         if ns is ip.user_ns:
             nvars_expected = nvars_user_ns
-        elif ns is ip.user_config_ns:
+        elif ns is ip.user_ns_hidden:
             nvars_expected = nvars_config_ns
         else:
             nvars_expected = 0

IPython/core/tests/test_magic.py

@@ -8,19 +8,15 b' from __future__ import absolute_import'
 # Imports
 #-----------------------------------------------------------------------------
 
-# stdlib
 import os
 import sys
 import tempfile
 import types
 from cStringIO import StringIO
 
-# third-party
 import nose.tools as nt
 
-# our own
-from IPython.utils import genutils
-from IPython.utils.platutils import find_cmd, get_long_path_name
+from IPython.utils.path import get_long_path_name
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 

IPython/core/tests/test_run.py

@@ -12,17 +12,12 b' from __future__ import absolute_import'
 # Imports
 #-----------------------------------------------------------------------------
 
-# stdlib
 import os
 import sys
 import tempfile
 
-# third-party
 import nose.tools as nt
 
-# our own
-from IPython.utils.platutils import find_cmd
-from IPython.utils import genutils
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 
@@ -142,10 +137,10 b' class TestMagicRunSimple(tt.TempFileMixin):'
         _ip.runlines('t = isinstance(f(), foo)')
         nt.assert_true(_ip.user_ns['t'])
 
-    # We have to skip these in win32 because genutils.getoutputerr() crashes,
+    # We have to skip these in win32 because getoutputerr() crashes,
     # due to the fact that subprocess does not support close_fds when
     # redirecting stdout/err.  So unless someone who knows more tells us how to
-    # implement genutils.getoutputerr() in win32, we're stuck avoiding these.
+    # implement getoutputerr() in win32, we're stuck avoiding these.
     @dec.skip_win32
     def test_obj_del(self):
         """Test that object's __del__ methods are called on exit."""

IPython/core/ultratb.py

@@ -93,9 +93,10 b' from inspect import getsourcefile, getfile, getmodule,\\'
 from IPython.utils import PyColorize
 from IPython.core import debugger, ipapi
 from IPython.core.display_trap import DisplayTrap
-from IPython.utils.ipstruct import Struct
 from IPython.core.excolors import exception_colors
-from IPython.utils.genutils import Term, uniq_stable, error, info
+from IPython.utils.data import uniq_stable
+from IPython.utils.io import Term
+from IPython.utils.warn import info, error
 
 # Globals
 # amount of space to put line numbers before verbose tracebacks
@@ -384,7 +385,8 b' class ListTB(TBTools):'
         
     def __call__(self, etype, value, elist):
         Term.cout.flush()
-        Term.cerr.writeln(self.text(etype,value,elist))
+        Term.cerr.write(self.text(etype,value,elist))
+        Term.cerr.write('\n')
 
     def text(self, etype, value, elist, context=5):
         """Return a color formatted string with the traceback info.
@@ -909,7 +911,8 b' class VerboseTB(TBTools):'
         (etype, evalue, etb) = info or sys.exc_info()
         self.tb = etb
         Term.cout.flush()
-        Term.cerr.writeln(self.text(etype, evalue, etb))
+        Term.cerr.write(self.text(etype, evalue, etb))
+        Term.cerr.write('\n')
 
     # Changed so an instance can just be called as VerboseTB_inst() and print
     # out the right info on its own.

IPython/deathrow/GnuplotRuntime.py

@@ -53,7 +53,7 b" __all__ = ['Gnuplot','gp','gp_new','Data','File','Func','GridData',"
            'pm3d_config','eps_fix_bbox']
 
 import os,tempfile,sys
-from IPython.utils.genutils import getoutput
+from IPython.utils.process import getoutput
 
 #---------------------------------------------------------------------------
 # Notes on mouse support for Gnuplot.py

IPython/deathrow/ipipe.py

@@ -133,10 +133,10 b' from IPython.external import simplegeneric'
 from IPython.external import path
 
 try:
-    from IPython.utils import genutils
+    from IPython.utils.io import Term
     from IPython.utils import generics
 except ImportError:
-    genutils = None
+    Term = None
     generics = None
 
 from IPython.core import ipapi
@@ -2168,7 +2168,7 b' class idump(Display):'
         self.datasepchar = "|"
 
     def display(self):
-        stream = genutils.Term.cout
+        stream = Term.cout
         allattrs = []
         attrset = set()
         colwidths = {}

IPython/deathrow/ipy_traits_completer.py

@@ -54,7 +54,7 b' from enthought.traits import api as T'
 # IPython imports
 from IPython.core.error import TryNext
 from IPython.core.ipapi import get as ipget
-from IPython.utils.genutils import dir2
+from IPython.utils.dir2 import dir2
 try:
     set
 except:

IPython/deathrow/twshell.py

@@ -14,7 +14,9 b' from IPython.core.iplib import InteractiveShell'
 from IPython.utils.ipstruct import Struct
 import Queue,thread,threading,signal
 from signal import signal, SIGINT
-from IPython.utils.genutils import Term,warn,error,flag_calls, ask_yes_no
+from IPython.utils.io import Term, ask_yes_no
+from IPython.utils.warn import warn, error
+from IPython.utils.decorators import flag_calls
 from IPython.core import shellglobals
 
 def install_gtk2():

IPython/extensions/pretty.py

@@ -39,7 +39,7 b' from IPython.core.error import TryNext'
 from IPython.external import pretty
 from IPython.core.component import Component
 from IPython.utils.traitlets import Bool, List
-from IPython.utils.genutils import Term
+from IPython.utils.io import Term
 from IPython.utils.autoattr import auto_attr
 from IPython.utils.importstring import import_item
 
@@ -128,9 +128,8 b' class PrettyResultDisplay(Component):'
 #-----------------------------------------------------------------------------
 
 
-def load_ipython_extension(ip=None):
+def load_ipython_extension(ip):
     """Load the extension in IPython as a hook."""
-    if ip is None: ip = get_ipython()
     global _loaded
     if not _loaded:
         prd = PrettyResultDisplay(ip, name='pretty_result_display')

IPython/external/mglob.py

@@ -213,7 +213,7 b' def main():'
     print "\n".join(expand(sys.argv[1:])),
 
 def mglob_f(self, arg):
-    from IPython.utils.genutils import SList
+    from IPython.utils.text import SList
     if arg.strip():
         return SList(expand(arg))
     print "Please specify pattern!"

IPython/frontend/linefrontendbase.py

@@ -39,9 +39,10 b' def common_prefix(strings):'
 
     return prefix
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Base class for the line-oriented front ends
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+
 class LineFrontEndBase(FrontEndBase):
     """ Concrete implementation of the FrontEndBase class. This is meant
     to be the base class behind all the frontend that are line-oriented,

IPython/frontend/prefilterfrontend.py

@@ -26,12 +26,12 b' import os'
 import re
 import __builtin__
 
-from IPython.core.ipapp import IPythonApp
+from IPython.core.iplib import InteractiveShell
 from IPython.kernel.core.redirector_output_trap import RedirectorOutputTrap
 
 from IPython.kernel.core.sync_traceback_trap import SyncTracebackTrap
 
-from IPython.utils.genutils import Term
+from IPython.utils.io import Term
 
 from linefrontendbase import LineFrontEndBase, common_prefix
 
@@ -50,9 +50,10 b' def mk_system_call(system_call_function, command):'
     my_system_call.__doc__ = "Calls %s" % command
     return my_system_call
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Frontend class using ipython0 to do the prefiltering. 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+
 class PrefilterFrontEnd(LineFrontEndBase):
     """ Class that uses ipython0 to do prefilter the input, do the
     completion and the magics.
@@ -65,19 +66,13 b' class PrefilterFrontEnd(LineFrontEndBase):'
 
     debug = False
     
-    def __init__(self, ipython0=None, argv=None, *args, **kwargs):
+    def __init__(self, ipython0=None, *args, **kwargs):
         """ Parameters
             ----------
 
             ipython0: an optional ipython0 instance to use for command
             prefiltering and completion.
-
-            argv : list, optional
-              Used as the instance's argv value.  If not given, [] is used.
         """
-        if argv is None:
-            argv = ['--no-banner']
-
         LineFrontEndBase.__init__(self, *args, **kwargs)
         self.shell.output_trap = RedirectorOutputTrap(
                             out_callback=self.write,
@@ -90,22 +85,19 b' class PrefilterFrontEnd(LineFrontEndBase):'
         # Start the ipython0 instance:
         self.save_output_hooks()
         if ipython0 is None:
-            # Instanciate an IPython0 interpreter to be able to use the
+            # Instanciate an IPython0 InteractiveShell to be able to use the
             # prefiltering.
             # Suppress all key input, to avoid waiting
             def my_rawinput(x=None):
                 return '\n'
             old_rawinput = __builtin__.raw_input
             __builtin__.raw_input = my_rawinput
-            ipython0 = IPythonApp(argv=argv, 
-                                  user_ns=self.shell.user_ns,
-                                  user_global_ns=self.shell.user_global_ns)
-            ipython0.initialize()
+            ipython0 = InteractiveShell(
+                parent=None, user_ns=self.shell.user_ns,
+                user_global_ns=self.shell.user_global_ns
+            )
             __builtin__.raw_input = old_rawinput
-        # XXX This will need to be updated as we refactor things, but for now,
-        # the .shell attribute of the ipythonapp instance conforms to the old
-        # api.
-        self.ipython0 = ipython0.shell
+        self.ipython0 = ipython0
         # Set the pager:
         self.ipython0.set_hook('show_in_pager', 
                     lambda s, string: self.write("\n" + string))

IPython/frontend/tests/test_asyncfrontendbase.py

@@ -1,14 +1,8 b''
 # encoding: utf-8
-
 """This file contains unittests for the asyncfrontendbase module."""
 
-__docformat__ = "restructuredtext en"
-                                                                            
-# Tell nose to skip this module
-__test__ = {}
-
 #---------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team                         
+#  Copyright (C) 2008-2009  The IPython Development Team                         
 #                                                                           
 #  Distributed under the terms of the BSD License.  The full license is in  
 #  the file COPYING, distributed as part of this software.                  

IPython/frontend/tests/test_prefilterfrontend.py

@@ -21,7 +21,6 b' from nose.tools import assert_equal'
 
 from IPython.frontend.prefilterfrontend import PrefilterFrontEnd
 from IPython.testing.globalipapp import get_ipython
-from IPython.testing.tools import default_argv
 
 #-----------------------------------------------------------------------------
 # Support utilities
@@ -35,7 +34,7 b' class TestPrefilterFrontEnd(PrefilterFrontEnd):'
 
     def __init__(self):
         self.out = StringIO()
-        PrefilterFrontEnd.__init__(self,argv=default_argv())
+        PrefilterFrontEnd.__init__(self)
         # Some more code for isolation (yeah, crazy)
         self._on_enter()
         self.out.flush()

IPython/frontend/tests/test_process.py

@@ -47,6 +47,7 b' def test_io():'
     assert result == test_string
 
 
[e-mail]_win32
 def test_kill():
     """ Check that we can kill a process, and its subprocess.
     """

IPython/frontend/wx/console_widget.py

@@ -135,9 +135,10 b' else:'
                 }
  
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # The console widget class
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+
 class ConsoleWidget(editwindow.EditWindow):
     """ Specialized styled text control view for console-like workflow.
 

IPython/frontend/wx/ipythonx.py

@@ -47,7 +47,7 b' class IPythonXController(WxController):'
         self._input_state = 'subprocess'
         self.write('\n', refresh=False)
         self.capture_output()
-        self.ipython0.shell.exit()
+        self.ipython0.exit()
         self.release_output()
         if not self.ipython0.exit_now:
             wx.CallAfter(self.new_prompt,

IPython/gui/wx/ipshell_nonblocking.py

@@ -23,9 +23,8 b' import os'
 import locale
 from thread_ex import ThreadEx
 
-import IPython
-from IPython.core import iplib, ipapp
-from IPython.utils import genutils
+from IPython.core import iplib
+from IPython.utils.io import Term
 
 ##############################################################################
 class _Helper(object):
@@ -88,12 +87,10 b' class NonBlockingIPShell(object):'
           via raise_exc()
     '''
 
-    def __init__(self, argv=[], user_ns={}, user_global_ns=None,
+    def __init__(self, user_ns={}, user_global_ns=None,
                  cin=None, cout=None, cerr=None,
                  ask_exit_handler=None):
         '''
-        @param argv: Command line options for IPython
-        @type argv: list
         @param user_ns: User namespace.
         @type user_ns: dictionary
         @param user_global_ns: User global namespace.
@@ -111,7 +108,7 b' class NonBlockingIPShell(object):'
         '''
         #ipython0 initialisation
         self._IP = None
-        self.init_ipython0(argv, user_ns, user_global_ns,
+        self.init_ipython0(user_ns, user_global_ns,
                           cin, cout, cerr,
                           ask_exit_handler)
         
@@ -131,7 +128,7 b' class NonBlockingIPShell(object):'
         self._help_text = None
         self._add_button = None
 
-    def init_ipython0(self, argv=[], user_ns={}, user_global_ns=None,
+    def init_ipython0(self, user_ns={}, user_global_ns=None,
                      cin=None, cout=None, cerr=None,
                      ask_exit_handler=None):
         ''' Initialize an ipython0 instance '''
@@ -141,27 +138,22 b' class NonBlockingIPShell(object):'
         #only one instance can be instanciated else tehre will be
         #cin/cout/cerr clash...
         if cin:
-            genutils.Term.cin = cin
+            Term.cin = cin
         if cout:
-            genutils.Term.cout = cout
+            Term.cout = cout
         if cerr:
-            genutils.Term.cerr = cerr
+            Term.cerr = cerr
         
         excepthook = sys.excepthook
 
         #Hack to save sys.displayhook, because ipython seems to overwrite it...
         self.sys_displayhook_ori = sys.displayhook
-
-        ipython0 = ipapp.IPythonApp(argv,user_ns=user_ns,
-                                    user_global_ns=user_global_ns)
-        ipython0.initialize()
-        self._IP = ipython0.shell
-        
-        ## self._IP = IPython.shell.make_IPython(
-        ##                             argv,user_ns=user_ns,
-        ##                             user_global_ns=user_global_ns,
-        ##                             embedded=True,
-        ##                             shell_class=IPython.shell.InteractiveShell)
+        ipython0 = iplib.InteractiveShell(
+            parent=None, config=None,
+            user_ns=user_ns,
+            user_global_ns=user_global_ns
+        )
+        self._IP = ipython0
 
         #we save ipython0 displayhook and we restore sys.displayhook
         self.displayhook = sys.displayhook
@@ -185,12 +177,10 b' class NonBlockingIPShell(object):'
         #we replace the help command
         self._IP.user_ns['help'] = _Helper(self._pager_help)
 
-        #we disable cpase magic... until we found a way to use it properly.
-        from IPython.core import ipapi
-        ip = ipapi.get()
+        #we disable cpaste magic... until we found a way to use it properly.
         def bypass_magic(self, arg):
             print '%this magic is currently disabled.'
-        ip.define_magic('cpaste', bypass_magic)
+        ipython0.define_magic('cpaste', bypass_magic)
 
         import __builtin__
         __builtin__.raw_input = self._raw_input
@@ -471,7 +461,7 b' class NonBlockingIPShell(object):'
         '''
 
         orig_stdout = sys.stdout
-        sys.stdout = genutils.Term.cout
+        sys.stdout = Term.cout
         #self.sys_displayhook_ori = sys.displayhook
         #sys.displayhook = self.displayhook
         

IPython/gui/wx/ipython_history.py

@@ -11,6 +11,7 b' __author__ = "Laurent Dufrechou"'
 __email__   = "laurent.dufrechou _at_ gmail.com"
 __license__ = "BSD"
 #-----------------------------------------
+
 class IPythonHistoryPanel(wx.Panel):
 
     def __init__(self, parent,flt_empty=True,

IPython/kernel/client.py

@@ -24,16 +24,25 b' The main classes in this module are:'
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
-# Imports
+# Warnings control
 #-----------------------------------------------------------------------------
 
-from cStringIO import StringIO
-import sys
 import warnings
 
-# from IPython.utils import growl
-# growl.start("IPython1 Client")
+# Twisted generates annoying warnings with Python 2.6, as will do other code
+# that imports 'sets' as of today
+warnings.filterwarnings('ignore', 'the sets module is deprecated',
+                        DeprecationWarning )
+
+# This one also comes from Twisted
+warnings.filterwarnings('ignore', 'the sha module is deprecated',
+                        DeprecationWarning)
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
 
+import sys
 
 from twisted.internet import reactor
 from twisted.internet.error import PotentialZombieWarning
@@ -74,8 +83,6 b' rit.setDaemon(True)'
 rit.start()
 
 
-
-
 __all__ = [
     'MapTask',
     'StringTask',

IPython/kernel/clientconnector.py

@@ -1,6 +1,4 b''
-#!/usr/bin/env python
 # encoding: utf-8
-
 """Facilities for handling client connections to the controller."""
 
 #-----------------------------------------------------------------------------
@@ -20,6 +18,8 b' import os'
 from IPython.kernel.fcutil import (
     Tub,
     find_furl,
+    is_valid_furl,
+    is_valid_furl_file,
     is_valid_furl_or_file,
     validate_furl_or_file,
     FURLError
@@ -33,7 +33,7 b' from IPython.kernel.twistedutil import ('
     sleep_deferred
 )
 from IPython.utils.importstring import import_item
-from IPython.utils.genutils import get_ipython_dir
+from IPython.utils.path import get_ipython_dir
 
 from twisted.internet import defer
 from twisted.internet.defer import inlineCallbacks, returnValue
@@ -66,18 +66,30 b' class AsyncClientConnector(object):'
     def _find_furl(self, profile='default', cluster_dir=None, 
                    furl_or_file=None, furl_file_name=None,
                    ipython_dir=None):
-        """Find a FURL file by profile+ipython_dir or cluster dir.
+        """Find a FURL file.
+
+        If successful, this returns a FURL file that exists on the file
+        system. The contents of the file have not been checked though. This
+        is because we often have to deal with FURL file whose buffers have
+        not been flushed.
 
         This raises an :exc:`~IPython.kernel.fcutil.FURLError` exception 
         if a FURL file can't be found.
+
+        This tries the following:
+        
+        1. By the name ``furl_or_file``.
+        2. By ``cluster_dir`` and ``furl_file_name``.
+        3. By cluster profile with a default of ``default``. This uses
+           ``ipython_dir``.
         """
         # Try by furl_or_file
         if furl_or_file is not None:
-            validate_furl_or_file(furl_or_file)
+            if is_valid_furl_or_file(furl_or_file):
             return furl_or_file
 
         if furl_file_name is None:
-            raise FURLError('A furl_file_name must be provided')
+            raise FURLError('A furl_file_name must be provided if furl_or_file is not')
 
         # Try by cluster_dir
         if cluster_dir is not None:
@@ -151,7 +163,7 b' class AsyncClientConnector(object):'
             The full path to a cluster directory.  This is useful if profiles
             are not being used.
         furl_or_file : str
-            A furl or a filename containing a FURLK. This is useful if you 
+            A furl or a filename containing a FURL. This is useful if you 
             simply know the location of the FURL file.
         ipython_dir : str
             The location of the ipython_dir if different from the default.
@@ -193,7 +205,7 b' class AsyncClientConnector(object):'
             The full path to a cluster directory.  This is useful if profiles
             are not being used.
         furl_or_file : str
-            A furl or a filename containing a FURLK. This is useful if you 
+            A furl or a filename containing a FURL. This is useful if you 
             simply know the location of the FURL file.
         ipython_dir : str
             The location of the ipython_dir if different from the default.
@@ -259,6 +271,9 b' class AsyncClientConnector(object):'
                 profile, cluster_dir, furl_or_file,
                 furl_file_name, ipython_dir
             )
+            # If this succeeds, we know the furl file exists and has a .furl
+            # extension, but it could still be empty. That is checked each
+            # connection attempt.
         except FURLError:
             return defer.fail(failure.Failure())
 
@@ -349,7 +364,7 b' class ClientConnector(object):'
             The full path to a cluster directory.  This is useful if profiles
             are not being used.
         furl_or_file : str
-            A furl or a filename containing a FURLK. This is useful if you 
+            A furl or a filename containing a FURL. This is useful if you 
             simply know the location of the FURL file.
         ipython_dir : str
             The location of the ipython_dir if different from the default.
@@ -390,7 +405,7 b' class ClientConnector(object):'
             The full path to a cluster directory.  This is useful if profiles
             are not being used.
         furl_or_file : str
-            A furl or a filename containing a FURLK. This is useful if you 
+            A furl or a filename containing a FURL. This is useful if you 
             simply know the location of the FURL file.
         ipython_dir : str
             The location of the ipython_dir if different from the default.

IPython/kernel/clientinterfaces.py

@@ -15,7 +15,7 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
-from zope.interface import Interface, implements
+from zope.interface import Interface
 
 class IFCClientInterfaceProvider(Interface):
 

IPython/kernel/clusterdir.py

@@ -24,13 +24,16 b' import warnings'
 
 from twisted.python import log
 
-from IPython.core import release
 from IPython.config.loader import PyFileConfigLoader
-from IPython.core.application import Application
+from IPython.core.application import Application, BaseAppConfigLoader
 from IPython.core.component import Component
-from IPython.utils.genutils import get_ipython_dir, get_ipython_package_dir
-from IPython.utils.traitlets import Unicode, Bool
-from IPython.utils import genutils
+from IPython.core.crashhandler import CrashHandler
+from IPython.core import release
+from IPython.utils.path import (
+    get_ipython_package_dir,
+    expand_path
+)
+from IPython.utils.traitlets import Unicode
 
 #-----------------------------------------------------------------------------
 # Warnings control
@@ -45,7 +48,7 b" warnings.filterwarnings('ignore', 'the sha module is deprecated',"
                         DeprecationWarning)
 
 #-----------------------------------------------------------------------------
-# Classes and functions
+# Module errors
 #-----------------------------------------------------------------------------
 
 class ClusterDirError(Exception):
@@ -56,6 +59,10 b' class PIDFileError(Exception):'
     pass
 
 
+#-----------------------------------------------------------------------------
+# Class for managing cluster directories
+#-----------------------------------------------------------------------------
+
 class ClusterDir(Component):
     """An object to manage the cluster directory and its resources.
 
@@ -225,44 +232,110 b' class ClusterDir(Component):'
             The path of the cluster directory.  This is expanded using
             :func:`IPython.utils.genutils.expand_path`.
         """
-        cluster_dir = genutils.expand_path(cluster_dir)
+        cluster_dir = expand_path(cluster_dir)
         if not os.path.isdir(cluster_dir):
             raise ClusterDirError('Cluster directory not found: %s' % cluster_dir)
         return ClusterDir(cluster_dir)
 
 
-# Default command line options for IPython cluster applications.
-cl_args = (
-    (('--ipython-dir',), dict(
-        dest='Global.ipython_dir',type=unicode,
-        help='Set to override default location of Global.ipython_dir.',
-        metavar='Global.ipython_dir') ),
-    (('-p', '--profile',), dict(
+#-----------------------------------------------------------------------------
+# Command line options
+#-----------------------------------------------------------------------------
+
+class ClusterDirConfigLoader(BaseAppConfigLoader):
+
+    def _add_cluster_profile(self, parser):
+        paa = parser.add_argument
+        paa('-p', '--profile',
         dest='Global.profile',type=unicode,
         help=
         """The string name of the profile to be used. This determines the name
         of the cluster dir as: cluster_<profile>. The default profile is named
         'default'.  The cluster directory is resolve this way if the
         --cluster-dir option is not used.""",
-        metavar='Global.profile') ),
-    (('--cluster-dir',), dict(
+            metavar='Global.profile')
+
+    def _add_cluster_dir(self, parser):
+        paa = parser.add_argument
+        paa('--cluster-dir',
         dest='Global.cluster_dir',type=unicode,
         help="""Set the cluster dir. This overrides the logic used by the
         --profile option.""",
-        metavar='Global.cluster_dir') ),
-    (('--work-dir',), dict(
+            metavar='Global.cluster_dir')
+
+    def _add_work_dir(self, parser):
+        paa = parser.add_argument
+        paa('--work-dir',
       dest='Global.work_dir',type=unicode,
       help='Set the working dir for the process.',
-      metavar='Global.work_dir') ),
-    (('--clean-logs',), dict(
+            metavar='Global.work_dir')
+
+    def _add_clean_logs(self, parser):
+        paa = parser.add_argument
+        paa('--clean-logs',
       dest='Global.clean_logs', action='store_true',
-      help='Delete old log flies before starting.') ),
-    (('--no-clean-logs',), dict(
+            help='Delete old log flies before starting.')
+
+    def _add_no_clean_logs(self, parser):
+        paa = parser.add_argument
+        paa('--no-clean-logs',
       dest='Global.clean_logs', action='store_false',
-      help="Don't Delete old log flies before starting.") ),
+            help="Don't Delete old log flies before starting.")
+
+    def _add_arguments(self):
+        super(ClusterDirConfigLoader, self)._add_arguments()
+        self._add_cluster_profile(self.parser)
+        self._add_cluster_dir(self.parser)
+        self._add_work_dir(self.parser)
+        self._add_clean_logs(self.parser)
+        self._add_no_clean_logs(self.parser)
+
+
+#-----------------------------------------------------------------------------
+# Crash handler for this application
+#-----------------------------------------------------------------------------
+
+
+_message_template = """\
+Oops, $self.app_name crashed. We do our best to make it stable, but...
+
+A crash report was automatically generated with the following information:
+  - A verbatim copy of the crash traceback.
+  - Data on your current $self.app_name configuration.
+
+It was left in the file named:
+\t'$self.crash_report_fname'
+If you can email this file to the developers, the information in it will help
+them in understanding and correcting the problem.
+
+You can mail it to: $self.contact_name at $self.contact_email
+with the subject '$self.app_name Crash Report'.
+
+If you want to do it now, the following command will work (under Unix):
+mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
+
+To ensure accurate tracking of this issue, please file a report about it at:
+$self.bug_tracker
+"""
+
+class ClusterDirCrashHandler(CrashHandler):
+    """sys.excepthook for IPython itself, leaves a detailed report on disk."""
+
+    message_template = _message_template
+
+    def __init__(self, app):
+        contact_name = release.authors['Brian'][0]
+        contact_email = release.authors['Brian'][1]
+        bug_tracker = 'https://bugs.launchpad.net/ipython/+filebug'
+        super(ClusterDirCrashHandler,self).__init__(
+            app, contact_name, contact_email, bug_tracker
     )
 
 
+#-----------------------------------------------------------------------------
+# Main application
+#-----------------------------------------------------------------------------
+
 class ApplicationWithClusterDir(Application):
     """An application that puts everything into a cluster directory.
 
@@ -282,10 +355,10 b' class ApplicationWithClusterDir(Application):'
     dir and named the value of the ``config_file_name`` class attribute.
     """
 
+    command_line_loader = ClusterDirConfigLoader
+    crash_handler_class = ClusterDirCrashHandler
     auto_create_cluster_dir = True
 
-    cl_arguments = Application.cl_arguments + cl_args
-
     def create_default_config(self):
         super(ApplicationWithClusterDir, self).create_default_config()
         self.default_config.Global.profile = u'default'
@@ -316,7 +389,7 b' class ApplicationWithClusterDir(Application):'
             cluster_dir = self.command_line_config.Global.cluster_dir
         except AttributeError:
             cluster_dir = self.default_config.Global.cluster_dir
-        cluster_dir = genutils.expand_path(cluster_dir)
+        cluster_dir = expand_path(cluster_dir)
         try:
             self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
         except ClusterDirError:
@@ -368,15 +441,17 b' class ApplicationWithClusterDir(Application):'
     def find_config_file_name(self):
         """Find the config file name for this application."""
         # For this type of Application it should be set as a class attribute.
-        if not hasattr(self, 'config_file_name'):
+        if not hasattr(self, 'default_config_file_name'):
             self.log.critical("No config filename found")
+        else:
+            self.config_file_name = self.default_config_file_name
 
     def find_config_file_paths(self):
-        # Include our own config directory last, so that users can still find
-        # our shipped copies of builtin config files even if they don't have
-        # them in their ipython cluster directory.
+        # Set the search path to to the cluster directory. We should NOT
+        # include IPython.config.default here as the default config files
+        # are ALWAYS automatically moved to the cluster directory.
         conf_dir = os.path.join(get_ipython_package_dir(), 'config', 'default')
-        self.config_file_paths = (self.cluster_dir, conf_dir)
+        self.config_file_paths = (self.cluster_dir,)
 
     def pre_construct(self):
         # The log and security dirs were set earlier, but here we put them
@@ -389,7 +464,7 b' class ApplicationWithClusterDir(Application):'
         pdir = self.cluster_dir_obj.pid_dir
         self.pid_dir = config.Global.pid_dir = pdir
         self.log.info("Cluster directory set to: %s" % self.cluster_dir)
-        config.Global.work_dir = unicode(genutils.expand_path(config.Global.work_dir))
+        config.Global.work_dir = unicode(expand_path(config.Global.work_dir))
         # Change to the working directory. We do this just before construct
         # is called so all the components there have the right working dir.
         self.to_work_dir()
@@ -461,3 +536,4 b' class ApplicationWithClusterDir(Application):'
                 return pid
         else:
             raise PIDFileError('pid file not found: %s' % pid_file)
+

IPython/kernel/controllerservice.py

@@ -37,20 +37,18 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
-import os, sys
+import os
 
 from twisted.application import service
-from twisted.internet import defer, reactor
-from twisted.python import log, components
+from twisted.python import log
 from zope.interface import Interface, implements, Attribute
-import zope.interface as zi
 
 from IPython.kernel.engineservice import \
     IEngineCore, \
     IEngineSerialized, \
     IEngineQueued
     
-from IPython.utils.genutils import get_ipython_dir
+from IPython.utils.path import get_ipython_dir
 from IPython.kernel import codeutil
 
 #-------------------------------------------------------------------------------

IPython/kernel/core/prompts.py

@@ -21,6 +21,8 b' __docformat__ = "restructuredtext en"'
 
 # Required modules
 import __builtin__
+import os
+import re
 import socket
 import sys
 
@@ -30,7 +32,8 b' from IPython.external.Itpl import ItplNS'
 from IPython.utils import coloransi
 from IPython.core import release
 from IPython.core.error import TryNext
-from IPython.utils.genutils import *
+from IPython.utils.io import Term
+from IPython.utils.warn import warn
 import IPython.utils.generics
 
 #****************************************************************************

IPython/kernel/core/tests/test_redirectors.py

@@ -22,15 +22,15 b' import sys'
 
 from twisted.trial import unittest
 
+from IPython.testing import decorators_trial as dec
+
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
 class TestRedirector(unittest.TestCase):
 
-    if sys.platform == 'win32':
-        skip = True
-    
+    @dec.skip_win32
     def test_redirector(self):
         """Checks that the redirector can be used to do synchronous capture.
         """
@@ -51,6 +51,7 b' class TestRedirector(unittest.TestCase):'
         result2 = "".join("%ic\n%i\n" %(i, i) for i in range(10))
         self.assertEquals(result1, result2)
 
+    @dec.skip_win32
     def test_redirector_output_trap(self):
         """Check the greedy trapping behavior of the traps.
         

IPython/kernel/engineconnector.py

@@ -17,8 +17,7 b''
 import os
 import cPickle as pickle
 
-from twisted.python import log, failure
-from twisted.internet import defer
+from twisted.python import log
 from twisted.internet.defer import inlineCallbacks, returnValue
 
 from IPython.kernel.fcutil import find_furl, validate_furl_or_file

IPython/kernel/enginefc.py

@@ -19,40 +19,36 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
-import os, time
 import cPickle as pickle
 
 from twisted.python import components, log, failure
-from twisted.python.failure import Failure
-from twisted.internet import defer, reactor, threads
-from twisted.internet.interfaces import IProtocolFactory
-from zope.interface import Interface, implements, Attribute
+from twisted.internet import defer, threads
+from zope.interface import Interface, implements
 
 from twisted.internet.base import DelayedCall
 DelayedCall.debug = True
 
+try:
+    from foolscap.api import Referenceable, DeadReferenceError
+except ImportError:
 from foolscap import Referenceable, DeadReferenceError
 from foolscap.referenceable import RemoteReference
 
 from IPython.kernel.pbutil import packageFailure, unpackageFailure
-from IPython.kernel.util import printer
-from IPython.kernel.twistedutil import gatherBoth
-from IPython.kernel import newserialized
-from IPython.kernel.error import  ProtocolError
-from IPython.kernel import controllerservice
 from IPython.kernel.controllerservice import IControllerBase
-from IPython.kernel.engineservice import \
-    IEngineBase, \
-    IEngineQueued, \
-    EngineService, \
+from IPython.kernel.engineservice import (
+    IEngineBase,
+    IEngineQueued,
     StrictDict
-from IPython.kernel.pickleutil import \
-    can, \
-    canDict, \
-    canSequence, \
-    uncan, \
-    uncanDict, \
+)
+from IPython.kernel.pickleutil import (
+    can,
+    canDict,
+    canSequence,
+    uncan,
+    uncanDict,
     uncanSequence
+)
 
 
 #-------------------------------------------------------------------------------

IPython/kernel/error.py

@@ -17,6 +17,9 b' __test__ = {}'
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
+
+from twisted.python import failure
+
 from IPython.kernel.core import error
 
 #-------------------------------------------------------------------------------
@@ -26,6 +29,7 b' from IPython.kernel.core import error'
 class KernelError(error.IPythonError):
     pass
 
+
 class NotDefined(KernelError):
     def __init__(self, name):
         self.name = name
@@ -36,78 +40,102 b' class NotDefined(KernelError):'
     
     __str__ = __repr__
 
+
 class QueueCleared(KernelError):
     pass
 
+
 class IdInUse(KernelError):
     pass
 
+
 class ProtocolError(KernelError):
     pass
 
+
 class ConnectionError(KernelError):
     pass
 
+
 class InvalidEngineID(KernelError):
     pass
     
+
 class NoEnginesRegistered(KernelError):
     pass
     
+
 class InvalidClientID(KernelError):
     pass
     
+
 class InvalidDeferredID(KernelError):
     pass
     
+
 class SerializationError(KernelError):
     pass
     
+
 class MessageSizeError(KernelError):
     pass
     
+
 class PBMessageSizeError(MessageSizeError):
     pass
     
+
 class ResultNotCompleted(KernelError):
     pass
     
+
 class ResultAlreadyRetrieved(KernelError):
     pass
     
 class ClientError(KernelError):
     pass
 
+
 class TaskAborted(KernelError):
     pass
 
+
 class TaskTimeout(KernelError):
     pass
 
+
 class NotAPendingResult(KernelError):
     pass
 
+
 class UnpickleableException(KernelError):
     pass
 
+
 class AbortedPendingDeferredError(KernelError):
     pass
 
+
 class InvalidProperty(KernelError):
     pass
 
+
 class MissingBlockArgument(KernelError):
     pass
 
+
 class StopLocalExecution(KernelError):
     pass
 
+
 class SecurityError(KernelError):
     pass
 
+
 class FileTimeoutError(KernelError):
     pass
 
+
 class TaskRejectError(KernelError):
     """Exception to raise when a task should be rejected by an engine.
     
@@ -122,6 +150,7 b' class TaskRejectError(KernelError):'
     properties don't have to be managed or tested by the controller.
     """
 
+
 class CompositeError(KernelError):
     def __init__(self, message, elist):
         Exception.__init__(self, *(message, elist))
@@ -176,9 +205,8 b' class CompositeError(KernelError):'
         else:
             raise et, ev, etb
 
-def collect_exceptions(rlist, method):
-    from twisted.python import failure
 
+def collect_exceptions(rlist, method):
     elist = []
     for r in rlist:
         if isinstance(r, failure.Failure):
@@ -204,4 +232,3 b' def collect_exceptions(rlist, method):'
         except CompositeError, e:
             raise e
             
-

IPython/kernel/fcutil.py

@@ -23,16 +23,19 b' import tempfile'
 from twisted.internet import reactor, defer
 from twisted.python import log
 
+import foolscap
+try:
+    from foolscap.api import Tub, UnauthenticatedTub
+except ImportError:
 from foolscap import Tub, UnauthenticatedTub
 
 from IPython.config.loader import Config
-
 from IPython.kernel.configobjfactory import AdaptedConfiguredObjectFactory
-
 from IPython.kernel.error import SecurityError
 
-from IPython.utils.traitlets import Int, Str, Bool, Instance
 from IPython.utils.importstring import import_item
+from IPython.utils.path import expand_path
+from IPython.utils.traitlets import Int, Str, Bool, Instance
 
 #-----------------------------------------------------------------------------
 # Code
@@ -57,17 +60,17 b' class FURLError(Exception):'
 
 def check_furl_file_security(furl_file, secure):
     """Remove the old furl_file if changing security modes."""
+    furl_file = expand_path(furl_file)
     if os.path.isfile(furl_file):
-        f = open(furl_file, 'r')
+        with open(furl_file, 'r') as f:
         oldfurl = f.read().strip()
-        f.close()
         if (oldfurl.startswith('pb://') and not secure) or (oldfurl.startswith('pbu://') and secure):
             os.remove(furl_file)
 
 
 def is_secure(furl):
     """Is the given FURL secure or not."""
-    if is_valid(furl):
+    if is_valid_furl(furl):
         if furl.startswith("pb://"):
             return True
         elif furl.startswith("pbu://"):
@@ -76,26 +79,45 b' def is_secure(furl):'
         raise FURLError("invalid FURL: %s" % furl)
 
 
-def is_valid(furl):
+def is_valid_furl(furl):
     """Is the str a valid FURL or not."""
     if isinstance(furl, str):
         if furl.startswith("pb://") or furl.startswith("pbu://"):
             return True
     else:
         return False
+    else:
+        return False
+
+
+def is_valid_furl_file(furl_or_file):
+    """See if furl_or_file exists and contains a valid FURL.
+
+    This doesn't try to read the contents because often we have to validate
+    FURL files that are created, but don't yet have a FURL written to them.
+    """
+    if isinstance(furl_or_file, (str, unicode)):
+        path, furl_filename = os.path.split(furl_or_file)
+        if os.path.isdir(path) and furl_filename.endswith('.furl'):
+            return True
+    return False
 
 
 def find_furl(furl_or_file):
-    """Find, validate and return a FURL in a string or file."""
-    if isinstance(furl_or_file, str):
-        if is_valid(furl_or_file):
+    """Find, validate and return a FURL in a string or file.
+
+    This calls :func:`IPython.utils.path.expand_path` on the argument to
+    properly handle ``~`` and ``$`` variables in the path.
+    """
+    if is_valid_furl(furl_or_file):
             return furl_or_file
-    if os.path.isfile(furl_or_file):
+    furl_or_file = expand_path(furl_or_file)
+    if is_valid_furl_file(furl_or_file):
         with open(furl_or_file, 'r') as f:
             furl = f.read().strip()
-        if is_valid(furl):
+        if is_valid_furl(furl):
             return furl
-    raise FURLError("Not a valid FURL or FURL file: %s" % furl_or_file)
+    raise FURLError("Not a valid FURL or FURL file: %r" % furl_or_file)
 
 
 def is_valid_furl_or_file(furl_or_file):
@@ -106,17 +128,14 b' def is_valid_furl_or_file(furl_or_file):'
     if the FURL file exists or to read its contents. This is useful for
     cases where auto re-connection is being used.
     """
-    if isinstance(furl_or_file, str):
-        if is_valid(furl_or_file):
-            return True
-    if isinstance(furl_or_file, (str, unicode)):
-        path, furl_filename = os.path.split(furl_or_file)
-        if os.path.isdir(path) and furl_filename.endswith('.furl'):
+    if is_valid_furl(furl_or_file) or is_valid_furl_file(furl_or_file):
             return True
+    else:
     return False
 
 
 def validate_furl_or_file(furl_or_file):
+    """Like :func:`is_valid_furl_or_file`, but raises an error."""
     if not is_valid_furl_or_file(furl_or_file):
         raise FURLError('Not a valid FURL or FURL file: %r' % furl_or_file)
 
@@ -265,8 +284,12 b' class FCServiceFactory(AdaptedConfiguredObjectFactory):'
         """Register the reference with the FURL file.
 
         The FURL file is created and then moved to make sure that when the
-        file appears, the buffer has been flushed and the file closed.
+        file appears, the buffer has been flushed and the file closed. This
+        is not done if we are re-using FURLS however.
         """
+        if self.reuse_furls:
+            self.tub.registerReference(ref, furlFile=furl_file)
+        else:
         temp_furl_file = get_temp_furlfile(furl_file)
         self.tub.registerReference(ref, furlFile=temp_furl_file)
         os.rename(temp_furl_file, furl_file)

IPython/kernel/ipclusterapp.py

@@ -22,161 +22,203 b' import signal'
 if os.name=='posix':
     from twisted.scripts._twistd_unix import daemonize
 
-from IPython.core import release
-from IPython.external.argparse import ArgumentParser
-from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault
-from IPython.utils.importstring import import_item
+from twisted.internet import reactor, defer
+from twisted.python import log, failure
+
 
+from IPython.external.argparse import ArgumentParser, SUPPRESS
+from IPython.utils.importstring import import_item
 from IPython.kernel.clusterdir import (
-    ApplicationWithClusterDir, ClusterDirError, PIDFileError
+    ApplicationWithClusterDir, ClusterDirConfigLoader,
+    ClusterDirError, PIDFileError
 )
 
-from twisted.internet import reactor, defer
-from twisted.python import log, failure
-
 
 #-----------------------------------------------------------------------------
-# The ipcluster application
+# Module level variables
 #-----------------------------------------------------------------------------
 
 
+default_config_file_name = u'ipcluster_config.py'
+
+
+_description = """\
+Start an IPython cluster for parallel computing.\n\n
+
+An IPython cluster consists of 1 controller and 1 or more engines.
+This command automates the startup of these processes using a wide
+range of startup methods (SSH, local processes, PBS, mpiexec,
+Windows HPC Server 2008). To start a cluster with 4 engines on your
+local host simply do 'ipcluster start -n 4'. For more complex usage 
+you will typically do 'ipcluster create -p mycluster', then edit
+configuration files, followed by 'ipcluster start -p mycluster -n 4'.
+"""
+
+
 # Exit codes for ipcluster
 
 # This will be the exit code if the ipcluster appears to be running because
 # a .pid file exists
 ALREADY_STARTED = 10
 
+
 # This will be the exit code if ipcluster stop is run, but there is not .pid
 # file to be found.
 ALREADY_STOPPED = 11
 
 
-class IPClusterCLLoader(ArgParseConfigLoader):
+#-----------------------------------------------------------------------------
+# Command line options
+#-----------------------------------------------------------------------------
+
+
+class IPClusterAppConfigLoader(ClusterDirConfigLoader):
+
+    def _add_arguments(self):
+        # Don't call ClusterDirConfigLoader._add_arguments as we don't want
+        # its defaults on self.parser. Instead, we will put those on
+        # default options on our subparsers.
 
-    def _add_other_arguments(self):
         # This has all the common options that all subcommands use
-        parent_parser1 = ArgumentParser(add_help=False,
-                                        argument_default=NoConfigDefault)
-        parent_parser1.add_argument('--ipython-dir', 
-            dest='Global.ipython_dir',type=unicode,
-            help='Set to override default location of Global.ipython_dir.',
-            metavar='Global.ipython_dir')
-        parent_parser1.add_argument('--log-level',
-            dest="Global.log_level",type=int,
-            help='Set the log level (0,10,20,30,40,50).  Default is 30.',
-            metavar='Global.log_level')
+        parent_parser1 = ArgumentParser(
+            add_help=False,
+            argument_default=SUPPRESS
+        )
+        self._add_ipython_dir(parent_parser1)
+        self._add_log_level(parent_parser1)
 
         # This has all the common options that other subcommands use
-        parent_parser2 = ArgumentParser(add_help=False,
-                                        argument_default=NoConfigDefault)
-        parent_parser2.add_argument('-p','--profile',
-            dest='Global.profile',type=unicode,
-            help='The string name of the profile to be used. This determines '
-            'the name of the cluster dir as: cluster_<profile>. The default profile '
-            'is named "default".  The cluster directory is resolve this way '
-            'if the --cluster-dir option is not used.',
-            metavar='Global.profile')
-        parent_parser2.add_argument('--cluster-dir',
-            dest='Global.cluster_dir',type=unicode,
-            help='Set the cluster dir. This overrides the logic used by the '
-            '--profile option.',
-            metavar='Global.cluster_dir'),
-        parent_parser2.add_argument('--work-dir',
-            dest='Global.work_dir',type=unicode,
-            help='Set the working dir for the process.',
-            metavar='Global.work_dir')
-        parent_parser2.add_argument('--log-to-file',
-            action='store_true', dest='Global.log_to_file', 
-            help='Log to a file in the log directory (default is stdout)'
+        parent_parser2 = ArgumentParser(
+            add_help=False,
+            argument_default=SUPPRESS
         )
+        self._add_cluster_profile(parent_parser2)
+        self._add_cluster_dir(parent_parser2)
+        self._add_work_dir(parent_parser2)
+        paa = parent_parser2.add_argument
+        paa('--log-to-file',
+            action='store_true', dest='Global.log_to_file', 
+            help='Log to a file in the log directory (default is stdout)')
 
+        # Create the object used to create the subparsers.
         subparsers = self.parser.add_subparsers(
             dest='Global.subcommand',
             title='ipcluster subcommands',
-            description='ipcluster has a variety of subcommands. '
-            'The general way of running ipcluster is "ipcluster <cmd> '
-            ' [options]""',
-            help='For more help, type "ipcluster <cmd> -h"')
+            description=
+            """ipcluster has a variety of subcommands. The general way of 
+            running ipcluster is 'ipcluster <cmd> [options]'. To get help
+            on a particular subcommand do 'ipcluster <cmd> -h'."""
+            # help="For more help, type 'ipcluster <cmd> -h'",
+        )
 
+        # The "list" subcommand parser
         parser_list = subparsers.add_parser(
             'list',
-            help='List all clusters in cwd and ipython_dir.',
-            parents=[parent_parser1]
+            parents=[parent_parser1],
+            argument_default=SUPPRESS,
+            help="List all clusters in cwd and ipython_dir.",
+            description=
+            """List all available clusters, by cluster directory, that can
+            be found in the current working directly or in the ipython
+            directory. Cluster directories are named using the convention
+            'cluster_<profile>'."""
         )
 
+        # The "create" subcommand parser
         parser_create = subparsers.add_parser(
             'create',
-            help='Create a new cluster directory.',
-            parents=[parent_parser1, parent_parser2] 
+            parents=[parent_parser1, parent_parser2],
+            argument_default=SUPPRESS,
+            help="Create a new cluster directory.",
+            description=
+            """Create an ipython cluster directory by its profile name or 
+            cluster directory path. Cluster directories contain 
+            configuration, log and security related files and are named 
+            using the convention 'cluster_<profile>'. By default they are
+            located in your ipython directory. Once created, you will
+            probably need to edit the configuration files in the cluster
+            directory to configure your cluster. Most users will create a
+            cluster directory by profile name, 
+            'ipcluster create -p mycluster', which will put the directory
+            in '<ipython_dir>/cluster_mycluster'. 
+            """
         )
-        parser_create.add_argument(
-            '--reset-config',
+        paa = parser_create.add_argument
+        paa('--reset-config',
             dest='Global.reset_config', action='store_true',
-            default=NoConfigDefault,
-            help='Recopy the default config files to the cluster directory. '
-            'You will loose any modifications you have made to these files.'
-        )
+            help=
+            """Recopy the default config files to the cluster directory.
+            You will loose any modifications you have made to these files.""")
 
+        # The "start" subcommand parser
         parser_start = subparsers.add_parser(
             'start',
-            help='Start a cluster.',
-            parents=[parent_parser1, parent_parser2]
+            parents=[parent_parser1, parent_parser2],
+            argument_default=SUPPRESS,
+            help="Start a cluster.",
+            description=
+            """Start an ipython cluster by its profile name or cluster 
+            directory. Cluster directories contain configuration, log and
+            security related files and are named using the convention
+            'cluster_<profile>' and should be creating using the 'start'
+            subcommand of 'ipcluster'. If your cluster directory is in 
+            the cwd or the ipython directory, you can simply refer to it
+            using its profile name, 'ipcluster start -n 4 -p <profile>`,
+            otherwise use the '--cluster-dir' option.
+            """
         )
-        parser_start.add_argument(
-            '-n', '--number',
+        paa = parser_start.add_argument
+        paa('-n', '--number',
             type=int, dest='Global.n',
             help='The number of engines to start.',
-            metavar='Global.n'
-        )
-        parser_start.add_argument('--clean-logs',
+            metavar='Global.n')
+        paa('--clean-logs',
             dest='Global.clean_logs', action='store_true',
-            help='Delete old log flies before starting.',
-        )
-        parser_start.add_argument('--no-clean-logs',
+            help='Delete old log flies before starting.')
+        paa('--no-clean-logs',
             dest='Global.clean_logs', action='store_false',
-            help="Don't delete old log flies before starting.",
-        )
-        parser_start.add_argument('--daemon',
+            help="Don't delete old log flies before starting.")
+        paa('--daemon',
             dest='Global.daemonize', action='store_true',
-            help='Daemonize the ipcluster program. This implies --log-to-file',
-        )
-        parser_start.add_argument('--no-daemon',
+            help='Daemonize the ipcluster program. This implies --log-to-file')
+        paa('--no-daemon',
             dest='Global.daemonize', action='store_false',
-            help="Dont't daemonize the ipcluster program.",
-        )
+            help="Dont't daemonize the ipcluster program.")
 
-        parser_start = subparsers.add_parser(
+        # The "stop" subcommand parser
+        parser_stop = subparsers.add_parser(
             'stop',
-            help='Stop a cluster.',
-            parents=[parent_parser1, parent_parser2]
+            parents=[parent_parser1, parent_parser2],
+            argument_default=SUPPRESS,
+            help="Stop a running cluster.",
+            description=
+            """Stop a running ipython cluster by its profile name or cluster 
+            directory. Cluster directories are named using the convention
+            'cluster_<profile>'. If your cluster directory is in 
+            the cwd or the ipython directory, you can simply refer to it
+            using its profile name, 'ipcluster stop -p <profile>`, otherwise
+            use the '--cluster-dir' option.
+            """
         )
-        parser_start.add_argument('--signal',
+        paa = parser_stop.add_argument
+        paa('--signal',
             dest='Global.signal', type=int,
             help="The signal number to use in stopping the cluster (default=2).",
-            metavar="Global.signal",
-        )
+            metavar="Global.signal")
 
 
-default_config_file_name = u'ipcluster_config.py'
-
-
-_description = """Start an IPython cluster for parallel computing.\n\n
-
-An IPython cluster consists of 1 controller and 1 or more engines.
-This command automates the startup of these processes using a wide
-range of startup methods (SSH, local processes, PBS, mpiexec,
-Windows HPC Server 2008). To start a cluster with 4 engines on your
-local host simply do "ipcluster start -n 4". For more complex usage 
-you will typically do "ipcluster create -p mycluster", then edit
-configuration files, followed by "ipcluster start -p mycluster -n 4".
-"""
+#-----------------------------------------------------------------------------
+# Main application
+#-----------------------------------------------------------------------------
 
 
 class IPClusterApp(ApplicationWithClusterDir):
 
     name = u'ipcluster'
     description = _description
-    config_file_name = default_config_file_name
+    usage = None
+    command_line_loader = IPClusterAppConfigLoader
+    default_config_file_name = default_config_file_name
     default_log_level = logging.INFO
     auto_create_cluster_dir = False
 
@@ -192,13 +234,6 b' class IPClusterApp(ApplicationWithClusterDir):'
         self.default_config.Global.signal = 2
         self.default_config.Global.daemonize = False
 
-    def create_command_line_config(self):
-        """Create and return a command line config loader."""
-        return IPClusterCLLoader(
-            description=self.description, 
-            version=release.version
-        )
-
     def find_resources(self):
         subcommand = self.command_line_config.Global.subcommand
         if subcommand=='list':
@@ -361,8 +396,11 b' class IPClusterApp(ApplicationWithClusterDir):'
                 log.msg('Unexpected error in ipcluster:')
                 log.msg(r.getTraceback())
             log.msg("IPython cluster: stopping")
-            self.stop_engines()
-            self.stop_controller()
+            # These return deferreds. We are not doing anything with them
+            # but we are holding refs to them as a reminder that they 
+            # do return deferreds.
+            d1 = self.stop_engines()
+            d2 = self.stop_controller()
             # Wait a few seconds to let things shut down.
             reactor.callLater(4.0, reactor.stop)
 
@@ -449,6 +487,7 b' class IPClusterApp(ApplicationWithClusterDir):'
                 # old .pid files.
                 self.remove_pid_file()
 
+
 def launch_new_instance():
     """Create and run the IPython cluster."""
     app = IPClusterApp()

IPython/kernel/ipcontrollerapp.py

@@ -18,20 +18,40 b' The IPython controller application.'
 from __future__ import with_statement
 
 import copy
-import os
 import sys
 
 from twisted.application import service
 from twisted.internet import reactor
 from twisted.python import log
 
-from IPython.config.loader import Config, NoConfigDefault
-from IPython.core import release
-from IPython.core.application import Application
+from IPython.config.loader import Config
 from IPython.kernel import controllerservice
-from IPython.kernel.clusterdir import ApplicationWithClusterDir
-from IPython.kernel.fcutil import FCServiceFactory
-from IPython.utils.traitlets import Str, Instance, Unicode
+from IPython.kernel.clusterdir import (
+    ApplicationWithClusterDir,
+    ClusterDirConfigLoader
+)
+from IPython.kernel.fcutil import FCServiceFactory, FURLError
+from IPython.utils.traitlets import Instance, Unicode
+
+
+#-----------------------------------------------------------------------------
+# Module level variables
+#-----------------------------------------------------------------------------
+
+
+#: The default config file name for this application
+default_config_file_name = u'ipcontroller_config.py'
+
+
+_description = """Start the IPython controller for parallel computing.
+
+The IPython controller provides a gateway between the IPython engines and
+clients. The controller needs to be started before the engines and can be
+configured using command line options or using a cluster directory. Cluster
+directories contain config, log and security files and are usually located in
+your .ipython directory and named as "cluster_<profile>". See the --profile
+and --cluster-dir options for details.
+"""
 
 #-----------------------------------------------------------------------------
 # Default interfaces
@@ -92,25 +112,27 b' class FCEngineServiceFactory(FCServiceFactory):'
 
 
 #-----------------------------------------------------------------------------
-# The main application
+# Command line options
 #-----------------------------------------------------------------------------
 
 
-cl_args = (
+class IPControllerAppConfigLoader(ClusterDirConfigLoader):
+
+    def _add_arguments(self):
+        super(IPControllerAppConfigLoader, self)._add_arguments()
+        paa = self.parser.add_argument
     # Client config
-    (('--client-ip',), dict(
+        paa('--client-ip',
         type=str, dest='FCClientServiceFactory.ip', 
         help='The IP address or hostname the controller will listen on for '
         'client connections.',
         metavar='FCClientServiceFactory.ip')
-    ),
-    (('--client-port',), dict(
+        paa('--client-port',
         type=int, dest='FCClientServiceFactory.port', 
         help='The port the controller will listen on for client connections. '
         'The default is to use 0, which will autoselect an open port.',
         metavar='FCClientServiceFactory.port')
-    ),
-    (('--client-location',), dict(
+        paa('--client-location',), dict(
         type=str, dest='FCClientServiceFactory.location',
         help='The hostname or IP that clients should connect to. This does '
         'not control which interface the controller listens on. Instead, this '
@@ -118,21 +140,18 b' cl_args = ('
         'clients know where to connect. Useful if the controller is listening '
         'on multiple interfaces.',
         metavar='FCClientServiceFactory.location')
-    ),
     # Engine config
-    (('--engine-ip',), dict(
+        paa('--engine-ip',
         type=str, dest='FCEngineServiceFactory.ip', 
         help='The IP address or hostname the controller will listen on for '
         'engine connections.',
         metavar='FCEngineServiceFactory.ip')
-    ),
-    (('--engine-port',), dict(
+        paa('--engine-port',
         type=int, dest='FCEngineServiceFactory.port',
         help='The port the controller will listen on for engine connections. '
         'The default is to use 0, which will autoselect an open port.',
         metavar='FCEngineServiceFactory.port')
-    ),
-    (('--engine-location',), dict(
+        paa('--engine-location',
         type=str, dest='FCEngineServiceFactory.location',
         help='The hostname or IP that engines should connect to. This does '
         'not control which interface the controller listens on. Instead, this '
@@ -140,60 +159,49 b' cl_args = ('
         'engines know where to connect. Useful if the controller is listening '
         'on multiple interfaces.',
         metavar='FCEngineServiceFactory.location')
-    ),
     # Global config
-    (('--log-to-file',), dict(
+        paa('--log-to-file',
         action='store_true', dest='Global.log_to_file',
         help='Log to a file in the log directory (default is stdout)')
-    ),
-    (('-r','--reuse-furls'), dict(
+        paa('-r','--reuse-furls', 
         action='store_true', dest='Global.reuse_furls',
         help='Try to reuse all FURL files. If this is not set all FURL files '
         'are deleted before the controller starts. This must be set if '
         'specific ports are specified by --engine-port or --client-port.')
-    ),
-    (('--no-secure',), dict(
+        paa('--no-secure',
         action='store_false', dest='Global.secure',
         help='Turn off SSL encryption for all connections.')
-    ),
-    (('--secure',), dict(
+        paa('--secure',
         action='store_true', dest='Global.secure',
         help='Turn off SSL encryption for all connections.')
-    )
-)
 
 
-_description = """Start the IPython controller for parallel computing.
-
-The IPython controller provides a gateway between the IPython engines and
-clients. The controller needs to be started before the engines and can be
-configured using command line options or using a cluster directory. Cluster
-directories contain config, log and security files and are usually located in
-your .ipython directory and named as "cluster_<profile>". See the --profile
-and --cluster-dir options for details.
-"""
-
-default_config_file_name = u'ipcontroller_config.py'
+#-----------------------------------------------------------------------------
+# The main application
+#-----------------------------------------------------------------------------
 
 
 class IPControllerApp(ApplicationWithClusterDir):
 
     name = u'ipcontroller'
     description = _description
-    config_file_name = default_config_file_name
+    command_line_loader = IPControllerAppConfigLoader
+    default_config_file_name = default_config_file_name
     auto_create_cluster_dir = True
-    cl_arguments = Application.cl_arguments + cl_args
 
     def create_default_config(self):
         super(IPControllerApp, self).create_default_config()
-        self.default_config.Global.reuse_furls = False
-        self.default_config.Global.secure = True
+        # Don't set defaults for Global.secure or Global.reuse_furls
+        # as those are set in a component.
         self.default_config.Global.import_statements = []
         self.default_config.Global.clean_logs = True
 
-    def post_load_command_line_config(self):
-        # Now setup reuse_furls
-        c = self.command_line_config
+    def pre_construct(self):
+        super(IPControllerApp, self).pre_construct()
+        c = self.master_config
+        # The defaults for these are set in FCClientServiceFactory and
+        # FCEngineServiceFactory, so we only set them here if the global
+        # options have be set to override the class level defaults.
         if hasattr(c.Global, 'reuse_furls'):
             c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls
             c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls
@@ -216,11 +224,19 b' class IPControllerApp(ApplicationWithClusterDir):'
         controller_service = controllerservice.ControllerService()
         controller_service.setServiceParent(self.main_service)
         # The client tub and all its refereceables
+        try:
         csfactory = FCClientServiceFactory(self.master_config, controller_service)
+        except FURLError, e:
+            log.err(e)
+            self.exit(0)
         client_service = csfactory.create()
         client_service.setServiceParent(self.main_service)
         # The engine tub
+        try:
         esfactory = FCEngineServiceFactory(self.master_config, controller_service)
+        except FURLError, e:
+            log.err(e)
+            self.exit(0)
         engine_service = esfactory.create()
         engine_service.setServiceParent(self.main_service)
 

IPython/kernel/ipengineapp.py

@@ -22,39 +22,21 b' from twisted.application import service'
 from twisted.internet import reactor
 from twisted.python import log
 
-from IPython.core.application import Application
-from IPython.kernel.clusterdir import ApplicationWithClusterDir
+from IPython.kernel.clusterdir import (
+    ApplicationWithClusterDir,
+    ClusterDirConfigLoader
+)
 from IPython.kernel.engineconnector import EngineConnector
 from IPython.kernel.engineservice import EngineService
 from IPython.kernel.fcutil import Tub
 from IPython.utils.importstring import import_item
 
 #-----------------------------------------------------------------------------
-# The main application
+# Module level variables
 #-----------------------------------------------------------------------------
 
-cl_args = (
-    # Controller config
-    (('--furl-file',), dict(
-        type=unicode, dest='Global.furl_file',
-        help='The full location of the file containing the FURL of the '
-        'controller. If this is not given, the FURL file must be in the '
-        'security directory of the cluster directory.  This location is '
-        'resolved using the --profile and --app-dir options.',
-        metavar='Global.furl_file')
-    ),
-    # MPI
-    (('--mpi',), dict(
-        type=str, dest='MPI.use',
-        help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).',
-        metavar='MPI.use')
-    ),
-    # Global config
-    (('--log-to-file',), dict(
-        action='store_true', dest='Global.log_to_file',
-        help='Log to a file in the log directory (default is stdout)')
-    )
-)
+#: The default config file name for this application
+default_config_file_name = u'ipengine_config.py'
 
 
 mpi4py_init = """from mpi4py import MPI as mpi
@@ -62,6 +44,7 b' mpi.size = mpi.COMM_WORLD.Get_size()'
 mpi.rank = mpi.COMM_WORLD.Get_rank()
 """
 
+
 pytrilinos_init = """from PyTrilinos import Epetra
 class SimpleStruct:
 pass
@@ -71,9 +54,6 b' mpi.size = 0'
 """
 
 
-default_config_file_name = u'ipengine_config.py'
-
-
 _description = """Start an IPython engine for parallel computing.\n\n
 
 IPython engines run in parallel and perform computations on behalf of a client
@@ -84,14 +64,47 b' usually located in your .ipython directory and named as "cluster_<profile>".'
 See the --profile and --cluster-dir options for details.
 """
 
+#-----------------------------------------------------------------------------
+# Command line options
+#-----------------------------------------------------------------------------
+
+
+class IPEngineAppConfigLoader(ClusterDirConfigLoader):
+
+    def _add_arguments(self):
+        super(IPEngineAppConfigLoader, self)._add_arguments()
+        paa = self.parser.add_argument
+        # Controller config
+        paa('--furl-file',
+            type=unicode, dest='Global.furl_file',
+            help='The full location of the file containing the FURL of the '
+            'controller. If this is not given, the FURL file must be in the '
+            'security directory of the cluster directory.  This location is '
+            'resolved using the --profile and --app-dir options.',
+            metavar='Global.furl_file')
+        # MPI
+        paa('--mpi',
+            type=str, dest='MPI.use',
+            help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).',
+            metavar='MPI.use')
+        # Global config
+        paa('--log-to-file',
+            action='store_true', dest='Global.log_to_file',
+            help='Log to a file in the log directory (default is stdout)')
+
+
+#-----------------------------------------------------------------------------
+# Main application
+#-----------------------------------------------------------------------------
+
 
 class IPEngineApp(ApplicationWithClusterDir):
 
     name = u'ipengine'
     description = _description
-    config_file_name = default_config_file_name
+    command_line_loader = IPEngineAppConfigLoader
+    default_config_file_name = default_config_file_name
     auto_create_cluster_dir = True
-    cl_arguments = Application.cl_arguments + cl_args
 
     def create_default_config(self):
         super(IPEngineApp, self).create_default_config()

IPython/kernel/launcher.py

@@ -21,11 +21,15 b' import sys'
 
 from IPython.core.component import Component
 from IPython.external import Itpl
-from IPython.utils.traitlets import Str, Int, List, Unicode, Enum
-from IPython.utils.platutils import find_cmd
-from IPython.kernel.twistedutil import gatherBoth, make_deferred, sleep_deferred
+from IPython.utils.traitlets import Str, Int, List, Unicode
+from IPython.utils.path import get_ipython_module_path
+from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError
+from IPython.kernel.twistedutil import (
+    gatherBoth,
+    make_deferred,
+    sleep_deferred
+)
 from IPython.kernel.winhpcjob import (
-    WinHPCJob, WinHPCTask,
     IPControllerTask, IPEngineTask,
     IPControllerJob, IPEngineSetJob
 )
@@ -38,46 +42,23 b' from twisted.internet.error import ProcessDone, ProcessTerminated'
 from twisted.python import log
 from twisted.python.failure import Failure
 
+
 #-----------------------------------------------------------------------------
-# Utilities
+# Paths to the kernel apps
 #-----------------------------------------------------------------------------
 
 
-def find_controller_cmd():
-    """Find the command line ipcontroller program in a cross platform way."""
-    if sys.platform == 'win32':
-        # This logic is needed because the ipcontroller script doesn't
-        # always get installed in the same way or in the same location.
-        from IPython.kernel import ipcontrollerapp
-        script_location = ipcontrollerapp.__file__.replace('.pyc', '.py')
-        # The -u option here turns on unbuffered output, which is required
-        # on Win32 to prevent wierd conflict and problems with Twisted.
-        # Also, use sys.executable to make sure we are picking up the 
-        # right python exe.
-        cmd = [sys.executable, '-u', script_location]
-    else:
-        # ipcontroller has to be on the PATH in this case.
-        cmd = ['ipcontroller']
-    return cmd
-
-
-def find_engine_cmd():
-    """Find the command line ipengine program in a cross platform way."""
-    if sys.platform == 'win32':
-        # This logic is needed because the ipengine script doesn't
-        # always get installed in the same way or in the same location.
-        from IPython.kernel import ipengineapp
-        script_location = ipengineapp.__file__.replace('.pyc', '.py')
-        # The -u option here turns on unbuffered output, which is required
-        # on Win32 to prevent wierd conflict and problems with Twisted.
-        # Also, use sys.executable to make sure we are picking up the 
-        # right python exe.
-        cmd = [sys.executable, '-u', script_location]
-    else:
-        # ipcontroller has to be on the PATH in this case.
-        cmd = ['ipengine']
-    return cmd
+ipcluster_cmd_argv = pycmd2argv(get_ipython_module_path(
+    'IPython.kernel.ipclusterapp'
+))
+
+ipengine_cmd_argv = pycmd2argv(get_ipython_module_path(
+    'IPython.kernel.ipengineapp'
+))
 
+ipcontroller_cmd_argv = pycmd2argv(get_ipython_module_path(
+    'IPython.kernel.ipcontrollerapp'
+))
 
 #-----------------------------------------------------------------------------
 # Base launchers and errors
@@ -333,7 +314,7 b' class LocalProcessLauncher(BaseLauncher):'
 class LocalControllerLauncher(LocalProcessLauncher):
     """Launch a controller as a regular external process."""
 
-    controller_cmd = List(find_controller_cmd(), config=True)
+    controller_cmd = List(ipcontroller_cmd_argv, config=True)
     # Command line arguments to ipcontroller.
     controller_args = List(['--log-to-file','--log-level', '40'], config=True)
 
@@ -351,7 +332,7 b' class LocalControllerLauncher(LocalProcessLauncher):'
 class LocalEngineLauncher(LocalProcessLauncher):
     """Launch a single engine as a regular externall process."""
 
-    engine_cmd = List(find_engine_cmd(), config=True)
+    engine_cmd = List(ipengine_cmd_argv, config=True)
     # Command line arguments for ipengine.
     engine_args = List(
         ['--log-to-file','--log-level', '40'], config=True
@@ -462,7 +443,7 b' class MPIExecLauncher(LocalProcessLauncher):'
 class MPIExecControllerLauncher(MPIExecLauncher):
     """Launch a controller using mpiexec."""
 
-    controller_cmd = List(find_controller_cmd(), config=True)
+    controller_cmd = List(ipcontroller_cmd_argv, config=True)
     # Command line arguments to ipcontroller.
     controller_args = List(['--log-to-file','--log-level', '40'], config=True)
     n = Int(1, config=False)
@@ -481,7 +462,7 b' class MPIExecControllerLauncher(MPIExecLauncher):'
 
 class MPIExecEngineSetLauncher(MPIExecLauncher):
 
-    engine_cmd = List(find_engine_cmd(), config=True)
+    engine_cmd = List(ipengine_cmd_argv, config=True)
     # Command line arguments for ipengine.
     engine_args = List(
         ['--log-to-file','--log-level', '40'], config=True
@@ -557,7 +538,10 b' class SSHEngineSetLauncher(BaseLauncher):'
 # This is only used on Windows.
 def find_job_cmd():
     if os.name=='nt':
+        try:
         return find_cmd('job')
+        except FindCmdError:
+            return 'job'
     else:
         return 'job'
 
@@ -831,28 +815,10 b' class PBSEngineSetLauncher(PBSLauncher):'
 #-----------------------------------------------------------------------------
 
 
-def find_ipcluster_cmd():
-    """Find the command line ipcluster program in a cross platform way."""
-    if sys.platform == 'win32':
-        # This logic is needed because the ipcluster script doesn't
-        # always get installed in the same way or in the same location.
-        from IPython.kernel import ipclusterapp
-        script_location = ipclusterapp.__file__.replace('.pyc', '.py')
-        # The -u option here turns on unbuffered output, which is required
-        # on Win32 to prevent wierd conflict and problems with Twisted.
-        # Also, use sys.executable to make sure we are picking up the 
-        # right python exe.
-        cmd = [sys.executable, '-u', script_location]
-    else:
-        # ipcontroller has to be on the PATH in this case.
-        cmd = ['ipcluster']
-    return cmd
-
-
 class IPClusterLauncher(LocalProcessLauncher):
     """Launch the ipcluster program in an external process."""
 
-    ipcluster_cmd = List(find_ipcluster_cmd(), config=True)
+    ipcluster_cmd = List(ipcluster_cmd_argv, config=True)
     # Command line arguments to pass to ipcluster.
     ipcluster_args = List(
         ['--clean-logs', '--log-to-file', '--log-level', '40'], config=True)

IPython/kernel/map.py

@@ -21,7 +21,7 b' __docformat__ = "restructuredtext en"'
 
 import types
 
-from IPython.utils.genutils import flatten as genutil_flatten
+from IPython.utils.data import flatten as utils_flatten
 
 #-------------------------------------------------------------------------------
 # Figure out which array packages are present and their array types
@@ -87,7 +87,7 b' class Map:'
                 return m['module'].concatenate(listOfPartitions)
         # Next try for Python sequence types
         if isinstance(testObject, (types.ListType, types.TupleType)):
-            return genutil_flatten(listOfPartitions)
+            return utils_flatten(listOfPartitions)
         # If we have scalars, just return listOfPartitions
         return listOfPartitions
 

IPython/kernel/mapper.py

@@ -18,8 +18,7 b' __docformat__ = "restructuredtext en"'
 from types import FunctionType
 from zope.interface import Interface, implements
 from IPython.kernel.task import MapTask
-from IPython.kernel.twistedutil import DeferredList, gatherBoth
-from IPython.kernel.util import printer
+from IPython.kernel.twistedutil import gatherBoth
 from IPython.kernel.error import collect_exceptions
 
 #----------------------------------------------------------------------------

IPython/kernel/multiengine.py

@@ -27,24 +27,17 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
-from new import instancemethod
-from types import FunctionType
-
-from twisted.application import service
 from twisted.internet import defer, reactor
 from twisted.python import log, components, failure
-from zope.interface import Interface, implements, Attribute
+from zope.interface import Interface, implements
 
-from IPython.utils import growl
-from IPython.kernel.util import printer
 from IPython.kernel.twistedutil import gatherBoth
-from IPython.kernel import map as Map
 from IPython.kernel import error
 from IPython.kernel.pendingdeferred import PendingDeferredManager, two_phase
-from IPython.kernel.controllerservice import \
-    ControllerAdapterBase, \
-    ControllerService, \
+from IPython.kernel.controllerservice import (
+    ControllerAdapterBase,
     IControllerBase
+)
 
 
 #-------------------------------------------------------------------------------

IPython/kernel/multiengineclient.py

@@ -17,13 +17,17 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 
 import sys
-import linecache
 import warnings
 
 from twisted.python import components
 from twisted.python.failure import Failure
 from zope.interface import Interface, implements, Attribute
 
+try:
+    from foolscap.api import DeadReferenceError
+except ImportError:
+    from foolscap import DeadReferenceError
+
 from IPython.utils.coloransi import TermColors
 
 from IPython.kernel.twistedutil import blockingCallFromThread
@@ -307,85 +311,6 b' class InteractiveMultiEngineClient(object):'
         """Return the number of available engines."""
         return len(self.get_ids())
     
-    #---------------------------------------------------------------------------
-    # Make this a context manager for with
-    #---------------------------------------------------------------------------
-    
-    def findsource_file(self,f):
-        linecache.checkcache()
-        s = findsource(f.f_code) # findsource is not defined!
-        lnum = f.f_lineno
-        wsource = s[0][f.f_lineno:]
-        return strip_whitespace(wsource)
-
-    def findsource_ipython(self,f):
-        from IPython.core import ipapi
-        self.ip = ipapi.get()
-        wsource = [l+'\n' for l in
-                   self.ip.input_hist_raw[-1].splitlines()[1:]] 
-        return strip_whitespace(wsource)
-        
-    def __enter__(self):
-        f = sys._getframe(1)
-        local_ns = f.f_locals
-        global_ns = f.f_globals
-        if f.f_code.co_filename == '<ipython console>':
-            s = self.findsource_ipython(f)
-        else:
-            s = self.findsource_file(f)
-
-        self._with_context_result = self.execute(s)
-
-    def __exit__ (self, etype, value, tb):
-        if issubclass(etype,error.StopLocalExecution):
-            return True
-
-
-def remote():
-    m = 'Special exception to stop local execution of parallel code.'
-    raise error.StopLocalExecution(m)
-
-def strip_whitespace(source):
-    # Expand tabs to avoid any confusion.
-    wsource = [l.expandtabs(4) for l in source]
-    # Detect the indentation level
-    done = False
-    for line in wsource:
-        if line.isspace():
-            continue
-        for col,char in enumerate(line):
-            if char != ' ':
-                done = True
-                break
-        if done:
-            break
-    # Now we know how much leading space there is in the code.  Next, we
-    # extract up to the first line that has less indentation.
-    # WARNINGS: we skip comments that may be misindented, but we do NOT yet
-    # detect triple quoted strings that may have flush left text.
-    for lno,line in enumerate(wsource):
-        lead = line[:col]
-        if lead.isspace():
-            continue
-        else:
-            if not lead.lstrip().startswith('#'):
-                break
-    # The real 'with' source is up to lno
-    src_lines = [l[col:] for l in wsource[:lno+1]]
-
-    # Finally, check that the source's first non-comment line begins with the
-    # special call 'remote()'
-    for nline,line in enumerate(src_lines):
-        if line.isspace() or line.startswith('#'):
-            continue
-        if 'remote()' in line:
-            break
-        else:
-            raise ValueError('remote() call missing at the start of code')
-    src = ''.join(src_lines[nline+1:])
-    #print 'SRC:\n<<<<<<<>>>>>>>\n%s<<<<<>>>>>>' % src  # dbg
-    return src
-
 
 #-------------------------------------------------------------------------------
 # The top-level MultiEngine client adaptor
@@ -445,17 +370,30 b' class FullBlockingMultiEngineClient(InteractiveMultiEngineClient):'
     def _findTargetsAndBlock(self, targets=None, block=None):
         return self._findTargets(targets), self._findBlock(block) 
     
+    def _bcft(self, *args, **kwargs):
+        try:
+            result = blockingCallFromThread(*args, **kwargs)
+        except DeadReferenceError:
+            raise error.ConnectionError(
+                """A connection error has occurred in trying to connect to the
+                controller. This is usually caused by the controller dying or 
+                being restarted. To resolve this issue try recreating the 
+                multiengine client."""
+            )
+        else:
+            return result
+
     def _blockFromThread(self, function, *args, **kwargs):
         block = kwargs.get('block', None)
         if block is None:
             raise error.MissingBlockArgument("'block' keyword argument is missing")
-        result = blockingCallFromThread(function, *args, **kwargs)
+        result = self._bcft(function, *args, **kwargs)
         if not block:
             result = PendingResult(self, result)
         return result
     
     def get_pending_deferred(self, deferredID, block):
-        return blockingCallFromThread(self.smultiengine.get_pending_deferred, deferredID, block)
+        return self._bcft(self.smultiengine.get_pending_deferred, deferredID, block)
     
     def barrier(self, pendingResults):
         """Synchronize a set of `PendingResults`.
@@ -505,7 +443,7 b' class FullBlockingMultiEngineClient(InteractiveMultiEngineClient):'
         controller.  This method allows the user to clear out all un-retrieved
         results on the controller. 
         """
-        r = blockingCallFromThread(self.smultiengine.clear_pending_deferreds)
+        r = self._bcft(self.smultiengine.clear_pending_deferreds)
         return r
     
     clear_pending_results = flush
@@ -529,7 +467,7 b' class FullBlockingMultiEngineClient(InteractiveMultiEngineClient):'
                 at a later time.
         """
         targets, block = self._findTargetsAndBlock(targets, block)
-        result = blockingCallFromThread(self.smultiengine.execute, lines,
+        result = self._bcft(self.smultiengine.execute, lines,
             targets=targets, block=block)
         if block:
             result = ResultList(result)
@@ -647,7 +585,7 b' class FullBlockingMultiEngineClient(InteractiveMultiEngineClient):'
                 at a later time.
         """
         targets, block = self._findTargetsAndBlock(targets, block)
-        result = blockingCallFromThread(self.smultiengine.get_result, i, targets=targets, block=block)
+        result = self._bcft(self.smultiengine.get_result, i, targets=targets, block=block)
         if block:
             result = ResultList(result)
         else:
@@ -773,7 +711,7 b' class FullBlockingMultiEngineClient(InteractiveMultiEngineClient):'
         """
         Returns the ids of currently registered engines.
         """
-        result = blockingCallFromThread(self.smultiengine.get_ids)
+        result = self._bcft(self.smultiengine.get_ids)
         return result
         
     #---------------------------------------------------------------------------

IPython/kernel/multienginefc.py

@@ -22,12 +22,14 b' from types import FunctionType'
 
 from zope.interface import Interface, implements
 from twisted.internet import defer
-from twisted.python import components, failure, log
+from twisted.python import components, failure
 
+try:
+    from foolscap.api import Referenceable
+except ImportError:
 from foolscap import Referenceable
 
 from IPython.kernel import error 
-from IPython.kernel.util import printer
 from IPython.kernel import map as Map
 from IPython.kernel.parallelfunction import ParallelFunction
 from IPython.kernel.mapper import (
@@ -36,14 +38,15 b' from IPython.kernel.mapper import ('
     IMapper
 )
 from IPython.kernel.twistedutil import gatherBoth
-from IPython.kernel.multiengine import (MultiEngine,
+from IPython.kernel.multiengine import (
     IMultiEngine,
     IFullSynchronousMultiEngine,
     ISynchronousMultiEngine)
-from IPython.kernel.multiengineclient import wrapResultList
 from IPython.kernel.pendingdeferred import PendingDeferredManager
-from IPython.kernel.pickleutil import (can, canDict,
-    canSequence, uncan, uncanDict, uncanSequence)
+from IPython.kernel.pickleutil import (
+    canDict,
+    canSequence, uncanDict, uncanSequence
+)
 
 from IPython.kernel.clientinterfaces import (
     IFCClientInterfaceProvider, 

IPython/kernel/pbutil.py

@@ -19,7 +19,6 b' import cPickle as pickle'
 
 from twisted.python.failure import Failure
 from twisted.python import failure
-import threading, sys
 
 from IPython.kernel import pbconfig
 from IPython.kernel.error import PBMessageSizeError, UnpickleableException
@@ -58,7 +57,7 b' def unpackageFailure(r):'
                 result = pickle.loads(r[8:])
             except pickle.PickleError:
                 return failure.Failure( \
-                    FailureUnpickleable("Could not unpickle failure."))
+                    UnpickleableException("Could not unpickle failure."))
             else:
                 return result
     return r

IPython/kernel/pendingdeferred.py

@@ -22,15 +22,11 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
-from twisted.application import service
-from twisted.internet import defer, reactor
-from twisted.python import log, components, failure
-from zope.interface import Interface, implements, Attribute
+from twisted.internet import defer
+from twisted.python import failure
 
-from IPython.kernel.twistedutil import gatherBoth
 from IPython.kernel import error
 from IPython.external import guid
-from IPython.utils import growl
 
 class PendingDeferredManager(object):
     """A class to track pending deferreds.

IPython/kernel/pickleutil.py

@@ -16,7 +16,6 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 
 from types import FunctionType
-from twisted.python import log
 
 class CannedObject(object):
     pass

IPython/kernel/task.py

@@ -19,19 +19,18 b' __docformat__ = "restructuredtext en"'
 # Tell nose to skip the testing of this module
 __test__ = {}
 
-import copy, time
+import time
 from types import FunctionType
 
-import zope.interface as zi, string
+import zope.interface as zi
 from twisted.internet import defer, reactor
 from twisted.python import components, log, failure
 
-from IPython.kernel.util import printer
 from IPython.kernel import engineservice as es, error
 from IPython.kernel import controllerservice as cs
-from IPython.kernel.twistedutil import gatherBoth, DeferredList
+from IPython.kernel.twistedutil import DeferredList
 
-from IPython.kernel.pickleutil import can, uncan, CannedFunction
+from IPython.kernel.pickleutil import can, uncan
 
 #-----------------------------------------------------------------------------
 # Definition of the Task objects

IPython/kernel/taskclient.py

@@ -19,7 +19,12 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 
 from zope.interface import Interface, implements
-from twisted.python import components, log
+from twisted.python import components
+
+try:
+    from foolscap.api import DeadReferenceError
+except ImportError:
+    from foolscap import DeadReferenceError
 
 from IPython.kernel.twistedutil import blockingCallFromThread
 from IPython.kernel import task, error
@@ -59,6 +64,19 b' class BlockingTaskClient(object):'
         self.task_controller = task_controller
         self.block = True
         
+    def _bcft(self, *args, **kwargs):
+        try:
+            result = blockingCallFromThread(*args, **kwargs)
+        except DeadReferenceError:
+            raise error.ConnectionError(
+                """A connection error has occurred in trying to connect to the
+                controller. This is usually caused by the controller dying or 
+                being restarted. To resolve this issue try recreating the 
+                task client."""
+            )
+        else:
+            return result
+
     def run(self, task, block=False):
         """Run a task on the `TaskController`.
         
@@ -71,7 +89,7 b' class BlockingTaskClient(object):'
         :Returns: The int taskid of the submitted task.  Pass this to 
             `get_task_result` to get the `TaskResult` object.
         """
-        tid = blockingCallFromThread(self.task_controller.run, task)
+        tid = self._bcft(self.task_controller.run, task)
         if block:
             return self.get_task_result(tid, block=True)
         else:
@@ -89,7 +107,7 b' class BlockingTaskClient(object):'
         
         :Returns: A `TaskResult` object that encapsulates the task result.
         """
-        return blockingCallFromThread(self.task_controller.get_task_result,
+        return self._bcft(self.task_controller.get_task_result,
             taskid, block)
     
     def abort(self, taskid):
@@ -100,7 +118,7 b' class BlockingTaskClient(object):'
             taskid : int
                 The taskid of the task to be aborted.
         """
-        return blockingCallFromThread(self.task_controller.abort, taskid)
+        return self._bcft(self.task_controller.abort, taskid)
     
     def barrier(self, taskids):
         """Block until a set of tasks are completed.
@@ -109,7 +127,7 b' class BlockingTaskClient(object):'
             taskids : list, tuple
                 A sequence of taskids to block on.
         """
-        return blockingCallFromThread(self.task_controller.barrier, taskids)
+        return self._bcft(self.task_controller.barrier, taskids)
     
     def spin(self):
         """
@@ -118,7 +136,7 b' class BlockingTaskClient(object):'
         This method only needs to be called in unusual situations where the
         scheduler is idle for some reason. 
         """
-        return blockingCallFromThread(self.task_controller.spin)
+        return self._bcft(self.task_controller.spin)
     
     def queue_status(self, verbose=False):
         """
@@ -132,7 +150,7 b' class BlockingTaskClient(object):'
         :Returns:
             A dict with the queue status.
         """
-        return blockingCallFromThread(self.task_controller.queue_status, verbose)
+        return self._bcft(self.task_controller.queue_status, verbose)
     
     def clear(self):
         """
@@ -143,7 +161,7 b' class BlockingTaskClient(object):'
         tasks.  Users should call this periodically to clean out these
         cached task results.
         """
-        return blockingCallFromThread(self.task_controller.clear)
+        return self._bcft(self.task_controller.clear)
     
     def map(self, func, *sequences):
         """

IPython/kernel/taskfc.py

@@ -19,17 +19,17 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 
 import cPickle as pickle
-import xmlrpclib, copy
 
 from zope.interface import Interface, implements
 from twisted.internet import defer
-from twisted.python import components, failure
+from twisted.python import components
 
+try:
+    from foolscap.api import Referenceable
+except ImportError:
 from foolscap import Referenceable
 
-from IPython.kernel.twistedutil import blockingCallFromThread
-from IPython.kernel import error, task as taskmodule, taskclient
-from IPython.kernel.pickleutil import can, uncan
+from IPython.kernel import task as taskmodule
 from IPython.kernel.clientinterfaces import (
     IFCClientInterfaceProvider, 
     IBlockingClientAdaptor

IPython/kernel/tests/test_enginefc.py

@@ -53,8 +53,8 b' class EngineFCTest(DeferredTestCase,'
       # Start a server and append to self.servers
       self.controller_reference = FCRemoteEngineRefFromService(self)
       self.controller_tub = Tub()
-      self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
-      self.controller_tub.setLocation('127.0.0.1:10105')
+      self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
+      self.controller_tub.setLocation('127.0.0.1:10111')
      
       furl = self.controller_tub.registerReference(self.controller_reference)
       self.controller_tub.startService()

IPython/kernel/tests/test_multienginefc.py

@@ -27,7 +27,7 b' from IPython.kernel.multiengine import IMultiEngine'
 from IPython.kernel.tests.multienginetest import IFullSynchronousMultiEngineTestCase
 from IPython.kernel.multienginefc import IFCSynchronousMultiEngine
 from IPython.kernel import multiengine as me
-from IPython.kernel.clientconnector import ClientConnector
+from IPython.kernel.clientconnector import AsyncClientConnector
 from IPython.kernel.parallelfunction import ParallelFunction
 from IPython.kernel.error import CompositeError
 from IPython.kernel.util import printer
@@ -40,15 +40,8 b' def _raise_it(f):'
         e.raise_exception()
 
 
-class FullSynchronousMultiEngineTestCase(DeferredTestCase, IFullSynchronousMultiEngineTestCase):
-
-    # XXX (fperez) this is awful: I'm fully disabling this entire test class.
-    # Right now it's blocking the tests from running at all, and I don't know
-    # how to fix it.  I hope Brian can have a stab at it, but at least by doing
-    # this we can run the entire suite to completion.
-    # Once the problem is cleared, remove this skip method.
-    skip = True
-    # END XXX
+class FullSynchronousMultiEngineTestCase(
+    DeferredTestCase, IFullSynchronousMultiEngineTestCase):
     
     def setUp(self):
     
@@ -60,14 +53,14 b' class FullSynchronousMultiEngineTestCase(DeferredTestCase, IFullSynchronousMulti'
         self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
 
         self.controller_tub = Tub()
-        self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
-        self.controller_tub.setLocation('127.0.0.1:10105')
+        self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
+        self.controller_tub.setLocation('127.0.0.1:10111')
     
         furl = self.controller_tub.registerReference(self.mec_referenceable)
         self.controller_tub.startService()
     
-        self.client_tub = ClientConnector()
-        d = self.client_tub.get_multiengine_client(furl)
+        self.client_tub = AsyncClientConnector()
+        d = self.client_tub.get_multiengine_client(furl_or_file=furl)
         d.addCallback(self.handle_got_client)
         return d
     
@@ -150,3 +143,4 b' class FullSynchronousMultiEngineTestCase(DeferredTestCase, IFullSynchronousMulti'
         d = f(range(10))
         d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
         return d
+

IPython/kernel/tests/test_taskfc.py

@@ -31,7 +31,7 b' from IPython.kernel.multienginefc import IFCSynchronousMultiEngine'
 from IPython.kernel.taskfc import IFCTaskController
 from IPython.kernel.util import printer
 from IPython.kernel.tests.tasktest import ITaskControllerTestCase
-from IPython.kernel.clientconnector import ClientConnector
+from IPython.kernel.clientconnector import AsyncClientConnector
 from IPython.kernel.error import CompositeError
 from IPython.kernel.parallelfunction import ParallelFunction
 
@@ -48,14 +48,6 b' def _raise_it(f):'
 
 class TaskTest(DeferredTestCase, ITaskControllerTestCase):
 
-    # XXX (fperez) this is awful: I'm fully disabling this entire test class.
-    # Right now it's blocking the tests from running at all, and I don't know
-    # how to fix it.  I hope Brian can have a stab at it, but at least by doing
-    # this we can run the entire suite to completion.
-    # Once the problem is cleared, remove this skip method.
-    skip = True
-    # END XXX
- 
     def setUp(self):
     
         self.engines = []
@@ -70,17 +62,17 b' class TaskTest(DeferredTestCase, ITaskControllerTestCase):'
         self.tc_referenceable = IFCTaskController(self.itc)
     
         self.controller_tub = Tub()
-        self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
-        self.controller_tub.setLocation('127.0.0.1:10105')
+        self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
+        self.controller_tub.setLocation('127.0.0.1:10111')
     
         mec_furl = self.controller_tub.registerReference(self.mec_referenceable)
         tc_furl = self.controller_tub.registerReference(self.tc_referenceable)
         self.controller_tub.startService()
     
-        self.client_tub = ClientConnector()
-        d = self.client_tub.get_multiengine_client(mec_furl)
+        self.client_tub = AsyncClientConnector()
+        d = self.client_tub.get_multiengine_client(furl_or_file=mec_furl)
         d.addCallback(self.handle_mec_client)
-        d.addCallback(lambda _: self.client_tub.get_task_client(tc_furl))
+        d.addCallback(lambda _: self.client_tub.get_task_client(furl_or_file=tc_furl))
         d.addCallback(self.handle_tc_client)
         return d
     
@@ -167,3 +159,4 b' class TaskTest(DeferredTestCase, ITaskControllerTestCase):'
         d = f(range(10))
         d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
         return d
+

IPython/kernel/twistedutil.py

@@ -15,7 +15,7 b''
 #-----------------------------------------------------------------------------
 
 import os, sys
-import threading, Queue, atexit
+import threading, Queue
 
 import twisted
 from twisted.internet import defer, reactor

IPython/kernel/winhpcjob.py

@@ -23,12 +23,10 b' import re'
 import uuid
 
 from xml.etree import ElementTree as ET
-from xml.dom import minidom
 
 from IPython.core.component import Component
-from IPython.external import Itpl
 from IPython.utils.traitlets import (
-    Str, Int, List, Unicode, Instance,
+    Str, Int, List, Instance,
     Enum, Bool, CStr
 )
 

IPython/lib/backgroundjobs.py

@@ -31,7 +31,7 b' import sys'
 import threading
 
 from IPython.core.ultratb import AutoFormattedTB
-from IPython.utils.genutils import warn,error
+from IPython.utils.warn import warn, error
 
 class BackgroundJobManager:
     """Class to manage a pool of backgrounded threaded jobs.

IPython/lib/deepreload.py

@@ -1,15 +1,18 b''
 # -*- coding: utf-8 -*-
 """
 A module to change reload() so that it acts recursively.
-To enable it type:
-    >>> import __builtin__, deepreload
-    >>> __builtin__.reload = deepreload.reload
+To enable it type::
 
-You can then disable it with:
-    >>> __builtin__.reload = deepreload.original_reload
+    import __builtin__, deepreload
+    __builtin__.reload = deepreload.reload
     
-Alternatively, you can add a dreload builtin alongside normal reload with:
-    >>> __builtin__.dreload = deepreload.reload
+You can then disable it with::
+
+    __builtin__.reload = deepreload.original_reload
+    
+Alternatively, you can add a dreload builtin alongside normal reload with::
+
+    __builtin__.dreload = deepreload.reload
     
 This code is almost entirely based on knee.py from the standard library.
 """

IPython/lib/demo.py

@@ -176,7 +176,8 b' import shlex'
 import sys
 
 from IPython.utils.PyColorize import Parser
-from IPython.utils.genutils import marquee, file_read, file_readlines, Term
+from IPython.utils.io import file_read, file_readlines, Term
+from IPython.utils.text import marquee
 
 __all__ = ['Demo','IPythonDemo','LineDemo','IPythonLineDemo','DemoError']
 
@@ -543,7 +544,7 b' class ClearMixin(object):'
         """Method called before executing each block.
         
         This one simply clears the screen."""
-        from IPython.utils.platutils import term_clear
+        from IPython.utils.terminal import term_clear
         term_clear()
 
 class ClearDemo(ClearMixin,Demo):

IPython/lib/pylabtools.py

@@ -12,10 +12,12 b' Fernando Perez.'
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
+
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
-from IPython.utils.genutils import flag_calls
+
+from IPython.utils.decorators import flag_calls
 
 #-----------------------------------------------------------------------------
 # Main classes and functions

IPython/quarantine/InterpreterExec.py

@@ -50,7 +50,7 b' del InteractiveShell,prefilter_shell'
 
 # Provide pysh and further shell-oriented services
 import os,sys,shutil
-from IPython.utils.genutils import system,shell,getoutput,getoutputerror
+from IPython.utils.process import system,shell,getoutput,getoutputerror
 
 # Short aliases for getting shell output as a string and a list
 sout = getoutput

IPython/quarantine/ext_rescapture.py

@@ -10,6 +10,7 b' var = !ls'
 
 from IPython.core import ipapi
 from IPython.core.error import TryNext
+from IPython.utils.text import make_quoted_expr
 from IPython.utils.genutils import *
 
 ip = ipapi.get()

IPython/quarantine/ipy_greedycompleter.py

@@ -12,7 +12,7 b' do the same in default completer.'
 from IPython.core import ipapi
 from IPython.core.error import TryNext
 from IPython.utils import generics
-from IPython.utils.genutils import dir2
+from IPython.utils.dir2 import dir2
 
 def attr_matches(self, text):
     """Compute matches when text contains a dot.

IPython/quarantine/ipy_jot.py

@@ -16,6 +16,7 b' import pickleshare'
 import inspect,pickle,os,sys,textwrap
 from IPython.core.fakemodule import FakeModule
 from IPython.utils.ipstruct import Struct 
+from IPython.utils.warn import error
 
 
 def refresh_variables(ip, key=None):

IPython/quarantine/ipy_pydb.py

@@ -1,6 +1,6 b''
 import inspect
 from IPython.core import ipapi
-from IPython.utils.genutils import arg_split
+from IPython.utils.process import arg_split
 ip = ipapi.get()
 
 from IPython.core import debugger

IPython/quarantine/jobctrl.py

@@ -45,10 +45,9 b' from subprocess import *'
 import os,shlex,sys,time
 import threading,Queue
 
-from IPython.utils import genutils
-
 from IPython.core import ipapi
 from IPython.core.error import TryNext
+from IPython.utils.text import make_quoted_expr
 
 if os.name == 'nt':
     def kill_process(pid):
@@ -126,8 +125,8 b' def jobctrl_prefilter_f(self,line):'
         
         line = ip.expand_aliases(fn,rest)
         if not _jobq:
-            return 'get_ipython().startjob(%s)' % genutils.make_quoted_expr(line)
-        return 'get_ipython().jobq(%s)' % genutils.make_quoted_expr(line)
+            return 'get_ipython().startjob(%s)' % make_quoted_expr(line)
+        return 'get_ipython().jobq(%s)' % make_quoted_expr(line)
 
     raise TryNext
 

IPython/testing/__init__.py

@@ -1,6 +1,17 b''
 """Testing support (tools to test IPython itself).
 """
 
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Functions
+#-----------------------------------------------------------------------------
+
 # User-level entry point for testing
 def test():
     """Run the entire IPython test suite.

IPython/testing/_doctest26.py

@@ -4,6 +4,17 b' This is just so we can use 2.6 features when running in 2.5, the code below is'
 copied verbatim from the stdlib's collections and doctest modules.
 """
 
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
 from keyword import iskeyword as _iskeyword
 from operator import itemgetter as _itemgetter
 import sys as _sys

IPython/testing/_paramtestpy2.py

@@ -1,10 +1,17 b''
 """Implementation of the parametric test support for Python 2.x
 """
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
-# Stdlib
 import unittest
 from compiler.consts import CO_GENERATOR
 

IPython/testing/_paramtestpy3.py

@@ -3,11 +3,18 b''
 Thanks for the py3 version to Robert Collins, from the Testing in Python
 mailing list.
 """
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
-# Stdlib
 import unittest
 from unittest import TestSuite
 

IPython/testing/globalipapp.py

@@ -9,18 +9,22 b' done.'
 from __future__ import absolute_import
 
 #-----------------------------------------------------------------------------
-# Module imports
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
 #-----------------------------------------------------------------------------
 
-# From the standard library
 import __builtin__
 import commands
-import new
 import os
 import sys
 
 from . import tools
-from IPython.utils.genutils import Term
 
 #-----------------------------------------------------------------------------
 # Functions
@@ -35,7 +39,7 b' class py_file_finder(object):'
         self.test_filename = test_filename
         
     def __call__(self,name):
-        from IPython.utils.genutils import get_py_filename
+        from IPython.utils.path import get_py_filename
         try:
             return get_py_filename(name)
         except IOError:
@@ -99,6 +103,7 b' def get_ipython():'
     # This will get replaced by the real thing once we start IPython below
     return start_ipython()
 
+
 def start_ipython():
     """Start a global IPython shell, which we need for IPython-specific syntax.
     """
@@ -110,7 +115,7 b' def start_ipython():'
     start_ipython.already_called = True
 
     # Ok,  first time we're called, go ahead
-    from IPython.core import ipapp, iplib
+    from IPython.core import iplib
     
     def xsys(cmd):
         """Execute a command and print its output.
@@ -128,26 +133,27 b' def start_ipython():'
     _main = sys.modules.get('__main__')
 
     # Create custom argv and namespaces for our IPython to be test-friendly
-    argv = tools.default_argv()
-    user_ns, global_ns = iplib.make_user_namespaces(ipnsdict(), {})
+    config = tools.default_config()
     
     # Create and initialize our test-friendly IPython instance.
-    ip = ipapp.IPythonApp(argv, user_ns=user_ns, user_global_ns=global_ns)
-    ip.initialize()
+    shell = iplib.InteractiveShell(
+        parent=None, config=config, 
+        user_ns=ipnsdict(), user_global_ns={}
+    )
 
     # A few more tweaks needed for playing nicely with doctests...
     
     # These traps are normally only active for interactive use, set them
     # permanently since we'll be mocking interactive sessions.
-    ip.shell.builtin_trap.set()
+    shell.builtin_trap.set()
 
     # Set error printing to stdout so nose can doctest exceptions
-    ip.shell.InteractiveTB.out_stream = 'stdout'
+    shell.InteractiveTB.out_stream = 'stdout'
     
     # Modify the IPython system call with one that uses getoutput, so that we
     # can capture subcommands and print them to Python's stdout, otherwise the
     # doctest machinery would miss them.
-    ip.shell.system = xsys
+    shell.system = xsys
 
     # IPython is ready, now clean up some global state...
     
@@ -160,7 +166,7 b' def start_ipython():'
     # So that ipython magics and aliases can be doctested (they work by making
     # a call into a global _ip object).  Also make the top-level get_ipython
     # now return this without recursively calling here again.
-    _ip = ip.shell
+    _ip = shell
     get_ipython = _ip.get_ipython
     __builtin__._ip = _ip
     __builtin__.get_ipython = get_ipython

IPython/testing/iptest.py

@@ -17,13 +17,19 b' will change in the future.'
 """
 
 #-----------------------------------------------------------------------------
-# Module imports
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib
 import os
 import os.path as path
-import platform
 import signal
 import sys
 import subprocess
@@ -31,15 +37,6 b' import tempfile'
 import time
 import warnings
 
-
-# Ugly,  but necessary hack to ensure the test suite finds our version of
-# IPython and not a possibly different one that may exist system-wide.
-# Note that this must be done here, so the imports that come next work
-# correctly even if IPython isn't installed yet.
-p = os.path
-ippath = p.abspath(p.join(p.dirname(__file__),'..','..'))
-sys.path.insert(0, ippath)
-
 # Note: monkeypatch!
 # We need to monkeypatch a small problem in nose itself first, before importing
 # it for actual use.  This should get into nose upstream, but its release cycle
@@ -50,11 +47,11 b' import nose.plugins.builtin'
 from nose.core import TestProgram
 
 # Our own imports
-from IPython.core import release
-from IPython.utils import genutils
-from IPython.utils.platutils import find_cmd, FindCmdError
+from IPython.utils.path import get_ipython_module_path
+from IPython.utils.process import find_cmd, pycmd2argv
+from IPython.utils.sysinfo import sys_info
+
 from IPython.testing import globalipapp
-from IPython.testing import tools
 from IPython.testing.plugin.ipdoctest import IPythonDoctest
 
 pjoin = path.join
@@ -64,16 +61,11 b' pjoin = path.join'
 # Globals
 #-----------------------------------------------------------------------------
 
-# By default, we assume IPython has been installed.  But if the test suite is
-# being run from a source tree that has NOT been installed yet, this flag can
-# be set to False by the entry point scripts, to let us know that we must call
-# the source tree versions of the scripts which manipulate sys.path instead of
-# assuming that things exist system-wide.
-INSTALLED = True
 
 #-----------------------------------------------------------------------------
 # Warnings control
 #-----------------------------------------------------------------------------
+
 # Twisted generates annoying warnings with Python 2.6, as will do other code
 # that imports 'sets' as of today
 warnings.filterwarnings('ignore', 'the sets module is deprecated',
@@ -124,9 +116,7 b" have['gobject'] = test_for('gobject')"
 def report():
     """Return a string with a summary report of test-related variables."""
 
-    out = [ genutils.sys_info() ]
-
-    out.append('\nRunning from an installed IPython: %s\n' % INSTALLED)
+    out = [ sys_info() ]
 
     avail = []
     not_avail = []
@@ -155,8 +145,8 b' def make_exclude():'
     
     For the IPythonDoctest plugin, we need to exclude certain patterns that
     cause testing problems.  We should strive to minimize the number of
-    skipped modules, since this means untested code.  As the testing
-    machinery solidifies, this list should eventually become empty.
+    skipped modules, since this means untested code.
+
     These modules and packages will NOT get scanned by nose at all for tests.
     """
     # Simple utility to make IPython paths more readably, we need a lot of
@@ -198,18 +188,12 b' def make_exclude():'
     if not have['objc']:
         exclusions.append(ipjoin('frontend', 'cocoa'))
 
-    if not sys.platform == 'win32':
-        exclusions.append(ipjoin('utils', 'platutils_win32'))
-
     # These have to be skipped on win32 because the use echo, rm, cd, etc.
     # See ticket https://bugs.launchpad.net/bugs/366982
     if sys.platform == 'win32':
         exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
         exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
 
-    if not os.name == 'posix':
-        exclusions.append(ipjoin('utils', 'platutils_posix'))
-
     if not have['pexpect']:
         exclusions.extend([ipjoin('scripts', 'irunner'),
                            ipjoin('lib', 'irunner')])
@@ -255,20 +239,13 b' class IPTester(object):'
         """Create new test runner."""
         p = os.path
         if runner == 'iptest':
-            if INSTALLED:
-                self.runner = tools.cmd2argv(
-                    p.abspath(find_cmd('iptest'))) + sys.argv[1:]
-            else:
-                # Find our own 'iptest' script OS-level entry point.  Don't
-                # look system-wide, so we are sure we pick up *this one*.  And
-                # pass through to subprocess call our own sys.argv
-                ippath = p.abspath(p.join(p.dirname(__file__),'..','..'))
-                script = p.join(ippath, 'iptest.py')
-                self.runner = tools.cmd2argv(script) + sys.argv[1:]
-                
-        else:
+            iptest_app = get_ipython_module_path('IPython.testing.iptest')
+            self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
+        elif runner == 'trial':
             # For trial, it needs to be installed system-wide
-            self.runner = tools.cmd2argv(p.abspath(find_cmd('trial')))
+            self.runner = pycmd2argv(p.abspath(find_cmd('trial')))
+        else:
+            raise Exception('Not a valid test runner: %s' % repr(runner))
         if params is None:
             params = []
         if isinstance(params, str):
@@ -290,6 +267,8 b' class IPTester(object):'
             # fashioned' way to do it, but it works just fine.  If someone
             # later can clean this up that's fine, as long as the tests run
             # reliably in win32.
+            # What types of problems are you having. They may be related to
+            # running Python in unboffered mode. BG.
             return os.system(' '.join(self.call_args))
     else:
         def _run_cmd(self):
@@ -343,9 +322,9 b' def make_runners():'
 
     # And add twisted ones if conditions are met
     if have['zope.interface'] and have['twisted'] and have['foolscap']:
-        # Note that we list the kernel here, though the bulk of it is
-        # twisted-based, because nose picks up doctests that twisted doesn't.
-        nose_pkg_names.append('kernel')
+        # We only list IPython.kernel for testing using twisted.trial as
+        # nose and twisted.trial have conflicts that make the testing system
+        # unstable.
         trial_pkg_names.append('kernel')
 
     # For debugging this code, only load quick stuff

IPython/testing/ipunittest.py

@@ -31,7 +31,6 b' from __future__ import absolute_import'
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
-
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------

IPython/testing/mkdoctests.py

@@ -38,7 +38,7 b' import tempfile'
 
 # IPython-specific libraries
 from IPython.lib import irunner
-from IPython.utils.genutils import fatal
+from IPython.utils.warn import fatal
 
 class IndentOut(object):
     """A simple output stream that indents all output by a fixed amount.

IPython/testing/nosepatch.py

@@ -5,10 +5,25 b' Once this is fixed in upstream nose we can disable it.'
 
 Note: merely importing this module causes the monkeypatch to be applied."""
 
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
 import unittest
 import nose.loader
 from inspect import ismethod, isfunction
 
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
 def getTestCaseNames(self, testCaseClass):
     """Override to select with selector, unless
     config.getTestCaseNamesCompat is True

IPython/testing/parametric.py

@@ -5,10 +5,27 b" parametric code. We just need to double-check that the new code doesn't clash"
 with Twisted (we know it works with nose and unittest).
 """
 
-__all__ = ['parametric','Parametric']
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
 
 from twisted.trial.unittest import TestCase
 
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
+__all__ = ['parametric','Parametric']
+
+
 def partial(f, *partial_args, **partial_kwargs):
     """Generate a partial class method.
 
@@ -20,6 +37,7 b' def partial(f, *partial_args, **partial_kwargs):'
 
     return partial_func
 
+
 def parametric(f):
     """Mark f as a parametric test.
 
@@ -27,6 +45,7 b' def parametric(f):'
     f._parametric = True
     return classmethod(f)
 
+
 def Parametric(cls):
     """Register parametric tests with a class.
 
@@ -56,3 +75,4 b' def Parametric(cls):'
 
         # rename test generator so it isn't called again by nose
         test_gen.im_func.func_name = '__done_' + test_name
+

IPython/testing/tests/test_decorators_trial.py

@@ -1,6 +1,6 b''
 # encoding: utf-8
 """
-Tests for decorators.py compatibility with Twisted.trial
+Tests for decorators_trial.py
 """
 
 #-----------------------------------------------------------------------------
@@ -14,24 +14,19 b' Tests for decorators.py compatibility with Twisted.trial'
 # Imports
 #-----------------------------------------------------------------------------
 
-# Tell nose to skip this module, since this is for twisted only
+# Tell nose to skip this module
 __test__ = {}
 
 import os
 import sys
 
 from twisted.trial import unittest
-import IPython.testing.decorators as dec
+import IPython.testing.decorators_trial as dec
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
-# Note: this code is identical to that in test_decorators, but that one uses
-# stdlib unittest, not the one from twisted, which we are using here. While
-# somewhat redundant, we want to check both with the stdlib and with twisted,
-# so the duplication is OK.
-
 class TestDecoratorsTrial(unittest.TestCase):
     
     @dec.skip()

IPython/testing/tools.py

@@ -15,22 +15,22 b' Authors'
 - Fernando Perez <[e-mail]>
 """
 
-#*****************************************************************************
+from __future__ import absolute_import
+
+#-----------------------------------------------------------------------------
 #       Copyright (C) 2009 The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#*****************************************************************************
+#-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
-# Required modules and packages
+# Imports
 #-----------------------------------------------------------------------------
-from __future__ import absolute_import
 
 import os
 import re
 import sys
-import tempfile
 
 try:
     # These tools are used by parts of the runtime, so we make the nose
@@ -41,7 +41,10 b' try:'
 except ImportError:
     has_nose = False
 
-from IPython.utils import genutils, platutils
+from IPython.config.loader import Config
+from IPython.utils.process import find_cmd, getoutputerror
+from IPython.utils.text import list_strings
+from IPython.utils.io import temp_pyfile
 
 from . import decorators as dec
 
@@ -49,13 +52,6 b' from . import decorators as dec'
 # Globals
 #-----------------------------------------------------------------------------
 
-# By default, we assume IPython has been installed.  But if the test suite is
-# being run from a source tree that has NOT been installed yet, this flag can
-# be set to False by the entry point scripts, to let us know that we must call
-# the source tree versions of the scripts which manipulate sys.path instead of
-# assuming that things exist system-wide.
-INSTALLED = True
-
 # Make a bunch of nose.tools assert wrappers that can be used in test
 # generators.  This will expose an assert* function for each one in nose.tools.
 
@@ -107,7 +103,7 b' def full_path(startPath,files):'
     ['/a.txt']
     """
 
-    files = genutils.list_strings(files)
+    files = list_strings(files)
     base = os.path.split(startPath)[0]
     return [ os.path.join(base,f) for f in files ]
 
@@ -156,63 +152,6 b' def parse_test_output(txt):'
 parse_test_output.__test__ = False
 
 
-def cmd2argv(cmd):
-    r"""Take the path of a command and return a list (argv-style).
-
-    For a given path ``cmd``, this returns [cmd] if cmd's extension is .exe,
-    .com or .bat, and ['python', cmd] otherwise.
-
-    This is mostly a Windows utility, to deal with the fact that the scripts in
-    Windows get wrapped in .exe entry points, so we have to call them
-    differently.
-
-    Parameters
-    ----------
-    cmd : string
-      The path of the command.
-
-    Returns
-    -------
-    argv-style list.
-
-    Examples
-    --------
-    In [2]: cmd2argv('/usr/bin/ipython')
-    Out[2]: ['python', '/usr/bin/ipython']
-
-    In [3]: cmd2argv(r'C:\Python26\Scripts\ipython.exe')
-    Out[3]: ['C:\\Python26\\Scripts\\ipython.exe']
-    """
-    ext = os.path.splitext(cmd)[1]
-    if ext in ['.exe', '.com', '.bat']:
-        return [cmd]
-    else:
-        return ['python', cmd]
-    
-
-def temp_pyfile(src, ext='.py'):
-    """Make a temporary python file, return filename and filehandle.
-
-    Parameters
-    ----------
-    src : string or list of strings (no need for ending newlines if list)
-      Source code to be written to the file.
-
-    ext : optional, string
-      Extension for the generated file.
-
-    Returns
-    -------
-    (filename, open filehandle)
-      It is the caller's responsibility to close the open file and unlink it.
-    """
-    fname = tempfile.mkstemp(ext)[1]
-    f = open(fname,'w')
-    f.write(src)
-    f.flush()
-    return fname, f
-
-
 def default_argv():
     """Return a valid default argv for creating testing instances of ipython"""
 
@@ -222,6 +161,15 b' def default_argv():'
             '--autocall=0']
     
 
+def default_config():
+    """Return a config object with good defaults for testing."""
+    config = Config()
+    config.InteractiveShell.colors = 'NoColor'
+    config.InteractiveShell.term_title = False,
+    config.InteractiveShell.autocall = 0
+    return config
+
+
 def ipexec(fname, options=None):
     """Utility to call 'ipython filename'.
 
@@ -252,20 +200,12 b' def ipexec(fname, options=None):'
     _ip = get_ipython()
     test_dir = os.path.dirname(__file__)
 
-    # Find the ipython script from the package we're using, so that the test
-    # suite can be run from the source tree without an installed IPython
-    p = os.path
-    if INSTALLED:
-        ipython_cmd = platutils.find_cmd('ipython')
-    else:
-        ippath = p.abspath(p.join(p.dirname(__file__),'..','..'))
-        ipython_script = p.join(ippath, 'ipython.py')
-        ipython_cmd = 'python "%s"' % ipython_script
+    ipython_cmd = find_cmd('ipython')
     # Absolute path for filename
-    full_fname = p.join(test_dir, fname)
+    full_fname = os.path.join(test_dir, fname)
     full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname)
     #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg
-    return genutils.getoutputerror(full_cmd)
+    return getoutputerror(full_cmd)
 
 
 def ipexec_validate(fname, expected_out, expected_err='',

IPython/testing/util.py

@@ -1,23 +1,24 b''
 # encoding: utf-8
 """This file contains utility classes for performing tests with Deferreds.
 """
-__docformat__ = "restructuredtext en"
-#-------------------------------------------------------------------------------
-#       Copyright (C) 2005  Fernando Perez <[e-mail]>
-#                           Brian E Granger <[e-mail]>
-#                           Benjamin Ragan-Kelley <[e-mail]>
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
 from twisted.trial import unittest
 from twisted.internet import defer
 
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
 class DeferredTestCase(unittest.TestCase):
     
     def assertDeferredEquals(self, deferred, expectedResult,

IPython/utils/generics.py

@@ -1,8 +1,9 b''
+# encoding: utf-8
 """Generic functions for extending IPython.
 
 See http://cheeseshop.python.org/pypi/simplegeneric.
 
-Here is an example from genutils.py::
+Here is an example from IPython.utils.text::
 
     def print_lsstring(arg):
         "Prettier (non-repr-like) and more informative printer for LSString"
@@ -12,19 +13,37 b' Here is an example from genutils.py::'
     print_lsstring = result_display.when_type(LSString)(print_lsstring)
 """
 
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
 from IPython.core.error import TryNext
 from IPython.external.simplegeneric import generic
 
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+
 @generic
 def result_display(result):
     """Print the result of computation."""
     raise TryNext
 
+
 @generic
 def inspect_object(obj):
     """Called when you do obj?"""
     raise TryNext
 
+
 @generic
 def complete_object(obj, prev_completions):
     """Custom completer dispatching for python objects.
@@ -41,3 +60,5 b' def complete_object(obj, prev_completions):'
     own_attrs + prev_completions.
     """
     raise TryNext
+
+

IPython/utils/growl.py

@@ -1,6 +1,22 b''
-#!/usr/bin/env python
 # encoding: utf-8
+"""
+Utilities using Growl on OS X for notifications.
+"""
 
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
 
 class IPythonGrowlError(Exception):
     pass

IPython/utils/importstring.py

@@ -1,4 +1,3 b''
-#!/usr/bin/env python
 # encoding: utf-8
 """
 A simple utility to import something by its string name.

IPython/utils/ipstruct.py

@@ -19,9 +19,7 b' Authors:'
 # Imports
 #-----------------------------------------------------------------------------
 
-import pprint
-
-from IPython.utils.genutils import list2dict2
+from IPython.utils.data import list2dict2
 
 __all__ = ['Struct']
 

IPython/utils/pickleshare.py

@@ -37,7 +37,6 b' from IPython.external.path import path as Path'
 import os,stat,time
 import cPickle as pickle
 import UserDict
-import warnings
 import glob
 
 def gethashfile(key):

IPython/utils/rlineimpl.py

@@ -1,10 +1,12 b''
 # -*- coding: utf-8 -*-
 """ Imports and provides the 'correct' version of readline for the platform.
 
-Readline is used throughout IPython as 'import IPython.utils.rlineimpl as readline'.
+Readline is used throughout IPython as::
+
+    import IPython.utils.rlineimpl as readline
 
 In addition to normal readline stuff, this module provides have_readline
-boolean and _outputfile variable used in genutils.
+boolean and _outputfile variable used in IPython.utils.
 """
 
 import sys

IPython/utils/syspathcontext.py

@@ -19,9 +19,12 b' Authors:'
 # Imports
 #-----------------------------------------------------------------------------
 
-
 import sys
 
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
 class appended_to_syspath(object):
     """A context for appending a directory to sys.path for a second."""
 

IPython/utils/tests/test_imports.py

@@ -3,21 +3,12 b''
 def test_import_coloransi():
     from IPython.utils import coloransi
 
-def test_import_DPyGetOpt():
-    from IPython.utils import DPyGetOpt
-
 def test_import_generics():
     from IPython.utils import generics
 
-def test_import_genutils():
-    from IPython.utils import genutils
-
 def test_import_ipstruct():
     from IPython.utils import ipstruct
 
-def test_import_platutils():
-    from IPython.utils import platutils
-
 def test_import_PyColorize():
     from IPython.utils import PyColorize
 
@@ -33,5 +24,3 b' def test_import_upgradedir():'
 def test_import_wildcard():
     from IPython.utils import wildcard
 
-def test_import_winconsole():
-    from IPython.utils import winconsole

IPython/utils/tests/test_notification.py

@@ -15,11 +15,7 b''
 
 import unittest
 
-from IPython.utils.notification import (
-    NotificationCenter,
-    NotificationError,
-    shared_center
-)
+from IPython.utils.notification import shared_center
 
 #-----------------------------------------------------------------------------
 # Support Classes

IPython/utils/tests/test_path.py

@@ -1,8 +1,5 b''
 # encoding: utf-8
-
-"""Tests for genutils.py"""
-
-__docformat__ = "restructuredtext en"
+"""Tests for IPython.utils.path.py"""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
@@ -15,27 +12,21 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-----------------------------------------------------------------------------
 
-# stdlib
 import os
 import shutil
 import sys
 import tempfile
-import unittest
 
-from cStringIO import StringIO
 from os.path import join, abspath, split
 
-# third-party
 import nose.tools as nt
 
 from nose import with_setup
-from nose.tools import raises
 
-# Our own
 import IPython
-from IPython.utils import genutils
 from IPython.testing import decorators as dec
-from IPython.testing.decorators import skipif, skip_if_not_win32
+from IPython.testing.decorators import skip_if_not_win32, skip_win32
+from IPython.utils import path
 
 # Platform-dependent imports
 try:
@@ -69,6 +60,7 b' def setup():'
     # problem because that exception is only defined on Windows...
     os.makedirs(IP_TEST_DIR)
 
+
 def teardown():
     """Teardown testenvironment for the module:
     
@@ -88,7 +80,7 b' def setup_environment():'
     each testfunction needs a pristine environment.
     """
     global oldstuff, platformstuff
-    oldstuff = (env.copy(), os.name, genutils.get_home_dir, IPython.__file__)
+    oldstuff = (env.copy(), os.name, path.get_home_dir, IPython.__file__)
 
     if os.name == 'nt':
         platformstuff = (wreg.OpenKey, wreg.QueryValueEx,)
@@ -97,7 +89,7 b' def setup_environment():'
 def teardown_environment():
     """Restore things that were remebered by the setup_environment function
     """
-    (oldenv, os.name, genutils.get_home_dir, IPython.__file__,) = oldstuff
+    (oldenv, os.name, get_home_dir, IPython.__file__,) = oldstuff
         
     for key in env.keys():
         if key not in oldenv:
@@ -112,10 +104,6 b' def teardown_environment():'
 with_environment = with_setup(setup_environment, teardown_environment)
 
 
-#
-# Tests for get_home_dir
-#
-
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_1():
@@ -126,9 +114,10 b' def test_get_home_dir_1():'
     #fake filename for IPython.__init__
     IPython.__file__ = abspath(join(HOME_TEST_DIR, "Lib/IPython/__init__.py"))
     
-    home_dir = genutils.get_home_dir()
+    home_dir = path.get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
     
+
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_2():
@@ -138,61 +127,78 b' def test_get_home_dir_2():'
     #fake filename for IPython.__init__
     IPython.__file__ = abspath(join(HOME_TEST_DIR, "Library.zip/IPython/__init__.py")).lower()
     
-    home_dir = genutils.get_home_dir()
+    home_dir = path.get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR).lower())
 
+
 @with_environment
+@skip_win32
 def test_get_home_dir_3():
     """Testcase $HOME is set, then use its value as home directory."""
     env["HOME"] = HOME_TEST_DIR
-    home_dir = genutils.get_home_dir()
+    home_dir = path.get_home_dir()
     nt.assert_equal(home_dir, env["HOME"])
 
+
 @with_environment
 def test_get_home_dir_4():
-    """Testcase $HOME is not set, os=='poix'. 
+    """Testcase $HOME is not set, os=='posix'. 
     This should fail with HomeDirError"""
     
     os.name = 'posix'
     if 'HOME' in env: del env['HOME']
-    nt.assert_raises(genutils.HomeDirError, genutils.get_home_dir)
+    nt.assert_raises(path.HomeDirError, path.get_home_dir)
+
         
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_5():
-    """Testcase $HOME is not set, os=='nt' 
-    env['HOMEDRIVE'],env['HOMEPATH'] points to path."""
+    """Using HOMEDRIVE + HOMEPATH, os=='nt'.
+
+    HOMESHARE is missing.
+    """
 
     os.name = 'nt'
-    if 'HOME' in env: del env['HOME']
+    env.pop('HOMESHARE', None)
     env['HOMEDRIVE'], env['HOMEPATH'] = os.path.splitdrive(HOME_TEST_DIR)
-
-    home_dir = genutils.get_home_dir()
+    home_dir = path.get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
 
+
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_6():
-    """Testcase $HOME is not set, os=='nt' 
-    env['HOMEDRIVE'],env['HOMEPATH'] do not point to path.
-    env['USERPROFILE'] points to path
+    """Using USERPROFILE, os=='nt'.
+
+    HOMESHARE, HOMEDRIVE, HOMEPATH are missing.
     """
 
     os.name = 'nt'
-    if 'HOME' in env: del env['HOME']
-    env['HOMEDRIVE'], env['HOMEPATH'] = os.path.abspath(TEST_FILE_PATH), "DOES NOT EXIST"
+    env.pop('HOMESHARE', None)
+    env.pop('HOMEDRIVE', None)
+    env.pop('HOMEPATH', None)
     env["USERPROFILE"] = abspath(HOME_TEST_DIR)
+    home_dir = path.get_home_dir()
+    nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
 
-    home_dir = genutils.get_home_dir()
+
+@skip_if_not_win32
+@with_environment
+def test_get_home_dir_7():
+    """Using HOMESHARE, os=='nt'."""
+
+    os.name = 'nt'
+    env["HOMESHARE"] = abspath(HOME_TEST_DIR)
+    home_dir = path.get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
 
 # Should we stub wreg fully so we can run the test on all platforms?
 @skip_if_not_win32
 @with_environment
-def test_get_home_dir_7():
-    """Testcase $HOME is not set, os=='nt'
+def test_get_home_dir_8():
+    """Using registry hack for 'My Documents', os=='nt'
     
-    env['HOMEDRIVE'],env['HOMEPATH'], env['USERPROFILE'] and others missing
+    HOMESHARE, HOMEDRIVE, HOMEPATH, USERPROFILE and others are missing.
     """
     os.name = 'nt'
     # Remove from stub environment all keys that may be set
@@ -211,116 +217,56 b' def test_get_home_dir_7():'
     wreg.OpenKey = OpenKey
     wreg.QueryValueEx = QueryValueEx
 
-    home_dir = genutils.get_home_dir()
+    home_dir = path.get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
 
-#
-# Tests for get_ipython_dir
-#
 
 @with_environment
 def test_get_ipython_dir_1():
     """test_get_ipython_dir_1, Testcase to see if we can call get_ipython_dir without Exceptions."""
     env['IPYTHON_DIR'] = "someplace/.ipython"
-    ipdir = genutils.get_ipython_dir()
+    ipdir = path.get_ipython_dir()
     nt.assert_equal(ipdir, "someplace/.ipython")
 
 
 @with_environment
 def test_get_ipython_dir_2():
     """test_get_ipython_dir_2, Testcase to see if we can call get_ipython_dir without Exceptions."""
-    genutils.get_home_dir = lambda : "someplace"
+    path.get_home_dir = lambda : "someplace"
     os.name = "posix"
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
-    ipdir = genutils.get_ipython_dir()
+    ipdir = path.get_ipython_dir()
     nt.assert_equal(ipdir, os.path.join("someplace", ".ipython"))
 
-#
-# Tests for popkey
-#
-
-def test_popkey_1():
-    """test_popkey_1, Basic usage test of popkey
-    """
-    dct = dict(a=1, b=2, c=3)
-    nt.assert_equal(genutils.popkey(dct, "a"), 1)
-    nt.assert_equal(dct, dict(b=2, c=3))
-    nt.assert_equal(genutils.popkey(dct, "b"), 2)
-    nt.assert_equal(dct, dict(c=3))
-    nt.assert_equal(genutils.popkey(dct, "c"), 3)
-    nt.assert_equal(dct, dict())
-
-def test_popkey_2():
-    """test_popkey_2, Test to see that popkey of non occuring keys
-    generates a KeyError exception
-    """
-    dct = dict(a=1, b=2, c=3)
-    nt.assert_raises(KeyError, genutils.popkey, dct, "d")
-
-def test_popkey_3():
-    """test_popkey_3, Tests to see that popkey calls returns the correct value
-    and that the key/value was removed from the dict.
-    """
-    dct = dict(a=1, b=2, c=3)
-    nt.assert_equal(genutils.popkey(dct, "A", 13), 13)
-    nt.assert_equal(dct, dict(a=1, b=2, c=3))
-    nt.assert_equal(genutils.popkey(dct, "B", 14), 14)
-    nt.assert_equal(dct, dict(a=1, b=2, c=3))
-    nt.assert_equal(genutils.popkey(dct, "C", 15), 15)
-    nt.assert_equal(dct, dict(a=1, b=2, c=3))
-    nt.assert_equal(genutils.popkey(dct, "a"), 1)
-    nt.assert_equal(dct, dict(b=2, c=3))
-    nt.assert_equal(genutils.popkey(dct, "b"), 2)
-    nt.assert_equal(dct, dict(c=3))
-    nt.assert_equal(genutils.popkey(dct, "c"), 3)
-    nt.assert_equal(dct, dict())
-
 
 def test_filefind():
     """Various tests for filefind"""
     f = tempfile.NamedTemporaryFile()
-    print 'fname:',f.name
-    alt_dirs = genutils.get_ipython_dir()
-    t = genutils.filefind(f.name,alt_dirs)
-    print 'found:',t
+    # print 'fname:',f.name
+    alt_dirs = path.get_ipython_dir()
+    t = path.filefind(f.name, alt_dirs)
+    # print 'found:',t
 
 
 def test_get_ipython_package_dir():
-    ipdir = genutils.get_ipython_package_dir()
+    ipdir = path.get_ipython_package_dir()
     nt.assert_true(os.path.isdir(ipdir))
 
 
-def test_tee_simple():
-    "Very simple check with stdout only"
-    chan = StringIO()
-    text = 'Hello'
-    tee = genutils.Tee(chan, channel='stdout')
-    print >> chan, text,
-    nt.assert_equal(chan.getvalue(), text)
-
+def test_get_ipython_module_path():
+    ipapp_path = path.get_ipython_module_path('IPython.core.ipapp')
+    nt.assert_true(os.path.isfile(ipapp_path))
 
-class TeeTestCase(dec.ParametricTestCase):
 
-    def tchan(self, channel, check='close'):
-        trap = StringIO()
-        chan = StringIO()
-        text = 'Hello'
[e-mail]_if_not_win32
+def test_get_long_path_name_win32():
+    p = path.get_long_path_name('c:\\docume~1')
+    nt.assert_equals(p,u'c:\\Documents and Settings') 
         
-        std_ori = getattr(sys, channel)
-        setattr(sys, channel, trap)
 
-        tee = genutils.Tee(chan, channel=channel)
-        print >> chan, text,
-        setattr(sys, channel, std_ori)
-        trap_val = trap.getvalue()
-        nt.assert_equals(chan.getvalue(), text)
-        if check=='close':
-            tee.close()
-        else:
-            del tee
[e-mail]_win32
+def test_get_long_path_name():
+    p = path.get_long_path_name('/usr/local')
+    nt.assert_equals(p,'/usr/local')
 
-    def test(self):
-        for chan in ['stdout', 'stderr']:
-            for check in ['close', 'del']:
-                yield self.tchan(chan, check)

IPython/utils/tests/test_process.py

@@ -14,12 +14,11 b' Tests for platutils.py'
 # Imports
 #-----------------------------------------------------------------------------
 
-import os
 import sys
 
 import nose.tools as nt
 
-from IPython.utils.platutils import find_cmd, FindCmdError, get_long_path_name
+from IPython.utils.process import find_cmd, FindCmdError
 from IPython.testing import decorators as dec
 
 #-----------------------------------------------------------------------------
@@ -60,13 +59,4 b' def test_find_cmd_fail():'
     nt.assert_raises(FindCmdError,find_cmd,'asdfasdf')
 
     
[e-mail]_if_not_win32
-def test_get_long_path_name_win32():
-    p = get_long_path_name('c:\\docume~1')
-    nt.assert_equals(p,u'c:\\Documents and Settings') 
 
-    
[e-mail]_win32
-def test_get_long_path_name():
-    p = get_long_path_name('/usr/local')
-    nt.assert_equals(p,'/usr/local')

IPython/utils/tests/test_traitlets.py

@@ -22,15 +22,11 b' Authors:'
 # Imports
 #-----------------------------------------------------------------------------
 
-import sys
-import os
-
-
 from unittest import TestCase
 
 from IPython.utils.traitlets import (
     HasTraitlets, MetaHasTraitlets, TraitletType, Any,
-    Int, Long, Float, Complex, Str, Unicode, Bool, TraitletError,
+    Int, Long, Float, Complex, Str, Unicode, TraitletError,
     Undefined, Type, This, Instance
 )
 

IPython/utils/wildcard.py

@@ -14,13 +14,10 b' Authors'
 #*****************************************************************************
 
 import __builtin__
-import exceptions
-import pdb
-import pprint
 import re
 import types
 
-from IPython.utils.genutils import dir2
+from IPython.utils.dir2 import dir2
 
 def create_typestr2type_dicts(dont_include_in_type2type2str=["lambda"]):
     """Return dictionaries mapping lower case typename to type objects, from

MANIFEST.in

@@ -1,5 +1,4 b''
 include ipython.py
-include iptest.py
 include setupbase.py
 include setupegg.py
 
@@ -9,7 +8,7 b' graft scripts'
 graft IPython/kernel
 graft IPython/config
 graft IPython/core
-graft IPython/deathrow
+# graft IPython/deathrow
 graft IPython/external
 graft IPython/frontend
 graft IPython/gui

docs/examples/kernel/mcdriver.py

@@ -1,71 +1,148 b''
 #!/usr/bin/env python
 """Run a Monte-Carlo options pricer in parallel."""
 
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+import time
 from IPython.kernel import client
 import numpy as np
 from mcpricer import price_options
+from matplotlib import pyplot as plt
+
+#-----------------------------------------------------------------------------
+# Setup parameters for the run
+#-----------------------------------------------------------------------------
+
+def ask_question(text, the_type, default):
+    s = '%s [%r]: ' % (text, the_type(default))
+    result = raw_input(s)
+    if result:
+        return the_type(result)
+    else:
+        return the_type(default)
+
+cluster_profile = ask_question("Cluster profile", str, "default")
+price = ask_question("Initial price", float, 100.0)
+rate = ask_question("Interest rate", float, 0.05)
+days = ask_question("Days to expiration", int, 260)
+paths = ask_question("Number of MC paths", int, 10000)
+n_strikes = ask_question("Number of strike values", int, 5)
+min_strike = ask_question("Min strike price", float, 90.0)
+max_strike = ask_question("Max strike price", float, 110.0)
+n_sigmas = ask_question("Number of volatility values", int, 5)
+min_sigma = ask_question("Min volatility", float, 0.1)
+max_sigma = ask_question("Max volatility", float, 0.4)
+
+strike_vals = np.linspace(min_strike, max_strike, n_strikes)
+sigma_vals = np.linspace(min_sigma, max_sigma, n_sigmas)
+
+#-----------------------------------------------------------------------------
+# Setup for parallel calculation
+#-----------------------------------------------------------------------------
 
 # The MultiEngineClient is used to setup the calculation and works with all
 # engine.
-mec = client.MultiEngineClient(profile='mycluster')
+mec = client.MultiEngineClient(profile=cluster_profile)
 
 # The TaskClient is an interface to the engines that provides dynamic load 
 # balancing at the expense of not knowing which engine will execute the code.
-tc = client.TaskClient(profile='mycluster')
+tc = client.TaskClient(profile=cluster_profile)
 
 # Initialize the common code on the engines. This Python module has the
 # price_options function that prices the options.
 mec.run('mcpricer.py')
 
-# Define the function that will make up our tasks. We basically want to
-# call the price_options function with all but two arguments (K, sigma)
-# fixed.
-def my_prices(K, sigma):
-    S = 100.0
-    r = 0.05
-    days = 260
-    paths = 100000
-    return price_options(S, K, sigma, r, days, paths)
-
-# Create arrays of strike prices and volatilities
-nK = 10
-nsigma = 10
-K_vals = np.linspace(90.0, 100.0, nK)
-sigma_vals = np.linspace(0.1, 0.4, nsigma)
-
-# Submit tasks to the TaskClient for each (K, sigma) pair as a MapTask.
-# The MapTask simply applies a function (my_prices) to the arguments:
-# my_prices(K, sigma) and returns the result.
+#-----------------------------------------------------------------------------
+# Perform parallel calculation
+#-----------------------------------------------------------------------------
+
+print "Running parallel calculation over strike prices and volatilities..."
+print "Strike prices: ", strike_vals
+print "Volatilities: ", sigma_vals
+sys.stdout.flush()
+
+# Submit tasks to the TaskClient for each (strike, sigma) pair as a MapTask.
+t1 = time.time()
 taskids = []
-for K in K_vals:
+for strike in strike_vals:
     for sigma in sigma_vals:
-        t = client.MapTask(my_prices, args=(K, sigma))
+        t = client.MapTask(
+            price_options, 
+            args=(price, strike, sigma, rate, days, paths)
+        )
         taskids.append(tc.run(t))
 
 print "Submitted tasks: ", len(taskids)
+sys.stdout.flush()
 
 # Block until all tasks are completed.
 tc.barrier(taskids)
+t2 = time.time()
+t = t2-t1
+
+print "Parallel calculation completed, time = %s s" % t
+print "Collecting results..."
 
 # Get the results using TaskClient.get_task_result.
 results = [tc.get_task_result(tid) for tid in taskids]
 
 # Assemble the result into a structured NumPy array.
-prices = np.empty(nK*nsigma,
+prices = np.empty(n_strikes*n_sigmas,
     dtype=[('ecall',float),('eput',float),('acall',float),('aput',float)]
 )
+
 for i, price_tuple in enumerate(results):
     prices[i] = price_tuple
-prices.shape = (nK, nsigma)
-K_vals, sigma_vals = np.meshgrid(K_vals, sigma_vals)
 
-def plot_options(sigma_vals, K_vals, prices):
+prices.shape = (n_strikes, n_sigmas)
+strike_mesh, sigma_mesh = np.meshgrid(strike_vals, sigma_vals)
+
+print "Results are available: strike_mesh, sigma_mesh, prices"
+print "To plot results type 'plot_options(sigma_mesh, strike_mesh, prices)'"
+
+#-----------------------------------------------------------------------------
+# Utilities
+#-----------------------------------------------------------------------------
+
+def plot_options(sigma_mesh, strike_mesh, prices):
     """
-    Make a contour plot of the option price in (sigma, K) space.
+    Make a contour plot of the option price in (sigma, strike) space.
     """
-    from matplotlib import pyplot as plt
-    plt.contourf(sigma_vals, K_vals, prices)
+    plt.figure(1)
+    
+    plt.subplot(221)
+    plt.contourf(sigma_mesh, strike_mesh, prices['ecall'])
+    plt.axis('tight')
+    plt.colorbar()
+    plt.title('European Call')
+    plt.ylabel("Strike Price")
+
+    plt.subplot(222)
+    plt.contourf(sigma_mesh, strike_mesh, prices['acall'])
+    plt.axis('tight')
     plt.colorbar()
-    plt.title("Option Price")
+    plt.title("Asian Call")
+
+    plt.subplot(223)
+    plt.contourf(sigma_mesh, strike_mesh, prices['eput'])
+    plt.axis('tight')
+    plt.colorbar()
+    plt.title("European Put")
     plt.xlabel("Volatility")
     plt.ylabel("Strike Price")
+
+    plt.subplot(224)
+    plt.contourf(sigma_mesh, strike_mesh, prices['aput'])
+    plt.axis('tight')
+    plt.colorbar()
+    plt.title("Asian Put")
+    plt.xlabel("Volatility")
+
+
+
+    
+
+

docs/examples/kernel/task_profiler.py

@@ -18,7 +18,7 b' overhead of a single task is about 0.001-0.01 seconds.'
 import random, sys
 from optparse import OptionParser
 
-from IPython.genutils import time
+from IPython.utils.timing import time
 from IPython.kernel import client
 
 def main():
@@ -51,7 +51,7 b' def main():'
     print tc.task_controller
     rc.block=True
     nengines = len(rc.get_ids())
-    rc.execute('from IPython.genutils import time')
+    rc.execute('from IPython.utils.timing import time')
 
     # the jobs should take a random time within a range
     times = [random.random()*(opts.tmax-opts.tmin)+opts.tmin for i in range(opts.n)]

docs/source/config/overview.txt

@@ -245,7 +245,7 b' this directory is determined by the following algorithm:'
 
 * If the ``--ipython-dir`` command line flag is given, its value is used.
 
-* If not, the value returned by :func:`IPython.utils.genutils.get_ipython_dir`
+* If not, the value returned by :func:`IPython.utils.path.get_ipython_dir`
   is used. This function will first look at the :envvar:`IPYTHON_DIR`
   environment variable and then default to the directory
   :file:`$HOME/.ipython`.

docs/source/development/doc_guide.txt

@@ -41,8 +41,8 b' A bit of Python code::'
 
 An interactive Python session::
 
-    >>> from IPython import genutils
-    >>> genutils.get_ipython_dir()
+    >>> from IPython.utils.path import get_ipython_dir
+    >>> get_ipython_dir()
     '/home/fperez/.ipython'
 
 An IPython session:

docs/source/development/ipgraph.txt

@@ -57,3 +57,5 b' methods in :class:`InteractiveShell` that manage code execution::'
     nx.draw_spectral(g, node_size=100, alpha=0.6, node_color='r',
                      font_size=10, node_shape='o')
     plt.show()
+
+

docs/source/development/testing.txt

@@ -244,7 +244,7 b' and an example of ``# all-random``::'
 
 When writing docstrings, you can use the ``@skip_doctest`` decorator to
 indicate that a docstring should *not* be treated as a doctest at all.  The
-difference betwee ``# all-random`` and ``@skip_doctest`` is that the former
+difference between ``# all-random`` and ``@skip_doctest`` is that the former
 executes the example but ignores output, while the latter doesn't execute any
 code.  ``@skip_doctest`` should be used for docstrings whose examples are
 purely informational.

docs/source/interactive/shell.txt

@@ -166,7 +166,7 b' ipy_user_conf.py.'
 String lists
 ============
 
-String lists (IPython.genutils.SList) are handy way to process output
+String lists (IPython.utils.text.SList) are handy way to process output
 from system commands. They are produced by ``var = !cmd`` syntax.
 
 First, we acquire the output of 'ls -l'::

docs/sphinxext/docscrape.py

@@ -8,7 +8,7 b' import re'
 import pydoc
 from StringIO import StringIO
 from warnings import warn
-4
+
 class Reader(object):
     """A line-based string reader.
 

docs/sphinxext/ipython_directive.py

@@ -59,12 +59,9 b' Authors'
 
 # Stdlib
 import cStringIO
-import imp
 import os
 import re
-import shutil
 import sys
-import warnings
 
 # To keep compatibility with various python versions
 try:
@@ -80,8 +77,8 b' from docutils.parsers.rst import directives'
 matplotlib.use('Agg')
 
 # Our own
-from IPython import Config, IPythonApp
-from IPython.utils.genutils import Term, Tee
+from IPython import Config, InteractiveShell
+from IPython.utils.io import Term
 
 #-----------------------------------------------------------------------------
 # Globals
@@ -208,8 +205,9 b' class EmbeddedSphinxShell(object):'
         Term.cerr = self.cout
 
         # For debugging, so we can see normal output, use this:
-        #Term.cout = genutils.Tee(self.cout, channel='stdout') # dbg
-        #Term.cerr = genutils.Tee(self.cout, channel='stderr') # dbg
+        # from IPython.utils.io import Tee
+        #Term.cout = Tee(self.cout, channel='stdout') # dbg
+        #Term.cerr = Tee(self.cout, channel='stderr') # dbg
 
         # Create config object for IPython
         config = Config()
@@ -221,15 +219,11 b' class EmbeddedSphinxShell(object):'
         config.InteractiveShell.autoindent = False
         config.InteractiveShell.colors = 'NoColor'
 
-        # Merge global config which can be used to override.
-        config._merge(CONFIG)
-
         # Create and initialize ipython, but don't start its mainloop
-        IP = IPythonApp(override_config=config)
-        IP.initialize()
+        IP = InteractiveShell(parent=None, config=config)
 
         # Store a few parts of IPython we'll need.
-        self.IP = IP.shell
+        self.IP = IP
         self.user_ns = self.IP.user_ns
         self.user_global_ns = self.IP.user_global_ns
                                     

setup.py

@@ -47,7 +47,7 b" if os.path.exists('MANIFEST'): os.remove('MANIFEST')"
 from distutils.core import setup
 
 # Our own imports
-from IPython.utils.genutils import target_update
+from IPython.utils.path import target_update
 
 from setupbase import (
     setup_args,