Work to address the review comments on Fernando's branch.

* Added comment about Magic(object) (r1224)
* Moved InteractiveTB.set_mode from IPythonApp -> InteractiveShell (r1229)
* Moved pylabtools.py to IPython/lib (r1229)
* Cleaned up comments and copyrights in testing (r1233)
* Added comment about ip.shell._ofind (r1237)
* Removed "Bye." from quitter (r1240).
* Refactored and removed :mod:`IPython.utils.genutils` and
  :mod:`IPython.utils.platutils`. These modules have been replaced by topical
  focused modules in :mod:`IPython.utils`.
* Refactored tests in :mod:`IPython.utils.tests`.
* Moved :func:`IPython.testing.tools.temp_pyfile` to :mod:`IPython.utils.io`.
* Moved :func:`IPython.testing.tools.cmd2argv` to
  :func:`IPython.testing.tools.pycmd2argv` and documented the fact that this
  only works with Python based command line programs.
* Created a new :func:`IPython.utils.path.get_ipython_module_path` to use
  in finding paths to IPython modules.

IPython/utils/attic.py

@@ -0,0 +1,167 b''
+# encoding: utf-8
+"""
+Older utilities that are not being used.
+
+WARNING: IF YOU NEED TO USE ONE OF THESE FUNCTIONS, PLEASE FIRST MOVE IT
+TO ANOTHER APPROPRIATE MODULE IN IPython.utils.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+import warnings
+
+from IPython.utils.warn import warn
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+def mutex_opts(dict,ex_op):
+    """Check for presence of mutually exclusive keys in a dict.
+
+    Call: mutex_opts(dict,[[op1a,op1b],[op2a,op2b]...]"""
+    for op1,op2 in ex_op:
+        if op1 in dict and op2 in dict:
+            raise ValueError,'\n*** ERROR in Arguments *** '\
+                  'Options '+op1+' and '+op2+' are mutually exclusive.'
+
+
+class EvalDict:
+    """
+    Emulate a dict which evaluates its contents in the caller's frame.
+
+    Usage:
+    >>> number = 19
+
+    >>> text = "python"
+
+    >>> print "%(text.capitalize())s %(number/9.0).1f rules!" % EvalDict()
+    Python 2.1 rules!
+    """
+
+    # This version is due to [e-mail] on c.l.py, and is basically a
+    # modified (shorter) version of:
+    # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66018 by
+    # Skip Montanaro ([e-mail]).
+
+    def __getitem__(self, name):
+        frame = sys._getframe(1)
+        return eval(name, frame.f_globals, frame.f_locals)
+
+EvalString = EvalDict  # for backwards compatibility
+
+
+def all_belong(candidates,checklist):
+    """Check whether a list of items ALL appear in a given list of options.
+
+    Returns a single 1 or 0 value."""
+
+    return 1-(0 in [x in checklist for x in candidates])
+
+
+def belong(candidates,checklist):
+    """Check whether a list of items appear in a given list of options.
+
+    Returns a list of 1 and 0, one for each candidate given."""
+
+    return [x in checklist for x in candidates]
+
+
+def with_obj(object, **args):
+    """Set multiple attributes for an object, similar to Pascal's with.
+
+    Example:
+    with_obj(jim,
+             born = 1960,
+             haircolour = 'Brown',
+             eyecolour = 'Green')
+
+    Credit: Greg Ewing, in
+    http://mail.python.org/pipermail/python-list/2001-May/040703.html.
+
+    NOTE: up until IPython 0.7.2, this was called simply 'with', but 'with'
+    has become a keyword for Python 2.5, so we had to rename it."""
+
+    object.__dict__.update(args)
+
+
+def map_method(method,object_list,*argseq,**kw):
+    """map_method(method,object_list,*args,**kw) -> list
+
+    Return a list of the results of applying the methods to the items of the
+    argument sequence(s).  If more than one sequence is given, the method is
+    called with an argument list consisting of the corresponding item of each
+    sequence. All sequences must be of the same length.
+
+    Keyword arguments are passed verbatim to all objects called.
+
+    This is Python code, so it's not nearly as fast as the builtin map()."""
+
+    out_list = []
+    idx = 0
+    for object in object_list:
+        try:
+            handler = getattr(object, method)
+        except AttributeError:
+            out_list.append(None)
+        else:
+            if argseq:
+                args = map(lambda lst:lst[idx],argseq)
+                #print 'ob',object,'hand',handler,'ar',args # dbg
+                out_list.append(handler(args,**kw))
+            else:
+                out_list.append(handler(**kw))
+        idx += 1
+    return out_list
+
+
+def import_fail_info(mod_name,fns=None):
+    """Inform load failure for a module."""
+
+    if fns == None:
+        warn("Loading of %s failed.\n" % (mod_name,))
+    else:
+        warn("Loading of %s from %s failed.\n" % (fns,mod_name))
+
+
+class NotGiven: pass
+
+def popkey(dct,key,default=NotGiven):
+    """Return dct[key] and delete dct[key].
+
+    If default is given, return it if dct[key] doesn't exist, otherwise raise
+    KeyError.  """
+
+    try:
+        val = dct[key]
+    except KeyError:
+        if default is NotGiven:
+            raise
+        else:
+            return default
+    else:
+        del dct[key]
+        return val
+
+
+def wrap_deprecated(func, suggest = '<nothing>'):
+    def newFunc(*args, **kwargs):
+        warnings.warn("Call to deprecated function %s, use %s instead" %
+                      ( func.__name__, suggest),
+                      category=DeprecationWarning,
+                      stacklevel = 2)
+        return func(*args, **kwargs)
+    return newFunc
+
+

IPython/utils/cursesimport.py

@@ -0,0 +1,31 b''
+# encoding: utf-8
+"""
+See if we have curses.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+# Curses and termios are Unix-only modules
+try:
+    import curses
+    # We need termios as well, so if its import happens to raise, we bail on
+    # using curses altogether.
+    import termios
+except ImportError:
+    use_curses = False
+else:
+    # Curses on Solaris may not be complete, so we can't use it there
+    use_curses = hasattr(curses,'initscr')
  No newline at end of file

IPython/utils/data.py

@@ -0,0 +1,106 b''
+# encoding: utf-8
+"""Utilities for working with data structures like lists, dicts and tuples.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import types
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def uniq_stable(elems):
+    """uniq_stable(elems) -> list
+
+    Return from an iterable, a list of all the unique elements in the input,
+    but maintaining the order in which they first appear.
+
+    A naive solution to this problem which just makes a dictionary with the
+    elements as keys fails to respect the stability condition, since
+    dictionaries are unsorted by nature.
+
+    Note: All elements in the input must be valid dictionary keys for this
+    routine to work, as it internally uses a dictionary for efficiency
+    reasons."""
+
+    unique = []
+    unique_dict = {}
+    for nn in elems:
+        if nn not in unique_dict:
+            unique.append(nn)
+            unique_dict[nn] = None
+    return unique
+
+
+def sort_compare(lst1, lst2, inplace=1):
+    """Sort and compare two lists.
+
+    By default it does it in place, thus modifying the lists. Use inplace = 0
+    to avoid that (at the cost of temporary copy creation)."""
+    if not inplace:
+        lst1 = lst1[:]
+        lst2 = lst2[:]
+    lst1.sort(); lst2.sort()
+    return lst1 == lst2
+
+
+def list2dict(lst):
+    """Takes a list of (key,value) pairs and turns it into a dict."""
+
+    dic = {}
+    for k,v in lst: dic[k] = v
+    return dic
+
+
+def list2dict2(lst, default=''):
+    """Takes a list and turns it into a dict.
+    Much slower than list2dict, but more versatile. This version can take
+    lists with sublists of arbitrary length (including sclars)."""
+
+    dic = {}
+    for elem in lst:
+        if type(elem) in (types.ListType,types.TupleType):
+            size = len(elem)
+            if  size == 0:
+                pass
+            elif size == 1:
+                dic[elem] = default
+            else:
+                k,v = elem[0], elem[1:]
+                if len(v) == 1: v = v[0]
+                dic[k] = v
+        else:
+            dic[elem] = default
+    return dic
+
+
+def flatten(seq):
+    """Flatten a list of lists (NOT recursive, only works for 2d lists)."""
+
+    return [x for subseq in seq for x in subseq]
+
+
+def get_slice(seq, start=0, stop=None, step=1):
+    """Get a slice of a sequence with variable step. Specify start,stop,step."""
+    if stop == None:
+        stop = len(seq)
+    item = lambda i: seq[i]
+    return map(item,xrange(start,stop,step))
+
+
+def chop(seq, size):
+    """Chop a sequence into chunks of the given size."""
+    chunk = lambda i: seq[i:i+size]
+    return map(chunk,xrange(0,len(seq),size))
+
+

IPython/utils/decorators.py

@@ -0,0 +1,46 b''
+# encoding: utf-8
+"""Decorators that don't go anywhere else.
+
+This module contains misc. decorators that don't really go with another module
+in :mod:`IPython.utils`. Beore putting something here please see if it should
+go into another topical module in :mod:`IPython.utils`.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def flag_calls(func):
+    """Wrap a function to detect and flag when it gets called.
+
+    This is a decorator which takes a function and wraps it in a function with
+    a 'called' attribute. wrapper.called is initialized to False.
+
+    The wrapper.called attribute is set to False right before each call to the
+    wrapped function, so if the call fails it remains False.  After the call
+    completes, wrapper.called is set to True and the output is returned.
+
+    Testing for truth in wrapper.called allows you to determine if a call to
+    func() was attempted and succeeded."""
+
+    def wrapper(*args,**kw):
+        wrapper.called = False
+        out = func(*args,**kw)
+        wrapper.called = True
+        return out
+
+    wrapper.called = False
+    wrapper.__doc__ = func.__doc__
+    return wrapper
+

IPython/utils/dir2.py

@@ -0,0 +1,82 b''
+# encoding: utf-8
+"""A fancy version of Python's builtin :func:`dir` function.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def get_class_members(cls):
+    ret = dir(cls)
+    if hasattr(cls,'__bases__'):
+        for base in cls.__bases__:
+            ret.extend(get_class_members(base))
+    return ret
+
+
+def dir2(obj):
+    """dir2(obj) -> list of strings
+
+    Extended version of the Python builtin dir(), which does a few extra
+    checks, and supports common objects with unusual internals that confuse
+    dir(), such as Traits and PyCrust.
+
+    This version is guaranteed to return only a list of true strings, whereas
+    dir() returns anything that objects inject into themselves, even if they
+    are later not really valid for attribute access (many extension libraries
+    have such bugs).
+    """
+
+    # Start building the attribute list via dir(), and then complete it
+    # with a few extra special-purpose calls.
+    words = dir(obj)
+
+    if hasattr(obj,'__class__'):
+        words.append('__class__')
+        words.extend(get_class_members(obj.__class__))
+    #if '__base__' in words: 1/0
+
+    # Some libraries (such as traits) may introduce duplicates, we want to
+    # track and clean this up if it happens
+    may_have_dupes = False
+
+    # this is the 'dir' function for objects with Enthought's traits
+    if hasattr(obj, 'trait_names'):
+        try:
+            words.extend(obj.trait_names())
+            may_have_dupes = True
+        except TypeError:
+            # This will happen if `obj` is a class and not an instance.
+            pass
+
+    # Support for PyCrust-style _getAttributeNames magic method.
+    if hasattr(obj, '_getAttributeNames'):
+        try:
+            words.extend(obj._getAttributeNames())
+            may_have_dupes = True
+        except TypeError:
+            # `obj` is a class and not an instance.  Ignore
+            # this error.
+            pass
+
+    if may_have_dupes:
+        # eliminate possible duplicates, as some traits may also
+        # appear as normal attributes in the dir() call.
+        words = list(set(words))
+        words.sort()
+
+    # filter out non-string attributes which may be stuffed by dir() calls
+    # and poor coding in third-party modules
+    return [w for w in words if isinstance(w, basestring)]
+

IPython/utils/doctestreload.py

@@ -0,0 +1,75 b''
+# encoding: utf-8
+"""
+A utility for handling the reloading of doctest.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def dhook_wrap(func,*a,**k):
+    """Wrap a function call in a sys.displayhook controller.
+
+    Returns a wrapper around func which calls func, with all its arguments and
+    keywords unmodified, using the default sys.displayhook.  Since IPython
+    modifies sys.displayhook, it breaks the behavior of certain systems that
+    rely on the default behavior, notably doctest.
+    """
+
+    def f(*a,**k):
+
+        dhook_s = sys.displayhook
+        sys.displayhook = sys.__displayhook__
+        try:
+            out = func(*a,**k)
+        finally:
+            sys.displayhook = dhook_s
+
+        return out
+
+    f.__doc__ = func.__doc__
+    return f
+
+
+def doctest_reload():
+    """Properly reload doctest to reuse it interactively.
+
+    This routine:
+
+      - imports doctest but does NOT reload it (see below).
+
+      - resets its global 'master' attribute to None, so that multiple uses of
+      the module interactively don't produce cumulative reports.
+
+      - Monkeypatches its core test runner method to protect it from IPython's
+      modified displayhook.  Doctest expects the default displayhook behavior
+      deep down, so our modification breaks it completely.  For this reason, a
+      hard monkeypatch seems like a reasonable solution rather than asking
+      users to manually use a different doctest runner when under IPython.
+
+    Notes
+    -----
+
+    This function *used to* reload doctest, but this has been disabled because
+    reloading doctest unconditionally can cause massive breakage of other
+    doctest-dependent modules already in memory, such as those for IPython's
+    own testing system.  The name wasn't changed to avoid breaking people's
+    code, but the reload call isn't actually made anymore."""
+
+    import doctest
+    doctest.master = None
+    doctest.DocTestRunner.run = dhook_wrap(doctest.DocTestRunner.run)
+

IPython/utils/frame.py

@@ -0,0 +1,85 b''
+# encoding: utf-8
+"""
+Utilities for working with stack frames.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def extract_vars(*names,**kw):
+    """Extract a set of variables by name from another frame.
+
+    :Parameters:
+      - `*names`: strings
+        One or more variable names which will be extracted from the caller's
+    frame.
+
+    :Keywords:
+      - `depth`: integer (0)
+        How many frames in the stack to walk when looking for your variables.
+
+
+    Examples:
+
+        In [2]: def func(x):
+           ...:     y = 1
+           ...:     print extract_vars('x','y')
+           ...:
+
+        In [3]: func('hello')
+        {'y': 1, 'x': 'hello'}
+    """
+
+    depth = kw.get('depth',0)
+    
+    callerNS = sys._getframe(depth+1).f_locals
+    return dict((k,callerNS[k]) for k in names)
+
+
+def extract_vars_above(*names):
+    """Extract a set of variables by name from another frame.
+
+    Similar to extractVars(), but with a specified depth of 1, so that names
+    are exctracted exactly from above the caller.
+
+    This is simply a convenience function so that the very common case (for us)
+    of skipping exactly 1 frame doesn't have to construct a special dict for
+    keyword passing."""
+
+    callerNS = sys._getframe(2).f_locals
+    return dict((k,callerNS[k]) for k in names)
+
+
+def debugx(expr,pre_msg=''):
+    """Print the value of an expression from the caller's frame.
+
+    Takes an expression, evaluates it in the caller's frame and prints both
+    the given expression and the resulting value (as well as a debug mark
+    indicating the name of the calling function.  The input must be of a form
+    suitable for eval().
+
+    An optional message can be passed, which will be prepended to the printed
+    expr->value pair."""
+
+    cf = sys._getframe(1)
+    print '[DBG:%s] %s%s -> %r' % (cf.f_code.co_name,pre_msg,expr,
+                                   eval(expr,cf.f_globals,cf.f_locals))
+
+
+# deactivate it by uncommenting the following line, which makes it a no-op
+#def debugx(expr,pre_msg=''): pass
+

IPython/utils/io.py

@@ -0,0 +1,292 b''
+# encoding: utf-8
+"""
+IO related utilities.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+import tempfile
+
+from IPython.external.Itpl import itpl, printpl
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+class IOStream:
+
+    def __init__(self,stream,fallback):
+        if not hasattr(stream,'write') or not hasattr(stream,'flush'):
+            stream = fallback
+        self.stream = stream
+        self._swrite = stream.write
+        self.flush = stream.flush
+
+    def write(self,data):
+        try:
+            self._swrite(data)
+        except:
+            try:
+                # print handles some unicode issues which may trip a plain
+                # write() call.  Attempt to emulate write() by using a
+                # trailing comma
+                print >> self.stream, data,
+            except:
+                # if we get here, something is seriously broken.
+                print >> sys.stderr, \
+                      'ERROR - failed to write data to stream:', self.stream
+
+    def writeln(self, data):
+        self.write(data)
+        self.write('\n')        
+                
+    def close(self):
+        pass
+
+
+class IOTerm:
+    """ Term holds the file or file-like objects for handling I/O operations.
+
+    These are normally just sys.stdin, sys.stdout and sys.stderr but for
+    Windows they can can replaced to allow editing the strings before they are
+    displayed."""
+
+    # In the future, having IPython channel all its I/O operations through
+    # this class will make it easier to embed it into other environments which
+    # are not a normal terminal (such as a GUI-based shell)
+    def __init__(self,cin=None,cout=None,cerr=None):
+        self.cin  = IOStream(cin,sys.stdin)
+        self.cout = IOStream(cout,sys.stdout)
+        self.cerr = IOStream(cerr,sys.stderr)
+
+
+# Global variable to be used for all I/O
+Term = IOTerm()
+
+
+import IPython.utils.rlineimpl as readline
+# Remake Term to use the readline i/o facilities
+if sys.platform == 'win32' and readline.have_readline:
+
+    Term = IOTerm(cout=readline._outputfile,cerr=readline._outputfile)
+
+
+class Tee(object):
+    """A class to duplicate an output stream to stdout/err.
+
+    This works in a manner very similar to the Unix 'tee' command.
+
+    When the object is closed or deleted, it closes the original file given to
+    it for duplication.
+    """
+    # Inspired by:
+    # http://mail.python.org/pipermail/python-list/2007-May/442737.html
+
+    def __init__(self, file_or_name, mode=None, channel='stdout'):
+        """Construct a new Tee object.
+
+        Parameters
+        ----------
+        file_or_name : filename or open filehandle (writable)
+          File that will be duplicated
+
+        mode : optional, valid mode for open().
+          If a filename was give, open with this mode.
+
+        channel : str, one of ['stdout', 'stderr']  
+        """
+        if channel not in ['stdout', 'stderr']:
+            raise ValueError('Invalid channel spec %s' % channel)
+        
+        if hasattr(file, 'write') and hasattr(file, 'seek'):
+            self.file = file_or_name
+        else:
+            self.file = open(file_or_name, mode)
+        self.channel = channel
+        self.ostream = getattr(sys, channel)
+        setattr(sys, channel, self)
+        self._closed = False
+
+    def close(self):
+        """Close the file and restore the channel."""
+        self.flush()
+        setattr(sys, self.channel, self.ostream)
+        self.file.close()
+        self._closed = True
+
+    def write(self, data):
+        """Write data to both channels."""
+        self.file.write(data)
+        self.ostream.write(data)
+        self.ostream.flush()
+
+    def flush(self):
+        """Flush both channels."""
+        self.file.flush()
+        self.ostream.flush()
+
+    def __del__(self):
+        if not self._closed:
+            self.close()
+
+
+def file_read(filename):
+    """Read a file and close it.  Returns the file source."""
+    fobj = open(filename,'r');
+    source = fobj.read();
+    fobj.close()
+    return source
+
+
+def file_readlines(filename):
+    """Read a file and close it.  Returns the file source using readlines()."""
+    fobj = open(filename,'r');
+    lines = fobj.readlines();
+    fobj.close()
+    return lines
+
+
+def raw_input_multi(header='', ps1='==> ', ps2='..> ',terminate_str = '.'):
+    """Take multiple lines of input.
+
+    A list with each line of input as a separate element is returned when a
+    termination string is entered (defaults to a single '.'). Input can also
+    terminate via EOF (^D in Unix, ^Z-RET in Windows).
+
+    Lines of input which end in \\ are joined into single entries (and a
+    secondary continuation prompt is issued as long as the user terminates
+    lines with \\). This allows entering very long strings which are still
+    meant to be treated as single entities.
+    """
+
+    try:
+        if header:
+            header += '\n'
+        lines = [raw_input(header + ps1)]
+    except EOFError:
+        return []
+    terminate = [terminate_str]
+    try:
+        while lines[-1:] != terminate:
+            new_line = raw_input(ps1)
+            while new_line.endswith('\\'):
+                new_line = new_line[:-1] + raw_input(ps2)
+            lines.append(new_line)
+
+        return lines[:-1]  # don't return the termination command
+    except EOFError:
+        print
+        return lines
+
+
+def raw_input_ext(prompt='',  ps2='... '):
+    """Similar to raw_input(), but accepts extended lines if input ends with \\."""
+
+    line = raw_input(prompt)
+    while line.endswith('\\'):
+        line = line[:-1] + raw_input(ps2)
+    return line
+
+
+def ask_yes_no(prompt,default=None):
+    """Asks a question and returns a boolean (y/n) answer.
+
+    If default is given (one of 'y','n'), it is used if the user input is
+    empty. Otherwise the question is repeated until an answer is given.
+
+    An EOF is treated as the default answer.  If there is no default, an
+    exception is raised to prevent infinite loops.
+
+    Valid answers are: y/yes/n/no (match is not case sensitive)."""
+
+    answers = {'y':True,'n':False,'yes':True,'no':False}
+    ans = None
+    while ans not in answers.keys():
+        try:
+            ans = raw_input(prompt+' ').lower()
+            if not ans:  # response was an empty string
+                ans = default
+        except KeyboardInterrupt:
+            pass
+        except EOFError:
+            if default in answers.keys():
+                ans = default
+                print
+            else:
+                raise
+
+    return answers[ans]
+
+
+class NLprinter:
+    """Print an arbitrarily nested list, indicating index numbers.
+
+    An instance of this class called nlprint is available and callable as a
+    function.
+
+    nlprint(list,indent=' ',sep=': ') -> prints indenting each level by 'indent'
+    and using 'sep' to separate the index from the value. """
+
+    def __init__(self):
+        self.depth = 0
+
+    def __call__(self,lst,pos='',**kw):
+        """Prints the nested list numbering levels."""
+        kw.setdefault('indent',' ')
+        kw.setdefault('sep',': ')
+        kw.setdefault('start',0)
+        kw.setdefault('stop',len(lst))
+        # we need to remove start and stop from kw so they don't propagate
+        # into a recursive call for a nested list.
+        start = kw['start']; del kw['start']
+        stop = kw['stop']; del kw['stop']
+        if self.depth == 0 and 'header' in kw.keys():
+            print kw['header']
+
+        for idx in range(start,stop):
+            elem = lst[idx]
+            if type(elem)==type([]):
+                self.depth += 1
+                self.__call__(elem,itpl('$pos$idx,'),**kw)
+                self.depth -= 1
+            else:
+                printpl(kw['indent']*self.depth+'$pos$idx$kw["sep"]$elem')
+
+nlprint = NLprinter()
+
+
+def temp_pyfile(src, ext='.py'):
+    """Make a temporary python file, return filename and filehandle.
+
+    Parameters
+    ----------
+    src : string or list of strings (no need for ending newlines if list)
+      Source code to be written to the file.
+
+    ext : optional, string
+      Extension for the generated file.
+
+    Returns
+    -------
+    (filename, open filehandle)
+      It is the caller's responsibility to close the open file and unlink it.
+    """
+    fname = tempfile.mkstemp(ext)[1]
+    f = open(fname,'w')
+    f.write(src)
+    f.flush()
+    return fname, f
+
+
+

IPython/utils/path.py

@@ -0,0 +1,342 b''
+# encoding: utf-8
+"""
+Utilities for path handling.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import sys
+
+import IPython
+from IPython.utils.process import xsys
+from IPython.utils.importstring import import_item
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+def _get_long_path_name(path):
+    """Dummy no-op."""
+    return path
+
+
+if sys.platform == 'win32':
+    def _get_long_path_name(path):
+        """Get a long path name (expand ~) on Windows using ctypes.
+
+        Examples
+        --------
+
+        >>> get_long_path_name('c:\\docume~1')
+        u'c:\\\\Documents and Settings'
+
+        """
+        try:
+            import ctypes
+        except ImportError:
+            raise ImportError('you need to have ctypes installed for this to work')
+        _GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
+        _GetLongPathName.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p,
+            ctypes.c_uint ]
+
+        buf = ctypes.create_unicode_buffer(260)
+        rv = _GetLongPathName(path, buf, 260)
+        if rv == 0 or rv > 260:
+            return path
+        else:
+            return buf.value
+
+
+def get_long_path_name(path):
+    """Expand a path into its long form.
+
+    On Windows this expands any ~ in the paths. On other platforms, it is
+    a null operation.
+    """
+    return _get_long_path_name(path)
+
+
+def get_py_filename(name):
+    """Return a valid python filename in the current directory.
+
+    If the given name is not a file, it adds '.py' and searches again.
+    Raises IOError with an informative message if the file isn't found."""
+
+    name = os.path.expanduser(name)
+    if not os.path.isfile(name) and not name.endswith('.py'):
+        name += '.py'
+    if os.path.isfile(name):
+        return name
+    else:
+        raise IOError,'File `%s` not found.' % name
+
+
+def filefind(filename, path_dirs=None):
+    """Find a file by looking through a sequence of paths.
+
+    This iterates through a sequence of paths looking for a file and returns
+    the full, absolute path of the first occurence of the file.  If no set of
+    path dirs is given, the filename is tested as is, after running through
+    :func:`expandvars` and :func:`expanduser`.  Thus a simple call::
+
+        filefind('myfile.txt')
+
+    will find the file in the current working dir, but::
+
+        filefind('~/myfile.txt')
+
+    Will find the file in the users home directory.  This function does not
+    automatically try any paths, such as the cwd or the user's home directory.
+    
+    Parameters
+    ----------
+    filename : str
+        The filename to look for.
+    path_dirs : str, None or sequence of str
+        The sequence of paths to look for the file in.  If None, the filename
+        need to be absolute or be in the cwd.  If a string, the string is
+        put into a sequence and the searched.  If a sequence, walk through
+        each element and join with ``filename``, calling :func:`expandvars`
+        and :func:`expanduser` before testing for existence.
+        
+    Returns
+    -------
+    Raises :exc:`IOError` or returns absolute path to file.
+    """
+    
+    # If paths are quoted, abspath gets confused, strip them...
+    filename = filename.strip('"').strip("'")
+    # If the input is an absolute path, just check it exists
+    if os.path.isabs(filename) and os.path.isfile(filename):
+        return filename
+        
+    if path_dirs is None:
+        path_dirs = ("",)
+    elif isinstance(path_dirs, basestring):
+        path_dirs = (path_dirs,)
+        
+    for path in path_dirs:
+        if path == '.': path = os.getcwd()
+        testname = expand_path(os.path.join(path, filename))
+        if os.path.isfile(testname):
+            return os.path.abspath(testname)
+        
+    raise IOError("File %r does not exist in any of the search paths: %r" % 
+                  (filename, path_dirs) )
+
+
+class HomeDirError(Exception):
+    pass
+
+
+def get_home_dir():
+    """Return the closest possible equivalent to a 'home' directory.
+
+    * On POSIX, we try $HOME.
+    * On Windows we try:
+      - %HOME%: rare, but some people with unix-like setups may have defined it
+      - %HOMESHARE%
+      - %HOMEDRIVE\%HOMEPATH%
+      - %USERPROFILE%
+      - Registry hack
+    * On Dos C:\
+ 
+    Currently only Posix and NT are implemented, a HomeDirError exception is
+    raised for all other OSes.
+    """
+
+    isdir = os.path.isdir
+    env = os.environ
+
+    # first, check py2exe distribution root directory for _ipython.
+    # This overrides all. Normally does not exist.
+
+    if hasattr(sys, "frozen"): #Is frozen by py2exe
+        if '\\library.zip\\' in IPython.__file__.lower():#libraries compressed to zip-file
+            root, rest = IPython.__file__.lower().split('library.zip')
+        else: 
+            root=os.path.join(os.path.split(IPython.__file__)[0],"../../")
+        root=os.path.abspath(root).rstrip('\\')
+        if isdir(os.path.join(root, '_ipython')):
+            os.environ["IPYKITROOT"] = root
+        return root.decode(sys.getfilesystemencoding())
+
+    if os.name == 'posix':
+        # Linux, Unix, AIX, OS X
+        try:
+            homedir = env['HOME']
+        except KeyError:
+            raise HomeDirError('Undefined $HOME, IPython cannot proceed.')
+        else:
+            return homedir.decode(sys.getfilesystemencoding())
+    elif os.name == 'nt':
+        # Now for win9x, XP, Vista, 7?
+        # For some strange reason all of these return 'nt' for os.name.
+        # First look for a network home directory. This will return the UNC
+        # path (\\server\\Users\%username%) not the mapped path (Z:\). This
+        # is needed when running IPython on cluster where all paths have to 
+        # be UNC.
+        try:
+            # A user with a lot of unix tools in win32 may have defined $HOME,
+            # honor it if it exists, but otherwise let the more typical
+            # %HOMESHARE% variable be used.
+            homedir = env.get('HOME')
+            if homedir is None:
+                homedir = env['HOMESHARE']
+        except KeyError:
+            pass
+        else:
+            if isdir(homedir):
+                return homedir.decode(sys.getfilesystemencoding())
+
+        # Now look for a local home directory
+        try:
+            homedir = os.path.join(env['HOMEDRIVE'],env['HOMEPATH'])
+        except KeyError:
+            pass
+        else:
+            if isdir(homedir):
+                return homedir.decode(sys.getfilesystemencoding())
+
+        # Now the users profile directory
+        try:
+            homedir = os.path.join(env['USERPROFILE'])
+        except KeyError:
+            pass
+        else:
+            if isdir(homedir):
+                return homedir.decode(sys.getfilesystemencoding())
+
+        # Use the registry to get the 'My Documents' folder.
+        try:
+            import _winreg as wreg
+            key = wreg.OpenKey(
+                wreg.HKEY_CURRENT_USER,
+                "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
+            )
+            homedir = wreg.QueryValueEx(key,'Personal')[0]
+            key.Close()
+        except:
+            pass
+        else:
+            if isdir(homedir):
+                return homedir.decode(sys.getfilesystemencoding())
+
+        # If all else fails, raise HomeDirError
+        raise HomeDirError('No valid home directory could be found')
+    elif os.name == 'dos':
+        # Desperate, may do absurd things in classic MacOS. May work under DOS.
+        return 'C:\\'.decode(sys.getfilesystemencoding())
+    else:
+        raise HomeDirError('No valid home directory could be found for your OS')
+
+
+def get_ipython_dir():
+    """Get the IPython directory for this platform and user.
+    
+    This uses the logic in `get_home_dir` to find the home directory
+    and the adds .ipython to the end of the path.
+    """
+    ipdir_def = '.ipython'
+    print get_home_dir
+    home_dir = get_home_dir()
+    print home_dir
+    #import pdb; pdb.set_trace()  # dbg
+    ipdir = os.environ.get(
+        'IPYTHON_DIR', os.environ.get(
+            'IPYTHONDIR', os.path.join(home_dir, ipdir_def)
+        )
+    )
+    return ipdir.decode(sys.getfilesystemencoding())
+
+
+def get_ipython_package_dir():
+    """Get the base directory where IPython itself is installed."""
+    ipdir = os.path.dirname(IPython.__file__)
+    return ipdir.decode(sys.getfilesystemencoding())
+
+
+def get_ipython_module_path(module_str):
+    """Find the path to an IPython module in this version of IPython.
+
+    This will always find the version of the module that is in this importable
+    IPython package. This will always return the path to the ``.py``
+    version of the module.
+    """
+    if module_str == 'IPython':
+        return os.path.join(get_ipython_package_dir(), '__init__.py')
+    mod = import_item(module_str)
+    the_path = mod.__file__.replace('.pyc', '.py')
+    the_path = the_path.replace('.pyo', '.py')
+    return the_path.decode(sys.getfilesystemencoding())
+
+
+def expand_path(s):
+    """Expand $VARS and ~names in a string, like a shell
+
+    :Examples:
+    
+       In [2]: os.environ['FOO']='test'
+
+       In [3]: expand_path('variable FOO is $FOO')
+       Out[3]: 'variable FOO is test'
+    """
+    # This is a pretty subtle hack. When expand user is given a UNC path
+    # on Windows (\\server\share$\%username%), os.path.expandvars, removes
+    # the $ to get (\\server\share\%username%). I think it considered $
+    # alone an empty var. But, we need the $ to remains there (it indicates
+    # a hidden share).
+    if os.name=='nt':
+        s = s.replace('$\\', 'IPYTHON_TEMP')
+    s = os.path.expandvars(os.path.expanduser(s))
+    if os.name=='nt':
+        s = s.replace('IPYTHON_TEMP', '$\\')
+    return s
+
+
+def target_outdated(target,deps):
+    """Determine whether a target is out of date.
+
+    target_outdated(target,deps) -> 1/0
+
+    deps: list of filenames which MUST exist.
+    target: single filename which may or may not exist.
+
+    If target doesn't exist or is older than any file listed in deps, return
+    true, otherwise return false.
+    """
+    try:
+        target_time = os.path.getmtime(target)
+    except os.error:
+        return 1
+    for dep in deps:
+        dep_time = os.path.getmtime(dep)
+        if dep_time > target_time:
+            #print "For target",target,"Dep failed:",dep # dbg
+            #print "times (dep,tar):",dep_time,target_time # dbg
+            return 1
+    return 0
+
+
+def target_update(target,deps,cmd):
+    """Update a target with a given command given a list of dependencies.
+
+    target_update(target,deps,cmd) -> runs cmd if target is outdated.
+
+    This is just a wrapper around target_outdated() which calls the given
+    command if target is outdated."""
+
+    if target_outdated(target,deps):
+        xsys(cmd)
+

IPython/utils/process.py

@@ -0,0 +1,368 b''
+# encoding: utf-8
+"""
+Utilities for working with external processes.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import sys
+import shlex
+import subprocess
+
+from IPython.utils.terminal import set_term_title
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+class FindCmdError(Exception):
+    pass
+
+
+def _find_cmd(cmd):
+    """Find the full path to a command using which."""
+    return os.popen('which %s' % cmd).read().strip()
+
+
+if os.name == 'posix':
+    def _find_cmd(cmd):
+        """Find the full path to a command using which."""
+        return getoutputerror('/usr/bin/env which %s' % cmd)[0]
+
+
+if sys.platform == 'win32':
+    def _find_cmd(cmd):
+        """Find the full path to a .bat or .exe using the win32api module."""
+        try:
+            from win32api import SearchPath
+        except ImportError:
+            raise ImportError('you need to have pywin32 installed for this to work')
+        else:
+            PATH = os.environ['PATH']
+            extensions = ['.exe', '.com', '.bat', '.py']
+            path = None
+            for ext in extensions:
+                try:
+                    path = SearchPath(PATH,cmd + ext)[0]
+                except:
+                    pass
+            if path is None:
+                raise OSError("command %r not found" % cmd)
+            else:
+                return path
+
+
+def find_cmd(cmd):
+    """Find absolute path to executable cmd in a cross platform manner.
+    
+    This function tries to determine the full path to a command line program
+    using `which` on Unix/Linux/OS X and `win32api` on Windows.  Most of the
+    time it will use the version that is first on the users `PATH`.  If
+    cmd is `python` return `sys.executable`.
+
+    Warning, don't use this to find IPython command line programs as there
+    is a risk you will find the wrong one.  Instead find those using the
+    following code and looking for the application itself::
+    
+        from IPython.utils.path import get_ipython_module_path
+        from IPython.utils.process import pycmd2argv
+        argv = pycmd2argv(get_ipython_module_path('IPython.core.ipapp'))
+
+    Parameters
+    ----------
+    cmd : str
+        The command line program to look for.
+    """
+    if cmd == 'python':
+        return os.path.abspath(sys.executable)
+    try:
+        path = _find_cmd(cmd)
+    except OSError:
+        raise FindCmdError('command could not be found: %s' % cmd)
+    # which returns empty if not found
+    if path == '':
+        raise FindCmdError('command could not be found: %s' % cmd)
+    return os.path.abspath(path)
+
+
+def pycmd2argv(cmd):
+    r"""Take the path of a python command and return a list (argv-style).
+
+    This only works on Python based command line programs and will find the
+    location of the ``python`` executable using ``sys.executable`` to make
+    sure the right version is used.
+
+    For a given path ``cmd``, this returns [cmd] if cmd's extension is .exe,
+    .com or .bat, and [, cmd] otherwise.  
+
+    Parameters
+    ----------
+    cmd : string
+      The path of the command.
+
+    Returns
+    -------
+    argv-style list.
+    """
+    ext = os.path.splitext(cmd)[1]
+    if ext in ['.exe', '.com', '.bat']:
+        return [cmd]
+    else:
+        if sys.platform == 'win32':
+            # The -u option here turns on unbuffered output, which is required
+            # on Win32 to prevent wierd conflict and problems with Twisted.
+            # Also, use sys.executable to make sure we are picking up the 
+            # right python exe.
+            return [sys.executable, '-u', cmd]
+        else:
+            return [sys.executable, cmd]
+
+
+def arg_split(s, posix=False):
+    """Split a command line's arguments in a shell-like manner.
+
+    This is a modified version of the standard library's shlex.split()
+    function, but with a default of posix=False for splitting, so that quotes
+    in inputs are respected."""
+
+    # XXX - there may be unicode-related problems here!!!  I'm not sure that
+    # shlex is truly unicode-safe, so it might be necessary to do
+    #
+    # s = s.encode(sys.stdin.encoding)
+    #
+    # first, to ensure that shlex gets a normal string.  Input from anyone who
+    # knows more about unicode and shlex than I would be good to have here...
+    lex = shlex.shlex(s, posix=posix)
+    lex.whitespace_split = True
+    return list(lex)
+
+
+def system(cmd, verbose=0, debug=0, header=''):
+    """Execute a system command, return its exit status.
+
+    Options:
+
+    - verbose (0): print the command to be executed.
+
+    - debug (0): only print, do not actually execute.
+
+    - header (''): Header to print on screen prior to the executed command (it
+    is only prepended to the command, no newlines are added).
+
+    Note: a stateful version of this function is available through the
+    SystemExec class."""
+
+    stat = 0
+    if verbose or debug: print header+cmd
+    sys.stdout.flush()
+    if not debug: stat = os.system(cmd)
+    return stat
+
+
+def abbrev_cwd():
+    """ Return abbreviated version of cwd, e.g. d:mydir """
+    cwd = os.getcwd().replace('\\','/')
+    drivepart = ''
+    tail = cwd
+    if sys.platform == 'win32':
+        if len(cwd) < 4:
+            return cwd
+        drivepart,tail = os.path.splitdrive(cwd)
+
+
+    parts = tail.split('/')
+    if len(parts) > 2:
+        tail = '/'.join(parts[-2:])
+
+    return (drivepart + (
+        cwd == '/' and '/' or tail))
+
+
+# This function is used by ipython in a lot of places to make system calls.
+# We need it to be slightly different under win32, due to the vagaries of
+# 'network shares'.  A win32 override is below.
+
+def shell(cmd, verbose=0, debug=0, header=''):
+    """Execute a command in the system shell, always return None.
+
+    Options:
+
+    - verbose (0): print the command to be executed.
+
+    - debug (0): only print, do not actually execute.
+
+    - header (''): Header to print on screen prior to the executed command (it
+    is only prepended to the command, no newlines are added).
+
+    Note: this is similar to system(), but it returns None so it can
+    be conveniently used in interactive loops without getting the return value
+    (typically 0) printed many times."""
+
+    stat = 0
+    if verbose or debug: print header+cmd
+    # flush stdout so we don't mangle python's buffering
+    sys.stdout.flush()
+
+    if not debug:
+        set_term_title("IPy " + cmd)
+        os.system(cmd)
+        set_term_title("IPy " + abbrev_cwd())
+
+# override shell() for win32 to deal with network shares
+if os.name in ('nt','dos'):
+
+    shell_ori = shell
+
+    def shell(cmd, verbose=0, debug=0, header=''):
+        if os.getcwd().startswith(r"\\"):
+            path = os.getcwd()
+            # change to c drive (cannot be on UNC-share when issuing os.system,
+            # as cmd.exe cannot handle UNC addresses)
+            os.chdir("c:")
+            # issue pushd to the UNC-share and then run the command
+            try:
+                shell_ori('"pushd %s&&"'%path+cmd,verbose,debug,header)
+            finally:
+                os.chdir(path)
+        else:
+            shell_ori(cmd,verbose,debug,header)
+
+    shell.__doc__ = shell_ori.__doc__
+
+    
+def getoutput(cmd, verbose=0, debug=0, header='', split=0):
+    """Dummy substitute for perl's backquotes.
+
+    Executes a command and returns the output.
+
+    Accepts the same arguments as system(), plus:
+
+    - split(0): if true, the output is returned as a list split on newlines.
+
+    Note: a stateful version of this function is available through the
+    SystemExec class.
+
+    This is pretty much deprecated and rarely used, getoutputerror may be 
+    what you need.
+
+    """
+
+    if verbose or debug: print header+cmd
+    if not debug:
+        pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout
+        output = pipe.read()
+        # stipping last \n is here for backwards compat.
+        if output.endswith('\n'):
+            output = output[:-1]
+        if split:
+            return output.split('\n')
+        else:
+            return output
+
+
+# for compatibility with older naming conventions
+xsys = system
+
+
+def getoutputerror(cmd, verbose=0, debug=0, header='', split=0):
+    """Return (standard output,standard error) of executing cmd in a shell.
+
+    Accepts the same arguments as system(), plus:
+
+    - split(0): if true, each of stdout/err is returned as a list split on
+    newlines.
+
+    Note: a stateful version of this function is available through the
+    SystemExec class."""
+
+    if verbose or debug: print header+cmd
+    if not cmd:
+        if split:
+            return [],[]
+        else:
+            return '',''
+    if not debug:
+        p = subprocess.Popen(cmd, shell=True,
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE,
+                             close_fds=True)
+        pin, pout, perr = (p.stdin, p.stdout, p.stderr)
+
+        tout = pout.read().rstrip()
+        terr = perr.read().rstrip()
+        pin.close()
+        pout.close()
+        perr.close()
+        if split:
+            return tout.split('\n'),terr.split('\n')
+        else:
+            return tout,terr
+
+
+class SystemExec:
+    """Access the system and getoutput functions through a stateful interface.
+
+    Note: here we refer to the system and getoutput functions from this
+    library, not the ones from the standard python library.
+
+    This class offers the system and getoutput functions as methods, but the
+    verbose, debug and header parameters can be set for the instance (at
+    creation time or later) so that they don't need to be specified on each
+    call.
+
+    For efficiency reasons, there's no way to override the parameters on a
+    per-call basis other than by setting instance attributes. If you need
+    local overrides, it's best to directly call system() or getoutput().
+
+    The following names are provided as alternate options:
+     - xsys: alias to system
+     - bq: alias to getoutput
+
+    An instance can then be created as:
+    >>> sysexec = SystemExec(verbose=1,debug=0,header='Calling: ')
+    """
+
+    def __init__(self, verbose=0, debug=0, header='', split=0):
+        """Specify the instance's values for verbose, debug and header."""
+        self.verbose = verbose
+        self.debug = debug
+        self.header = header
+        self.split = split
+
+    def system(self, cmd):
+        """Stateful interface to system(), with the same keyword parameters."""
+
+        system(cmd, self.verbose, self.debug, self.header)
+
+    def shell(self, cmd):
+        """Stateful interface to shell(), with the same keyword parameters."""
+
+        shell(cmd, self.verbose, self.debug, self.header)
+
+    xsys = system  # alias
+
+    def getoutput(self, cmd):
+        """Stateful interface to getoutput()."""
+
+        return getoutput(cmd, self.verbose, self.debug, self.header, self.split)
+
+    def getoutputerror(self, cmd):
+        """Stateful interface to getoutputerror()."""
+
+        return getoutputerror(cmd, self.verbose, self.debug, self.header, self.split)
+
+    bq = getoutput  # alias
+
+

IPython/utils/sysinfo.py

@@ -0,0 +1,100 b''
+# encoding: utf-8
+"""
+Utilities for getting information about a system.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import platform
+import sys
+import subprocess
+
+from IPython.core import release
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def sys_info():
+    """Return useful information about IPython and the system, as a string.
+
+    Examples
+    --------
+    In [1]: print(sys_info())
+    IPython version: 0.11.bzr.r1340   # random
+    BZR revision   : 1340
+    Platform info  : os.name -> posix, sys.platform -> linux2
+                   : Linux-2.6.31-17-generic-i686-with-Ubuntu-9.10-karmic
+    Python info    : 2.6.4 (r264:75706, Dec  7 2009, 18:45:15) 
+    [GCC 4.4.1]
+    """
+    out = []
+    out.append('IPython version: %s' % release.version)
+    out.append('BZR revision   : %s' % release.revision)
+    out.append('Platform info  : os.name -> %s, sys.platform -> %s' %
+               (os.name,sys.platform) )
+    out.append('               : %s' % platform.platform())
+    out.append('Python info    : %s' % sys.version)
+    out.append('')  # ensure closing newline
+    return '\n'.join(out)
+
+
+def _num_cpus_unix():
+    """Return the number of active CPUs on a Unix system."""
+    return os.sysconf("SC_NPROCESSORS_ONLN")
+
+
+def _num_cpus_darwin():
+    """Return the number of active CPUs on a Darwin system."""
+    p = subprocess.Popen(['sysctl','-n','hw.ncpu'],stdout=subprocess.PIPE)
+    return p.stdout.read()
+
+
+def _num_cpus_windows():
+    """Return the number of active CPUs on a Windows system."""
+    return os.environ.get("NUMBER_OF_PROCESSORS")
+
+
+def num_cpus():
+   """Return the effective number of CPUs in the system as an integer.
+
+   This cross-platform function makes an attempt at finding the total number of
+   available CPUs in the system, as returned by various underlying system and
+   python calls.
+
+   If it can't find a sensible answer, it returns 1 (though an error *may* make
+   it return a large positive number that's actually incorrect).
+   """
+
+   # Many thanks to the Parallel Python project (http://www.parallelpython.com)
+   # for the names of the keys we needed to look up for this function.  This
+   # code was inspired by their equivalent function.
+
+   ncpufuncs = {'Linux':_num_cpus_unix,
+                'Darwin':_num_cpus_darwin,
+                'Windows':_num_cpus_windows,
+                # On Vista, python < 2.5.2 has a bug and returns 'Microsoft'
+                # See http://bugs.python.org/issue1082 for details.
+                'Microsoft':_num_cpus_windows,
+                }
+
+   ncpufunc = ncpufuncs.get(platform.system(),
+                            # default to unix version (Solaris, AIX, etc)
+                            _num_cpus_unix)
+
+   try:
+       ncpus = max(1,int(ncpufunc()))
+   except:
+       ncpus = 1
+   return ncpus
+

IPython/utils/terminal.py

@@ -0,0 +1,162 b''
+# encoding: utf-8
+"""
+Utilities for working with terminals.
+
+Authors:
+
+* Brian E. Granger
+* Fernando Perez
+* Alexander Belchenko (e-mail: bialix AT ukr.net)
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import struct
+import sys
+import warnings
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+# This variable is part of the expected API of the module:
+ignore_termtitle = True
+
+
+def _term_clear():
+    pass
+
+
+if os.name == 'posix':
+    def _term_clear():
+        os.system('clear')
+
+
+if sys.platform == 'win32':
+    def _term_clear():
+        os.system('cls')
+
+
+def term_clear():
+    _term_clear()
+
+
+def toggle_set_term_title(val):
+    """Control whether set_term_title is active or not.
+
+    set_term_title() allows writing to the console titlebar.  In embedded
+    widgets this can cause problems, so this call can be used to toggle it on
+    or off as needed.
+
+    The default state of the module is for the function to be disabled.
+
+    Parameters
+    ----------
+      val : bool
+        If True, set_term_title() actually writes to the terminal (using the
+        appropriate platform-specific module).  If False, it is a no-op.
+    """
+    global ignore_termtitle
+    ignore_termtitle = not(val)
+
+
+def _set_term_title(*args,**kw):
+    """Dummy no-op."""
+    pass
+
+
+def _set_term_title_xterm(title):
+    """ Change virtual terminal title in xterm-workalikes """
+    sys.stdout.write('\033]0;%s\007' % title)
+
+if os.name == 'posix':
+    TERM = os.environ.get('TERM','')
+    if (TERM == 'xterm') or (TERM == 'xterm-color'):
+        _set_term_title = _set_term_title_xterm
+
+
+if sys.platform == 'win32':
+    try:
+        import ctypes
+
+        SetConsoleTitleW = ctypes.windll.kernel32.SetConsoleTitleW
+        SetConsoleTitleW.argtypes = [ctypes.c_wchar_p]
+    
+        def _set_term_title(title):
+            """Set terminal title using ctypes to access the Win32 APIs."""
+            SetConsoleTitleW(title)
+    except ImportError:
+        def _set_term_title(title):
+            """Set terminal title using the 'title' command."""
+            global ignore_termtitle
+
+            try:
+                # Cannot be on network share when issuing system commands
+                curr = os.getcwd()
+                os.chdir("C:")
+                ret = os.system("title " + title)
+            finally:
+                os.chdir(curr)
+            if ret:
+                # non-zero return code signals error, don't try again
+                ignore_termtitle = True
+
+
+def set_term_title(title):
+    """Set terminal title using the necessary platform-dependent calls."""
+    if ignore_termtitle:
+        return
+    _set_term_title(title)
+
+
+def freeze_term_title():
+    warnings.warn("This function is deprecated, use toggle_set_term_title()")
+    global ignore_termtitle
+    ignore_termtitle = True
+
+
+def get_terminal_size(defaultx=80, defaulty=25):
+    return defaultx, defaulty
+
+
+if sys.platform == 'win32':
+    def get_terminal_size(defaultx=80, defaulty=25):
+        """Return size of current terminal console.
+
+        This function try to determine actual size of current working
+        console window and return tuple (sizex, sizey) if success,
+        or default size (defaultx, defaulty) otherwise.
+
+        Dependencies: ctypes should be installed.
+
+        Author: Alexander Belchenko (e-mail: bialix AT ukr.net)
+        """
+        try:
+            import ctypes
+        except ImportError:
+            return defaultx, defaulty
+
+        h = ctypes.windll.kernel32.GetStdHandle(-11)
+        csbi = ctypes.create_string_buffer(22)
+        res = ctypes.windll.kernel32.GetConsoleScreenBufferInfo(h, csbi)
+
+        if res:
+            (bufx, bufy, curx, cury, wattr,
+             left, top, right, bottom, maxx, maxy) = struct.unpack(
+                "hhhhHhhhhhh", csbi.raw)
+            sizex = right - left + 1
+            sizey = bottom - top + 1
+            return (sizex, sizey)
+        else:
+            return (defaultx, defaulty)
+

IPython/utils/tests/test_io.py

@@ -0,0 +1,61 b''
+# encoding: utf-8
+"""Tests for io.py"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+
+from cStringIO import StringIO
+
+import nose.tools as nt
+
+from IPython.testing import decorators as dec
+from IPython.utils.io import Tee
+
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
+
+
+def test_tee_simple():
+    "Very simple check with stdout only"
+    chan = StringIO()
+    text = 'Hello'
+    tee = Tee(chan, channel='stdout')
+    print >> chan, text,
+    nt.assert_equal(chan.getvalue(), text)
+
+
+class TeeTestCase(dec.ParametricTestCase):
+
+    def tchan(self, channel, check='close'):
+        trap = StringIO()
+        chan = StringIO()
+        text = 'Hello'
+        
+        std_ori = getattr(sys, channel)
+        setattr(sys, channel, trap)
+
+        tee = Tee(chan, channel=channel)
+        print >> chan, text,
+        setattr(sys, channel, std_ori)
+        trap_val = trap.getvalue()
+        nt.assert_equals(chan.getvalue(), text)
+        if check=='close':
+            tee.close()
+        else:
+            del tee
+
+    def test(self):
+        for chan in ['stdout', 'stderr']:
+            for check in ['close', 'del']:
+                yield self.tchan(chan, check)

IPython/utils/text.py

@@ -0,0 +1,473 b''
+# encoding: utf-8
+"""
+Utilities for working with strings and text.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import __main__
+
+import os
+import re
+import shutil
+import types
+
+from IPython.external.path import path
+
+from IPython.utils.generics import result_display
+from IPython.utils.io import nlprint
+from IPython.utils.data import flatten
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+StringTypes = types.StringTypes
+
+
+def unquote_ends(istr):
+    """Remove a single pair of quotes from the endpoints of a string."""
+
+    if not istr:
+        return istr
+    if (istr[0]=="'" and istr[-1]=="'") or \
+       (istr[0]=='"' and istr[-1]=='"'):
+        return istr[1:-1]
+    else:
+        return istr
+
+
+class LSString(str):
+    """String derivative with a special access attributes.
+
+    These are normal strings, but with the special attributes:
+
+        .l (or .list) : value as list (split on newlines).
+        .n (or .nlstr): original value (the string itself).
+        .s (or .spstr): value as whitespace-separated string.
+        .p (or .paths): list of path objects
+
+    Any values which require transformations are computed only once and
+    cached.
+
+    Such strings are very useful to efficiently interact with the shell, which
+    typically only understands whitespace-separated options for commands."""
+
+    def get_list(self):
+        try:
+            return self.__list
+        except AttributeError:
+            self.__list = self.split('\n')
+            return self.__list
+
+    l = list = property(get_list)
+
+    def get_spstr(self):
+        try:
+            return self.__spstr
+        except AttributeError:
+            self.__spstr = self.replace('\n',' ')
+            return self.__spstr
+
+    s = spstr = property(get_spstr)
+
+    def get_nlstr(self):
+        return self
+
+    n = nlstr = property(get_nlstr)
+
+    def get_paths(self):
+        try:
+            return self.__paths
+        except AttributeError:
+            self.__paths = [path(p) for p in self.split('\n') if os.path.exists(p)]
+            return self.__paths
+
+    p = paths = property(get_paths)
+
+
+def print_lsstring(arg):
+    """ Prettier (non-repr-like) and more informative printer for LSString """
+    print "LSString (.p, .n, .l, .s available). Value:"
+    print arg
+
+
+print_lsstring = result_display.when_type(LSString)(print_lsstring)
+
+
+class SList(list):
+    """List derivative with a special access attributes.
+
+    These are normal lists, but with the special attributes:
+
+        .l (or .list) : value as list (the list itself).
+        .n (or .nlstr): value as a string, joined on newlines.
+        .s (or .spstr): value as a string, joined on spaces.
+        .p (or .paths): list of path objects
+
+    Any values which require transformations are computed only once and
+    cached."""
+
+    def get_list(self):
+        return self
+
+    l = list = property(get_list)
+
+    def get_spstr(self):
+        try:
+            return self.__spstr
+        except AttributeError:
+            self.__spstr = ' '.join(self)
+            return self.__spstr
+
+    s = spstr = property(get_spstr)
+
+    def get_nlstr(self):
+        try:
+            return self.__nlstr
+        except AttributeError:
+            self.__nlstr = '\n'.join(self)
+            return self.__nlstr
+
+    n = nlstr = property(get_nlstr)
+
+    def get_paths(self):
+        try:
+            return self.__paths
+        except AttributeError:
+            self.__paths = [path(p) for p in self if os.path.exists(p)]
+            return self.__paths
+
+    p = paths = property(get_paths)
+
+    def grep(self, pattern, prune = False, field = None):
+        """ Return all strings matching 'pattern' (a regex or callable)
+
+        This is case-insensitive. If prune is true, return all items
+        NOT matching the pattern.
+
+        If field is specified, the match must occur in the specified
+        whitespace-separated field.
+
+        Examples::
+
+            a.grep( lambda x: x.startswith('C') )
+            a.grep('Cha.*log', prune=1)
+            a.grep('chm', field=-1)
+        """
+
+        def match_target(s):
+            if field is None:
+                return s
+            parts = s.split()
+            try:
+                tgt = parts[field]
+                return tgt
+            except IndexError:
+                return ""
+
+        if isinstance(pattern, basestring):
+            pred = lambda x : re.search(pattern, x, re.IGNORECASE)
+        else:
+            pred = pattern
+        if not prune:
+            return SList([el for el in self if pred(match_target(el))])
+        else:
+            return SList([el for el in self if not pred(match_target(el))])
+
+    def fields(self, *fields):
+        """ Collect whitespace-separated fields from string list
+
+        Allows quick awk-like usage of string lists.
+
+        Example data (in var a, created by 'a = !ls -l')::
+            -rwxrwxrwx  1 ville None      18 Dec 14  2006 ChangeLog
+            drwxrwxrwx+ 6 ville None       0 Oct 24 18:05 IPython
+
+        a.fields(0) is ['-rwxrwxrwx', 'drwxrwxrwx+']
+        a.fields(1,0) is ['1 -rwxrwxrwx', '6 drwxrwxrwx+']
+        (note the joining by space).
+        a.fields(-1) is ['ChangeLog', 'IPython']
+
+        IndexErrors are ignored.
+
+        Without args, fields() just split()'s the strings.
+        """
+        if len(fields) == 0:
+            return [el.split() for el in self]
+
+        res = SList()
+        for el in [f.split() for f in self]:
+            lineparts = []
+
+            for fd in fields:
+                try:
+                    lineparts.append(el[fd])
+                except IndexError:
+                    pass
+            if lineparts:
+                res.append(" ".join(lineparts))
+
+        return res
+
+    def sort(self,field= None,  nums = False):
+        """ sort by specified fields (see fields())
+
+        Example::
+            a.sort(1, nums = True)
+
+        Sorts a by second field, in numerical order (so that 21 > 3)
+
+        """
+
+        #decorate, sort, undecorate
+        if field is not None:
+            dsu = [[SList([line]).fields(field),  line] for line in self]
+        else:
+            dsu = [[line,  line] for line in self]
+        if nums:
+            for i in range(len(dsu)):
+                numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
+                try:
+                    n = int(numstr)
+                except ValueError:
+                    n = 0;
+                dsu[i][0] = n
+
+
+        dsu.sort()
+        return SList([t[1] for t in dsu])
+
+
+def print_slist(arg):
+    """ Prettier (non-repr-like) and more informative printer for SList """
+    print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
+    if hasattr(arg,  'hideonce') and arg.hideonce:
+        arg.hideonce = False
+        return
+
+    nlprint(arg)
+
+
+print_slist = result_display.when_type(SList)(print_slist)
+
+
+def esc_quotes(strng):
+    """Return the input string with single and double quotes escaped out"""
+
+    return strng.replace('"','\\"').replace("'","\\'")
+
+
+def make_quoted_expr(s):
+    """Return string s in appropriate quotes, using raw string if possible.
+
+    XXX - example removed because it caused encoding errors in documentation
+    generation.  We need a new example that doesn't contain invalid chars.
+
+    Note the use of raw string and padding at the end to allow trailing
+    backslash.
+    """
+
+    tail = ''
+    tailpadding = ''
+    raw  = ''
+    if "\\" in s:
+        raw = 'r'
+        if s.endswith('\\'):
+            tail = '[:-1]'
+            tailpadding = '_'
+    if '"' not in s:
+        quote = '"'
+    elif "'" not in s:
+        quote = "'"
+    elif '"""' not in s and not s.endswith('"'):
+        quote = '"""'
+    elif "'''" not in s and not s.endswith("'"):
+        quote = "'''"
+    else:
+        # give up, backslash-escaped string will do
+        return '"%s"' % esc_quotes(s)
+    res = raw + quote + s + tailpadding + quote + tail
+    return res
+
+
+def qw(words,flat=0,sep=None,maxsplit=-1):
+    """Similar to Perl's qw() operator, but with some more options.
+
+    qw(words,flat=0,sep=' ',maxsplit=-1) -> words.split(sep,maxsplit)
+
+    words can also be a list itself, and with flat=1, the output will be
+    recursively flattened.
+
+    Examples:
+
+    >>> qw('1 2')
+    ['1', '2']
+
+    >>> qw(['a b','1 2',['m n','p q']])
+    [['a', 'b'], ['1', '2'], [['m', 'n'], ['p', 'q']]]
+
+    >>> qw(['a b','1 2',['m n','p q']],flat=1)
+    ['a', 'b', '1', '2', 'm', 'n', 'p', 'q']
+    """
+
+    if type(words) in StringTypes:
+        return [word.strip() for word in words.split(sep,maxsplit)
+                if word and not word.isspace() ]
+    if flat:
+        return flatten(map(qw,words,[1]*len(words)))
+    return map(qw,words)
+
+
+def qwflat(words,sep=None,maxsplit=-1):
+    """Calls qw(words) in flat mode. It's just a convenient shorthand."""
+    return qw(words,1,sep,maxsplit)
+
+
+def qw_lol(indata):
+    """qw_lol('a b') -> [['a','b']],
+    otherwise it's just a call to qw().
+
+    We need this to make sure the modules_some keys *always* end up as a
+    list of lists."""
+
+    if type(indata) in StringTypes:
+        return [qw(indata)]
+    else:
+        return qw(indata)
+
+
+def grep(pat,list,case=1):
+    """Simple minded grep-like function.
+    grep(pat,list) returns occurrences of pat in list, None on failure.
+
+    It only does simple string matching, with no support for regexps. Use the
+    option case=0 for case-insensitive matching."""
+
+    # This is pretty crude. At least it should implement copying only references
+    # to the original data in case it's big. Now it copies the data for output.
+    out=[]
+    if case:
+        for term in list:
+            if term.find(pat)>-1: out.append(term)
+    else:
+        lpat=pat.lower()
+        for term in list:
+            if term.lower().find(lpat)>-1: out.append(term)
+
+    if len(out): return out
+    else: return None
+
+
+def dgrep(pat,*opts):
+    """Return grep() on dir()+dir(__builtins__).
+
+    A very common use of grep() when working interactively."""
+
+    return grep(pat,dir(__main__)+dir(__main__.__builtins__),*opts)
+
+
+def idgrep(pat):
+    """Case-insensitive dgrep()"""
+
+    return dgrep(pat,0)
+
+
+def igrep(pat,list):
+    """Synonym for case-insensitive grep."""
+
+    return grep(pat,list,case=0)
+
+
+def indent(str,nspaces=4,ntabs=0):
+    """Indent a string a given number of spaces or tabstops.
+
+    indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
+    """
+    if str is None:
+        return
+    ind = '\t'*ntabs+' '*nspaces
+    outstr = '%s%s' % (ind,str.replace(os.linesep,os.linesep+ind))
+    if outstr.endswith(os.linesep+ind):
+        return outstr[:-len(ind)]
+    else:
+        return outstr
+
+def native_line_ends(filename,backup=1):
+    """Convert (in-place) a file to line-ends native to the current OS.
+
+    If the optional backup argument is given as false, no backup of the
+    original file is left.  """
+
+    backup_suffixes = {'posix':'~','dos':'.bak','nt':'.bak','mac':'.bak'}
+
+    bak_filename = filename + backup_suffixes[os.name]
+
+    original = open(filename).read()
+    shutil.copy2(filename,bak_filename)
+    try:
+        new = open(filename,'wb')
+        new.write(os.linesep.join(original.splitlines()))
+        new.write(os.linesep) # ALWAYS put an eol at the end of the file
+        new.close()
+    except:
+        os.rename(bak_filename,filename)
+    if not backup:
+        try:
+            os.remove(bak_filename)
+        except:
+            pass
+
+
+def list_strings(arg):
+    """Always return a list of strings, given a string or list of strings
+    as input.
+
+    :Examples:
+
+        In [7]: list_strings('A single string')
+        Out[7]: ['A single string']
+
+        In [8]: list_strings(['A single string in a list'])
+        Out[8]: ['A single string in a list']
+
+        In [9]: list_strings(['A','list','of','strings'])
+        Out[9]: ['A', 'list', 'of', 'strings']
+    """
+
+    if isinstance(arg,basestring): return [arg]
+    else: return arg
+
+
+def marquee(txt='',width=78,mark='*'):
+    """Return the input string centered in a 'marquee'.
+
+    :Examples:
+
+        In [16]: marquee('A test',40)
+        Out[16]: '**************** A test ****************'
+
+        In [17]: marquee('A test',40,'-')
+        Out[17]: '---------------- A test ----------------'
+
+        In [18]: marquee('A test',40,' ')
+        Out[18]: '                 A test                 '
+
+    """
+    if not txt:
+        return (mark*width)[:width]
+    nmark = (width-len(txt)-2)/len(mark)/2
+    if nmark < 0: nmark =0
+    marks = mark*nmark
+    return '%s %s %s' % (marks,txt,marks)
+
+

IPython/utils/timing.py

@@ -0,0 +1,116 b''
+# encoding: utf-8
+"""
+Utilities for timing code execution.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import time
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+# If possible (Unix), use the resource module instead of time.clock()
+try:
+    import resource
+    def clocku():
+        """clocku() -> floating point number
+
+        Return the *USER* CPU time in seconds since the start of the process.
+        This is done via a call to resource.getrusage, so it avoids the
+        wraparound problems in time.clock()."""
+
+        return resource.getrusage(resource.RUSAGE_SELF)[0]
+
+    def clocks():
+        """clocks() -> floating point number
+
+        Return the *SYSTEM* CPU time in seconds since the start of the process.
+        This is done via a call to resource.getrusage, so it avoids the
+        wraparound problems in time.clock()."""
+
+        return resource.getrusage(resource.RUSAGE_SELF)[1]
+
+    def clock():
+        """clock() -> floating point number
+
+        Return the *TOTAL USER+SYSTEM* CPU time in seconds since the start of
+        the process.  This is done via a call to resource.getrusage, so it
+        avoids the wraparound problems in time.clock()."""
+
+        u,s = resource.getrusage(resource.RUSAGE_SELF)[:2]
+        return u+s
+
+    def clock2():
+        """clock2() -> (t_user,t_system)
+
+        Similar to clock(), but return a tuple of user/system times."""
+        return resource.getrusage(resource.RUSAGE_SELF)[:2]
+except ImportError:
+    # There is no distinction of user/system time under windows, so we just use
+    # time.clock() for everything...
+    clocku = clocks = clock = time.clock
+    def clock2():
+        """Under windows, system CPU time can't be measured.
+
+        This just returns clock() and zero."""
+        return time.clock(),0.0
+
+    
+def timings_out(reps,func,*args,**kw):
+    """timings_out(reps,func,*args,**kw) -> (t_total,t_per_call,output)
+
+    Execute a function reps times, return a tuple with the elapsed total
+    CPU time in seconds, the time per call and the function's output.
+
+    Under Unix, the return value is the sum of user+system time consumed by
+    the process, computed via the resource module.  This prevents problems
+    related to the wraparound effect which the time.clock() function has.
+
+    Under Windows the return value is in wall clock seconds. See the
+    documentation for the time module for more details."""
+
+    reps = int(reps)
+    assert reps >=1, 'reps must be >= 1'
+    if reps==1:
+        start = clock()
+        out = func(*args,**kw)
+        tot_time = clock()-start
+    else:
+        rng = xrange(reps-1) # the last time is executed separately to store output
+        start = clock()
+        for dummy in rng: func(*args,**kw)
+        out = func(*args,**kw)  # one last time
+        tot_time = clock()-start
+    av_time = tot_time / reps
+    return tot_time,av_time,out
+
+
+def timings(reps,func,*args,**kw):
+    """timings(reps,func,*args,**kw) -> (t_total,t_per_call)
+
+    Execute a function reps times, return a tuple with the elapsed total CPU
+    time in seconds and the time per call. These are just the first two values
+    in timings_out()."""
+
+    return timings_out(reps,func,*args,**kw)[0:2]
+
+
+def timing(func,*args,**kw):
+    """timing(func,*args,**kw) -> t_total
+
+    Execute a function once, return the elapsed total CPU time in
+    seconds. This is just the first value in timings_out()."""
+
+    return timings_out(1,func,*args,**kw)[0]
+

IPython/utils/warn.py

@@ -0,0 +1,66 b''
+# encoding: utf-8
+"""
+Utilities for warnings.  Shoudn't we just use the built in warnings module.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+
+from IPython.utils.io import Term
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def warn(msg,level=2,exit_val=1):
+    """Standard warning printer. Gives formatting consistency.
+
+    Output is sent to Term.cerr (sys.stderr by default).
+
+    Options:
+
+    -level(2): allows finer control:
+      0 -> Do nothing, dummy function.
+      1 -> Print message.
+      2 -> Print 'WARNING:' + message. (Default level).
+      3 -> Print 'ERROR:' + message.
+      4 -> Print 'FATAL ERROR:' + message and trigger a sys.exit(exit_val).
+
+    -exit_val (1): exit value returned by sys.exit() for a level 4
+    warning. Ignored for all other levels."""
+
+    if level>0:
+        header = ['','','WARNING: ','ERROR: ','FATAL ERROR: ']
+        print >> Term.cerr, '%s%s' % (header[level],msg)
+        if level == 4:
+            print >> Term.cerr,'Exiting.\n'
+            sys.exit(exit_val)
+
+            
+def info(msg):
+    """Equivalent to warn(msg,level=1)."""
+
+    warn(msg,level=1)
+
+    
+def error(msg):
+    """Equivalent to warn(msg,level=3)."""
+
+    warn(msg,level=3)
+
+    
+def fatal(msg,exit_val=1):
+    """Equivalent to warn(msg,exit_val=exit_val,level=4)."""
+
+    warn(msg,exit_val=exit_val,level=4)
+

IPython/__init__.py

@@ -30,14 +30,14 b" if sys.version[0:3] < '2.5':"
 
 
 # Make it easy to import extensions - they are always directly on pythonpath.
-# Therefore, non-IPython modules can be added to extensions directory
+# Therefore, non-IPython modules can be added to extensions directory.
+# This should probably be in ipapp.py.
 sys.path.append(os.path.join(os.path.dirname(__file__), "extensions"))
 
 #-----------------------------------------------------------------------------
 # Setup the top level names
 #-----------------------------------------------------------------------------
 
-# In some cases, these are causing circular imports.
 from .config.loader import Config
 from .core import release
 from .core.application import Application

IPython/config/loader.py

@@ -23,7 +23,7 b' import os'
 import sys
 
 from IPython.external import argparse
-from IPython.utils.genutils import filefind
+from IPython.utils.path import filefind
 
 #-----------------------------------------------------------------------------
 # Exceptions

IPython/core/alias.py

@@ -28,9 +28,9 b' import sys'
 from IPython.core.component import Component
 from IPython.core.splitinput import split_user_input
 
-from IPython.utils.traitlets import CBool, List, Instance
-from IPython.utils.genutils import error
+from IPython.utils.traitlets import List
 from IPython.utils.autoattr import auto_attr
+from IPython.utils.warn import warn, error
 
 #-----------------------------------------------------------------------------
 # Utilities

IPython/core/application.py

@@ -33,7 +33,7 b' import os'
 import sys
 
 from IPython.core import release, crashhandler
-from IPython.utils.genutils import get_ipython_dir, get_ipython_package_dir
+from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
 from IPython.config.loader import (
     PyFileConfigLoader,
     ArgParseConfigLoader,

IPython/core/completer.py

@@ -69,19 +69,20 b' used, and this module (and the readline module) are silently inactive.'
 import __builtin__
 import __main__
 import glob
+import inspect
 import itertools
 import keyword
 import os
 import re
 import shlex
 import sys
-import types
 
-import IPython.utils.rlineimpl as readline
 from IPython.core.error import TryNext
 from IPython.core.prefilter import ESC_MAGIC
 from IPython.utils import generics
-from IPython.utils.genutils import debugx, dir2
+from IPython.utils.frame import debugx
+from IPython.utils.dir2 import dir2
+import IPython.utils.rlineimpl as readline
 
 #-----------------------------------------------------------------------------
 # Globals
@@ -216,7 +217,6 b' class Completer:'
         with a __getattr__ hook is evaluated.
 
         """
-        import re
 
         #print 'Completer->attr_matches, txt=%r' % text # dbg
         # Another option, seems to work great. Catches things like ''.<tab>

IPython/core/component.py

@@ -27,7 +27,7 b' from weakref import WeakValueDictionary'
 from IPython.utils.importstring import import_item
 from IPython.config.loader import Config
 from IPython.utils.traitlets import (
-    HasTraitlets, TraitletError, MetaHasTraitlets, Instance, This
+    HasTraitlets, MetaHasTraitlets, Instance, This
 )
 
 

IPython/core/crashhandler.py

@@ -26,7 +26,7 b' from pprint import pformat'
 # Our own
 from IPython.core import release
 from IPython.core import ultratb
-from IPython.utils.genutils import sys_info
+from IPython.utils.sysinfo import sys_info
 
 from IPython.external.Itpl import itpl
 

IPython/core/debugger.py

@@ -26,15 +26,13 b' http://www.python.org/2.2.3/license.html"""'
 #*****************************************************************************
 
 import bdb
-import cmd
 import linecache
-import os
 import sys
 
 from IPython.utils import PyColorize
 from IPython.core import ipapi
 from IPython.utils import coloransi
-from IPython.utils.genutils import Term
+from IPython.utils.io import Term
 from IPython.core.excolors import exception_colors
 
 # See if we can use pydb.

IPython/core/display_trap.py

@@ -24,8 +24,6 b' import sys'
 
 from IPython.core.component import Component
 
-from IPython.utils.autoattr import auto_attr
-
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------

IPython/core/embed.py

@@ -24,6 +24,7 b' Notes'
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
+import __main__
 
 import sys
 from contextlib import nested
@@ -33,7 +34,7 b' from IPython.core.iplib import InteractiveShell'
 from IPython.core.ipapp import load_default_config
 
 from IPython.utils.traitlets import Bool, Str, CBool
-from IPython.utils.genutils import ask_yes_no
+from IPython.utils.io import ask_yes_no
 
 
 #-----------------------------------------------------------------------------

IPython/core/excolors.py

@@ -10,8 +10,6 b' Color schemes for exception handling code in IPython.'
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
-#****************************************************************************
-# Required modules
 from IPython.utils.coloransi import ColorSchemeTable, TermColors, ColorScheme
 
 def exception_colors():

IPython/core/history.py

@@ -5,7 +5,8 b''
 import fnmatch
 import os
 
-from IPython.utils.genutils import Term, ask_yes_no, warn
+from IPython.utils.io import Term, ask_yes_no
+from IPython.utils.warn import warn
 from IPython.core import ipapi
 
 def magic_history(self, parameter_s = ''):

IPython/core/hooks.py

@@ -43,9 +43,12 b' somewhere in your configuration files or ipython command line.'
 
 import os, bisect
 import sys
-from IPython.utils.genutils import Term, shell
+
 from pprint import PrettyPrinter
 
+from IPython.utils.io import Term
+from IPython.utils.process import shell
+
 from IPython.core.error import TryNext
 
 # List here all the default hooks.  For now it's just the editor functions

IPython/core/ipapi.py

@@ -18,8 +18,6 b' has been made into a component, this module will be sent to deathrow.'
 # Imports
 #-----------------------------------------------------------------------------
 
-from IPython.core.error import TryNext, UsageError, IPythonCoreError
-
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------

IPython/core/ipapp.py

@@ -36,7 +36,7 b' from IPython.config.loader import ('
 #    NoConfigDefault,
 )
 from IPython.lib import inputhook
-from IPython.utils.genutils import filefind, get_ipython_dir
+from IPython.utils.path import filefind, get_ipython_dir
 from . import usage
 
 #-----------------------------------------------------------------------------
@@ -499,7 +499,6 b' class IPythonApp(Application):'
         self._run_exec_lines()
         self._run_exec_files()
         self._run_cmd_line_code()
-        self._configure_xmode()
 
     def _enable_gui_pylab(self):
         """Enable GUI event loop integration, taking pylab into account."""
@@ -624,11 +623,6 b' class IPythonApp(Application):'
                 self.log.warn("Error in executing file in user namespace: %s" % fname)
                 self.shell.showtraceback()
 
-    def _configure_xmode(self):
-        # XXX - shouldn't this be read from the config?  I'm still a little
-        # lost with all the details of handling the new config guys...
-        self.shell.InteractiveTB.set_mode(mode=self.shell.xmode)
-        
     def start_app(self):
         if self.master_config.Global.interact:
             self.log.debug("Starting IPython's mainloop...")

IPython/core/iplib.py

@@ -20,7 +20,6 b' from __future__ import with_statement'
 from __future__ import absolute_import
 
 import __builtin__
-import StringIO
 import bdb
 import codeop
 import exceptions
@@ -47,29 +46,35 b' from IPython.core.logger import Logger'
 from IPython.core.magic import Magic
 from IPython.core.prefilter import PrefilterManager
 from IPython.core.prompts import CachedOutput
-from IPython.core.pylabtools import pylab_activate
 from IPython.core.usage import interactive_usage, default_banner
+import IPython.core.hooks
 from IPython.external.Itpl import ItplNS
 from IPython.lib.inputhook import enable_gui
 from IPython.lib.backgroundjobs import BackgroundJobManager
+from IPython.lib.pylabtools import pylab_activate
 from IPython.utils import PyColorize
 from IPython.utils import pickleshare
-from IPython.utils.genutils import get_ipython_dir
+from IPython.utils.doctestreload import doctest_reload
 from IPython.utils.ipstruct import Struct
-from IPython.utils.platutils import toggle_set_term_title, set_term_title
+from IPython.utils.io import Term, ask_yes_no
+from IPython.utils.path import get_home_dir, get_ipython_dir, HomeDirError
+from IPython.utils.process import (
+    abbrev_cwd,
+    getoutput,
+    getoutputerror
+)
+# import IPython.utils.rlineimpl as readline
 from IPython.utils.strdispatch import StrDispatch
 from IPython.utils.syspathcontext import prepended_to_syspath
-
-# XXX - need to clean up this import * line
-from IPython.utils.genutils import *
-
-# from IPython.utils import growl
-# growl.start("IPython")
-
+from IPython.utils.terminal import toggle_set_term_title, set_term_title
+from IPython.utils.warn import warn, error, fatal
 from IPython.utils.traitlets import (
     Int, Str, CBool, CaselessStrEnum, Enum, List, Unicode
 )
 
+# from IPython.utils import growl
+# growl.start("IPython")
+
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
@@ -658,7 +663,6 b' class InteractiveShell(Component, Magic):'
         self.strdispatchers = {}
 
         # Set all default hooks, defined in the IPython.hooks module.
-        import IPython.core.hooks
         hooks = IPython.core.hooks
         for hook_name in hooks.__all__:
             # default hooks have priority 100, i.e. low; user hooks should have
@@ -1164,7 +1168,9 b' class InteractiveShell(Component, Magic):'
         Convert func into callable that saves & restores
         history around the call """
 
-        if not self.has_readline:
+        if self.has_readline:
+            from IPython.utils import rlineimpl as readline
+        else:
             return func
 
         def wrapper():
@@ -1198,6 +1204,9 b' class InteractiveShell(Component, Magic):'
         # and add any custom exception handlers the user may have specified
         self.set_custom_exc(*custom_exceptions)
 
+        # Set the exception mode
+        self.InteractiveTB.set_mode(mode=self.xmode)
+
     def set_custom_exc(self,exc_tuple,handler):
         """set_custom_exc(exc_tuple,handler)
 

IPython/core/macro.py

@@ -7,7 +7,7 b''
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
-from IPython.utils.genutils import Term
+from IPython.utils.io import Term
 from IPython.core.autocall import IPyAutocall
 
 class Macro(IPyAutocall):

IPython/core/magic.py

@@ -1,35 +1,33 b''
-# -*- coding: utf-8 -*-
+# encoding: utf-8
 """Magic functions for InteractiveShell.
 """
 
-#*****************************************************************************
-#       Copyright (C) 2001 Janko Hauser <[e-mail]> and
-#       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
-#
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2001 Janko Hauser <[e-mail]> and
+#  Copyright (C) 2001-2007 Fernando Perez <[e-mail]>
+#  Copyright (C) 2008-2009  The IPython Development Team
+
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#*****************************************************************************
+#-----------------------------------------------------------------------------
 
-#****************************************************************************
-# Modules and globals
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
 
-# Python standard modules
 import __builtin__
 import bdb
 import inspect
 import os
-import pdb
-import pydoc
 import sys
 import shutil
 import re
-import tempfile
 import time
-import cPickle as pickle
 import textwrap
+import types
 from cStringIO import StringIO
 from getopt import getopt,GetoptError
-from pprint import pprint, pformat
+from pprint import pformat
 
 # cProfile was added in Python2.5
 try:
@@ -42,10 +40,7 b' except ImportError:'
     except ImportError:
         profile = pstats = None
 
-# Homebrewed
 import IPython
-import IPython.utils.generics
-
 from IPython.core import debugger, oinspect
 from IPython.core.error import TryNext
 from IPython.core.error import UsageError
@@ -53,20 +48,24 b' from IPython.core.fakemodule import FakeModule'
 from IPython.core.macro import Macro
 from IPython.core.page import page
 from IPython.core.prefilter import ESC_MAGIC
-from IPython.core.pylabtools import mpl_runner
+from IPython.lib.pylabtools import mpl_runner
 from IPython.lib.inputhook import enable_gui
-from IPython.external.Itpl import Itpl, itpl, printpl,itplns
+from IPython.external.Itpl import itpl, printpl
 from IPython.testing import decorators as testdec
-from IPython.utils import platutils
-from IPython.utils import wildcard
-from IPython.utils.PyColorize import Parser
+from IPython.utils.io import Term, file_read, nlprint
+from IPython.utils.path import get_py_filename
+from IPython.utils.process import arg_split, abbrev_cwd
+from IPython.utils.terminal import set_term_title
+from IPython.utils.text import LSString, SList, StringTypes
+from IPython.utils.timing import clock, clock2
+from IPython.utils.warn import warn, error
 from IPython.utils.ipstruct import Struct
+import IPython.utils.generics
 
-# XXX - We need to switch to explicit imports here with genutils
-from IPython.utils.genutils import *
-
-#***************************************************************************
+#-----------------------------------------------------------------------------
 # Utility functions
+#-----------------------------------------------------------------------------
+
 def on_off(tag):
     """Return an ON/OFF string for a 1/0 input. Simple utility function."""
     return ['OFF','ON'][tag]
@@ -94,6 +93,9 b' def compress_dhist(dh):'
 # on construction of the main InteractiveShell object.  Something odd is going
 # on with super() calls, Component and the MRO... For now leave it as-is, but
 # eventually this needs to be clarified.
+# BG: This is because InteractiveShell inherits from this, but is itself a 
+# Component. This messes up the MRO in some way. The fix is that we need to
+# make Magic a component that InteractiveShell does not subclass.
 
 class Magic:
     """Magic functions for InteractiveShell.
@@ -277,7 +279,7 b' python-profiler package from non-free.""")'
     def arg_err(self,func):
         """Print docstring if incorrect arguments were passed"""
         print 'Error in arguments:'
-        print OInspect.getdoc(func)
+        print oinspect.getdoc(func)
 
     def format_latex(self,strng):
         """Format a string for latex inclusion."""
@@ -1170,7 +1172,7 b' Currently the magic system has the following functions:\\n"""'
             started  = logger.logstart(logfname,loghead,logmode,
                                        log_output,timestamp,log_raw_input)
         except:
-            rc.opts.logfile = old_logfile
+            self.shell.logfile = old_logfile
             warn("Couldn't start log: %s" % sys.exc_info()[1])
         else:
             # log input history up to this point, optionally interleaving
@@ -2811,7 +2813,7 b' Defaulting color scheme to \'NoColor\'"""'
             try:                
                 os.chdir(os.path.expanduser(ps))
                 if self.shell.term_title:
-                    platutils.set_term_title('IPython: ' + abbrev_cwd())
+                    set_term_title('IPython: ' + abbrev_cwd())
             except OSError:
                 print sys.exc_info()[1]
             else:
@@ -2824,7 +2826,7 b' Defaulting color scheme to \'NoColor\'"""'
         else:
             os.chdir(self.shell.home_dir)
             if self.shell.term_title:
-                platutils.set_term_title('IPython: ' + '~')
+                set_term_title('IPython: ' + '~')
             cwd = os.getcwd()
             dhist = self.shell.user_ns['_dh']
             

IPython/core/oinspect.py

@@ -27,10 +27,11 b' import sys'
 import types
 
 # IPython's own
-from IPython.utils import PyColorize
-from IPython.utils.genutils import indent, Term
 from IPython.core.page import page
 from IPython.external.Itpl import itpl
+from IPython.utils import PyColorize
+from IPython.utils.io import Term
+from IPython.utils.text import indent
 from IPython.utils.wildcard import list_namespace
 from IPython.utils.coloransi import *
 

IPython/core/page.py

@@ -30,15 +30,15 b' rid of that dependency, we could move it there.'
 import os
 import re
 import sys
+import tempfile
 
 from IPython.core import ipapi
 from IPython.core.error import TryNext
-from IPython.utils.genutils import (
-    chop, Term, USE_CURSES
-)
-
-if os.name == "nt":
-    from IPython.utils.winconsole import get_console_size
+from IPython.utils.cursesimport import use_curses
+from IPython.utils.data import chop
+from IPython.utils.io import Term
+from IPython.utils.process import xsys
+from IPython.utils.terminal import get_terminal_size
 
 
 #-----------------------------------------------------------------------------
@@ -47,7 +47,7 b' if os.name == "nt":'
 
 esc_re = re.compile(r"(\x1b[^m]+m)")
 
-def page_dumb(strng,start=0,screen_lines=25):
+def page_dumb(strng, start=0, screen_lines=25):
     """Very dumb 'pager' in Python, for when nothing else works.
 
     Only moves forward, same interface as page(), except for pager_cmd and
@@ -69,8 +69,8 b' def page_dumb(strng,start=0,screen_lines=25):'
                 last_escape = esc_list[-1]
         print >>Term.cout, last_escape + os.linesep.join(screens[-1])
 
-#----------------------------------------------------------------------------
-def page(strng,start=0,screen_lines=0,pager_cmd = None):
+
+def page(strng, start=0, screen_lines=0, pager_cmd=None):
     """Print a string, piping through a pager after a certain length.
 
     The screen_lines parameter specifies the number of *usable* lines of your
@@ -93,7 +93,7 b' def page(strng,start=0,screen_lines=0,pager_cmd = None):'
 
     # Some routines may auto-compute start offsets incorrectly and pass a
     # negative value.  Offset to 0 for robustness.
-    start = max(0,start)
+    start = max(0, start)
 
     # first, try the hook
     ip = ipapi.get()
@@ -120,19 +120,16 b' def page(strng,start=0,screen_lines=0,pager_cmd = None):'
     # terminals. If someone later feels like refining it, it's not hard.
     numlines = max(num_newlines,int(len_str/80)+1)
 
-    if os.name == "nt":
-        screen_lines_def = get_console_size(defaulty=25)[1]
-    else:
-        screen_lines_def = 25 # default value if we can't auto-determine
+    screen_lines_def = get_terminal_size()[1]
 
     # auto-determine screen size
     if screen_lines <= 0:
         if TERM=='xterm' or TERM=='xterm-color':
-            use_curses = USE_CURSES
+            local_use_curses = use_curses
         else:
             # curses causes problems on many terminals other than xterm.
-            use_curses = False
-        if use_curses:
+            local_use_curses = False
+        if local_use_curses:
             import termios
             import curses
             # There is a bug in curses, where *sometimes* it fails to properly
@@ -201,8 +198,8 b' def page(strng,start=0,screen_lines=0,pager_cmd = None):'
         if retval is not None:
             page_dumb(strng,screen_lines=screen_lines)
 
-#----------------------------------------------------------------------------
-def page_file(fname,start = 0, pager_cmd = None):
+
+def page_file(fname, start=0, pager_cmd=None):
     """Page a file, using an optional pager command and starting line.
     """
 
@@ -221,12 +218,12 b' def page_file(fname,start = 0, pager_cmd = None):'
         except:
             print 'Unable to show file',`fname`
 
-#----------------------------------------------------------------------------
-def get_pager_cmd(pager_cmd = None):
-    """Return a pager command.
 
-    Makes some attempts at finding an OS-correct one."""
+def get_pager_cmd(pager_cmd=None):
+    """Return a pager command.
 
+    Makes some attempts at finding an OS-correct one.
+    """
     if os.name == 'posix':
         default_pager_cmd = 'less -r'  # -r for color control sequences
     elif os.name in ['nt','dos']:
@@ -239,8 +236,8 b' def get_pager_cmd(pager_cmd = None):'
             pager_cmd = default_pager_cmd
     return pager_cmd
 
-#-----------------------------------------------------------------------------
-def get_pager_start(pager,start):
+
+def get_pager_start(pager, start):
     """Return the string for paging files with an offset.
 
     This is the '+N' argument which less and more (under Unix) accept.
@@ -255,8 +252,8 b' def get_pager_start(pager,start):'
         start_string = ''
     return start_string
 
-#----------------------------------------------------------------------------
-# (X)emacs on W32 doesn't like to be bypassed with msvcrt.getch()
+
+# (X)emacs on win32 doesn't like to be bypassed with msvcrt.getch()
 if os.name == 'nt' and os.environ.get('TERM','dumb') != 'emacs':
     import msvcrt
     def page_more():
@@ -280,7 +277,7 b' else:'
         else:
             return True
 
-#----------------------------------------------------------------------------
+
 def snip_print(str,width = 75,print_full = 0,header = ''):
     """Print a string snipping the midsection to fit in width.
 
@@ -305,4 +302,5 b" def snip_print(str,width = 75,print_full = 0,header = ''):"
     if snip and print_full == 2:
         if raw_input(header+' Snipped. View (y/n)? [N]').lower() == 'y':
             page(str)
-    return snip
  No newline at end of file
+    return snip
+

IPython/core/prefilter.py

@@ -27,10 +27,7 b' Authors:'
 
 import __builtin__
 import codeop
-import keyword
-import os
 import re
-import sys
 
 from IPython.core.alias import AliasManager
 from IPython.core.autocall import IPyAutocall
@@ -39,7 +36,8 b' from IPython.core.splitinput import split_user_input'
 from IPython.core.page import page
 
 from IPython.utils.traitlets import List, Int, Any, Str, CBool, Bool
-from IPython.utils.genutils import make_quoted_expr, Term
+from IPython.utils.io import Term
+from IPython.utils.text import make_quoted_expr
 from IPython.utils.autoattr import auto_attr
 
 #-----------------------------------------------------------------------------
@@ -158,11 +156,12 b' class LineInfo(object):'
         without worrying about *further* damaging state.
         """
         if not self._oinfo:
+            # ip.shell._ofind is actually on the Magic class!
             self._oinfo = ip.shell._ofind(self.ifun)
         return self._oinfo
 
     def __str__(self):                                                         
-        return "Lineinfo [%s|%s|%s]" %(self.pre,self.ifun,self.the_rest) 
+        return "Lineinfo [%s|%s|%s]" %(self.pre, self.ifun, self.the_rest) 
 
 
 #-----------------------------------------------------------------------------

IPython/core/prompts.py

@@ -12,23 +12,20 b' Classes for handling input/output prompts.'
 #*****************************************************************************
 
 #****************************************************************************
-# Required modules
+
 import __builtin__
 import os
+import re
 import socket
 import sys
-import time
 
-# IPython's own
-from IPython.utils import coloransi
 from IPython.core import release
 from IPython.external.Itpl import ItplNS
 from IPython.core.error import TryNext
-from IPython.utils.ipstruct import Struct
-from IPython.core.macro import Macro
+from IPython.utils import coloransi
 import IPython.utils.generics
-
-from IPython.utils.genutils import *
+from IPython.utils.warn import warn
+from IPython.utils.io import Term
 
 #****************************************************************************
 #Color schemes for Prompts.

IPython/core/quitter.py

@@ -19,7 +19,11 b' Authors'
 # Imports
 #-----------------------------------------------------------------------------
 
-import sys
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
 
 class Quitter(object):
     """Simple class to handle exit, similar to Python 2.5's.
@@ -40,4 +44,4 b' class Quitter(object):'
     # Repr MUST return a string, else display like pprint hooks get confused
     def __repr__(self):
         self.shell.ask_exit()
-        return 'Bye.'
+        return ''

IPython/core/tests/test_magic.py

@@ -8,19 +8,15 b' from __future__ import absolute_import'
 # Imports
 #-----------------------------------------------------------------------------
 
-# stdlib
 import os
 import sys
 import tempfile
 import types
 from cStringIO import StringIO
 
-# third-party
 import nose.tools as nt
 
-# our own
-from IPython.utils import genutils
-from IPython.utils.platutils import find_cmd, get_long_path_name
+from IPython.utils.path import get_long_path_name
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 

IPython/core/tests/test_run.py

@@ -12,17 +12,12 b' from __future__ import absolute_import'
 # Imports
 #-----------------------------------------------------------------------------
 
-# stdlib
 import os
 import sys
 import tempfile
 
-# third-party
 import nose.tools as nt
 
-# our own
-from IPython.utils.platutils import find_cmd
-from IPython.utils import genutils
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 
@@ -142,10 +137,10 b' class TestMagicRunSimple(tt.TempFileMixin):'
         _ip.runlines('t = isinstance(f(), foo)')
         nt.assert_true(_ip.user_ns['t'])
 
-    # We have to skip these in win32 because genutils.getoutputerr() crashes,
+    # We have to skip these in win32 because getoutputerr() crashes,
     # due to the fact that subprocess does not support close_fds when
     # redirecting stdout/err.  So unless someone who knows more tells us how to
-    # implement genutils.getoutputerr() in win32, we're stuck avoiding these.
+    # implement getoutputerr() in win32, we're stuck avoiding these.
     @dec.skip_win32
     def test_obj_del(self):
         """Test that object's __del__ methods are called on exit."""

IPython/core/ultratb.py

@@ -93,9 +93,10 b' from inspect import getsourcefile, getfile, getmodule,\\'
 from IPython.utils import PyColorize
 from IPython.core import debugger, ipapi
 from IPython.core.display_trap import DisplayTrap
-from IPython.utils.ipstruct import Struct
 from IPython.core.excolors import exception_colors
-from IPython.utils.genutils import Term, uniq_stable, error, info
+from IPython.utils.data import uniq_stable
+from IPython.utils.io import Term
+from IPython.utils.warn import info, error
 
 # Globals
 # amount of space to put line numbers before verbose tracebacks

IPython/deathrow/GnuplotRuntime.py

@@ -53,7 +53,7 b" __all__ = ['Gnuplot','gp','gp_new','Data','File','Func','GridData',"
            'pm3d_config','eps_fix_bbox']
 
 import os,tempfile,sys
-from IPython.utils.genutils import getoutput
+from IPython.utils.process import getoutput
 
 #---------------------------------------------------------------------------
 # Notes on mouse support for Gnuplot.py

IPython/deathrow/ipipe.py

@@ -133,10 +133,10 b' from IPython.external import simplegeneric'
 from IPython.external import path
 
 try:
-    from IPython.utils import genutils
+    from IPython.utils.io import Term
     from IPython.utils import generics
 except ImportError:
-    genutils = None
+    Term = None
     generics = None
 
 from IPython.core import ipapi
@@ -2168,7 +2168,7 b' class idump(Display):'
         self.datasepchar = "|"
 
     def display(self):
-        stream = genutils.Term.cout
+        stream = Term.cout
         allattrs = []
         attrset = set()
         colwidths = {}

IPython/deathrow/ipy_traits_completer.py

@@ -54,7 +54,7 b' from enthought.traits import api as T'
 # IPython imports
 from IPython.core.error import TryNext
 from IPython.core.ipapi import get as ipget
-from IPython.utils.genutils import dir2
+from IPython.utils.dir2 import dir2
 try:
     set
 except:

IPython/deathrow/twshell.py

@@ -14,7 +14,9 b' from IPython.core.iplib import InteractiveShell'
 from IPython.utils.ipstruct import Struct
 import Queue,thread,threading,signal
 from signal import signal, SIGINT
-from IPython.utils.genutils import Term,warn,error,flag_calls, ask_yes_no
+from IPython.utils.io import Term, ask_yes_no
+from IPython.utils.warn import warn, error
+from IPython.utils.decorators import flag_calls
 from IPython.core import shellglobals
 
 def install_gtk2():

IPython/extensions/pretty.py

@@ -39,7 +39,7 b' from IPython.core.error import TryNext'
 from IPython.external import pretty
 from IPython.core.component import Component
 from IPython.utils.traitlets import Bool, List
-from IPython.utils.genutils import Term
+from IPython.utils.io import Term
 from IPython.utils.autoattr import auto_attr
 from IPython.utils.importstring import import_item
 

IPython/external/mglob.py

@@ -213,7 +213,7 b' def main():'
     print "\n".join(expand(sys.argv[1:])),
 
 def mglob_f(self, arg):
-    from IPython.utils.genutils import SList
+    from IPython.utils.text import SList
     if arg.strip():
         return SList(expand(arg))
     print "Please specify pattern!"

IPython/frontend/prefilterfrontend.py

@@ -31,7 +31,7 b' from IPython.kernel.core.redirector_output_trap import RedirectorOutputTrap'
 
 from IPython.kernel.core.sync_traceback_trap import SyncTracebackTrap
 
-from IPython.utils.genutils import Term
+from IPython.utils.io import Term
 
 from linefrontendbase import LineFrontEndBase, common_prefix
 

IPython/gui/wx/ipshell_nonblocking.py

@@ -25,7 +25,7 b' from thread_ex import ThreadEx'
 
 import IPython
 from IPython.core import iplib, ipapp
-from IPython.utils import genutils
+from IPython.utils.io import Term
 
 ##############################################################################
 class _Helper(object):
@@ -141,11 +141,11 b' class NonBlockingIPShell(object):'
         #only one instance can be instanciated else tehre will be
         #cin/cout/cerr clash...
         if cin:
-            genutils.Term.cin = cin
+            Term.cin = cin
         if cout:
-            genutils.Term.cout = cout
+            Term.cout = cout
         if cerr:
-            genutils.Term.cerr = cerr
+            Term.cerr = cerr
         
         excepthook = sys.excepthook
 
@@ -471,7 +471,7 b' class NonBlockingIPShell(object):'
         '''
 
         orig_stdout = sys.stdout
-        sys.stdout = genutils.Term.cout
+        sys.stdout = Term.cout
         #self.sys_displayhook_ori = sys.displayhook
         #sys.displayhook = self.displayhook
         

IPython/kernel/client.py

@@ -27,7 +27,6 b' The main classes in this module are:'
 # Imports
 #-----------------------------------------------------------------------------
 
-from cStringIO import StringIO
 import sys
 import warnings
 

IPython/kernel/clientconnector.py

@@ -33,7 +33,7 b' from IPython.kernel.twistedutil import ('
     sleep_deferred
 )
 from IPython.utils.importstring import import_item
-from IPython.utils.genutils import get_ipython_dir
+from IPython.utils.path import get_ipython_dir
 
 from twisted.internet import defer
 from twisted.internet.defer import inlineCallbacks, returnValue

IPython/kernel/clientinterfaces.py

@@ -15,7 +15,7 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
-from zope.interface import Interface, implements
+from zope.interface import Interface
 
 class IFCClientInterfaceProvider(Interface):
 

IPython/kernel/clusterdir.py

@@ -24,13 +24,14 b' import warnings'
 
 from twisted.python import log
 
-from IPython.core import release
 from IPython.config.loader import PyFileConfigLoader
 from IPython.core.application import Application
 from IPython.core.component import Component
-from IPython.utils.genutils import get_ipython_dir, get_ipython_package_dir
-from IPython.utils.traitlets import Unicode, Bool
-from IPython.utils import genutils
+from IPython.utils.path import (
+    get_ipython_package_dir,
+    expand_path
+)
+from IPython.utils.traitlets import Unicode
 
 #-----------------------------------------------------------------------------
 # Warnings control
@@ -225,7 +226,7 b' class ClusterDir(Component):'
             The path of the cluster directory.  This is expanded using
             :func:`IPython.utils.genutils.expand_path`.
         """
-        cluster_dir = genutils.expand_path(cluster_dir)
+        cluster_dir = expand_path(cluster_dir)
         if not os.path.isdir(cluster_dir):
             raise ClusterDirError('Cluster directory not found: %s' % cluster_dir)
         return ClusterDir(cluster_dir)
@@ -316,7 +317,7 b' class ApplicationWithClusterDir(Application):'
             cluster_dir = self.command_line_config.Global.cluster_dir
         except AttributeError:
             cluster_dir = self.default_config.Global.cluster_dir
-        cluster_dir = genutils.expand_path(cluster_dir)
+        cluster_dir = expand_path(cluster_dir)
         try:
             self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
         except ClusterDirError:
@@ -389,7 +390,7 b' class ApplicationWithClusterDir(Application):'
         pdir = self.cluster_dir_obj.pid_dir
         self.pid_dir = config.Global.pid_dir = pdir
         self.log.info("Cluster directory set to: %s" % self.cluster_dir)
-        config.Global.work_dir = unicode(genutils.expand_path(config.Global.work_dir))
+        config.Global.work_dir = unicode(expand_path(config.Global.work_dir))
         # Change to the working directory. We do this just before construct
         # is called so all the components there have the right working dir.
         self.to_work_dir()

IPython/kernel/contexts.py

@@ -24,9 +24,7 b' __docformat__ = "restructuredtext en"'
 import linecache
 import sys
 
-from twisted.internet.error import ConnectionRefusedError
-
-from IPython.core.ultratb import _fixed_getinnerframes, findsource
+from IPython.core.ultratb import findsource
 from IPython.core import ipapi
 
 from IPython.kernel import error

IPython/kernel/controllerservice.py

@@ -37,20 +37,18 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
-import os, sys
+import os
 
 from twisted.application import service
-from twisted.internet import defer, reactor
-from twisted.python import log, components
+from twisted.python import log
 from zope.interface import Interface, implements, Attribute
-import zope.interface as zi
 
 from IPython.kernel.engineservice import \
     IEngineCore, \
     IEngineSerialized, \
     IEngineQueued
     
-from IPython.utils.genutils import get_ipython_dir
+from IPython.utils.path import get_ipython_dir
 from IPython.kernel import codeutil
 
 #-------------------------------------------------------------------------------

IPython/kernel/core/prompts.py

@@ -21,6 +21,8 b' __docformat__ = "restructuredtext en"'
 
 # Required modules
 import __builtin__
+import os
+import re
 import socket
 import sys
 
@@ -30,7 +32,8 b' from IPython.external.Itpl import ItplNS'
 from IPython.utils import coloransi
 from IPython.core import release
 from IPython.core.error import TryNext
-from IPython.utils.genutils import *
+from IPython.utils.io import Term
+from IPython.utils.warn import warn
 import IPython.utils.generics
 
 #****************************************************************************
@@ -240,7 +243,7 b' class BasePrompt(object):'
         This must be called every time the color settings change, because the
         prompt_specials global may have changed."""
 
-        import os,time  # needed in locals for prompt string handling
+        import os, time  # needed in locals for prompt string handling
         loc = locals()
         self.p_str = ItplNS('%s%s%s' %
                             ('${self.sep}${self.col_p}',

IPython/kernel/engineconnector.py

@@ -17,8 +17,7 b''
 import os
 import cPickle as pickle
 
-from twisted.python import log, failure
-from twisted.internet import defer
+from twisted.python import log
 from twisted.internet.defer import inlineCallbacks, returnValue
 
 from IPython.kernel.fcutil import find_furl, validate_furl_or_file

IPython/kernel/enginefc.py

@@ -19,14 +19,11 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
-import os, time
 import cPickle as pickle
 
 from twisted.python import components, log, failure
-from twisted.python.failure import Failure
-from twisted.internet import defer, reactor, threads
-from twisted.internet.interfaces import IProtocolFactory
-from zope.interface import Interface, implements, Attribute
+from twisted.internet import defer, threads
+from zope.interface import Interface, implements
 
 from twisted.internet.base import DelayedCall
 DelayedCall.debug = True
@@ -35,24 +32,20 b' from foolscap import Referenceable, DeadReferenceError'
 from foolscap.referenceable import RemoteReference
 
 from IPython.kernel.pbutil import packageFailure, unpackageFailure
-from IPython.kernel.util import printer
-from IPython.kernel.twistedutil import gatherBoth
-from IPython.kernel import newserialized
-from IPython.kernel.error import  ProtocolError
-from IPython.kernel import controllerservice
 from IPython.kernel.controllerservice import IControllerBase
-from IPython.kernel.engineservice import \
-    IEngineBase, \
-    IEngineQueued, \
-    EngineService, \
+from IPython.kernel.engineservice import (
+    IEngineBase,
+    IEngineQueued,
     StrictDict
-from IPython.kernel.pickleutil import \
-    can, \
-    canDict, \
-    canSequence, \
-    uncan, \
-    uncanDict, \
+)
+from IPython.kernel.pickleutil import (
+    can,
+    canDict,
+    canSequence,
+    uncan,
+    uncanDict,
     uncanSequence
+)
 
 
 #-------------------------------------------------------------------------------

IPython/kernel/ipcontrollerapp.py

@@ -18,20 +18,18 b' The IPython controller application.'
 from __future__ import with_statement
 
 import copy
-import os
 import sys
 
 from twisted.application import service
 from twisted.internet import reactor
 from twisted.python import log
 
-from IPython.config.loader import Config, NoConfigDefault
-from IPython.core import release
+from IPython.config.loader import Config
 from IPython.core.application import Application
 from IPython.kernel import controllerservice
 from IPython.kernel.clusterdir import ApplicationWithClusterDir
 from IPython.kernel.fcutil import FCServiceFactory
-from IPython.utils.traitlets import Str, Instance, Unicode
+from IPython.utils.traitlets import Instance, Unicode
 
 #-----------------------------------------------------------------------------
 # Default interfaces

IPython/kernel/launcher.py

@@ -21,11 +21,15 b' import sys'
 
 from IPython.core.component import Component
 from IPython.external import Itpl
-from IPython.utils.traitlets import Str, Int, List, Unicode, Enum
-from IPython.utils.platutils import find_cmd
-from IPython.kernel.twistedutil import gatherBoth, make_deferred, sleep_deferred
+from IPython.utils.traitlets import Str, Int, List, Unicode
+from IPython.utils.path import get_ipython_module_path
+from IPython.utils.process import find_cmd, pycmd2argv
+from IPython.kernel.twistedutil import (
+    gatherBoth,
+    make_deferred,
+    sleep_deferred
+)
 from IPython.kernel.winhpcjob import (
-    WinHPCJob, WinHPCTask,
     IPControllerTask, IPEngineTask,
     IPControllerJob, IPEngineSetJob
 )
@@ -38,46 +42,23 b' from twisted.internet.error import ProcessDone, ProcessTerminated'
 from twisted.python import log
 from twisted.python.failure import Failure
 
+
 #-----------------------------------------------------------------------------
-# Utilities
+# Paths to the kernel apps
 #-----------------------------------------------------------------------------
 
 
-def find_controller_cmd():
-    """Find the command line ipcontroller program in a cross platform way."""
-    if sys.platform == 'win32':
-        # This logic is needed because the ipcontroller script doesn't
-        # always get installed in the same way or in the same location.
-        from IPython.kernel import ipcontrollerapp
-        script_location = ipcontrollerapp.__file__.replace('.pyc', '.py')
-        # The -u option here turns on unbuffered output, which is required
-        # on Win32 to prevent wierd conflict and problems with Twisted.
-        # Also, use sys.executable to make sure we are picking up the 
-        # right python exe.
-        cmd = [sys.executable, '-u', script_location]
-    else:
-        # ipcontroller has to be on the PATH in this case.
-        cmd = ['ipcontroller']
-    return cmd
-
-
-def find_engine_cmd():
-    """Find the command line ipengine program in a cross platform way."""
-    if sys.platform == 'win32':
-        # This logic is needed because the ipengine script doesn't
-        # always get installed in the same way or in the same location.
-        from IPython.kernel import ipengineapp
-        script_location = ipengineapp.__file__.replace('.pyc', '.py')
-        # The -u option here turns on unbuffered output, which is required
-        # on Win32 to prevent wierd conflict and problems with Twisted.
-        # Also, use sys.executable to make sure we are picking up the 
-        # right python exe.
-        cmd = [sys.executable, '-u', script_location]
-    else:
-        # ipcontroller has to be on the PATH in this case.
-        cmd = ['ipengine']
-    return cmd
+ipcluster_cmd_argv = pycmd2argv(get_ipython_module_path(
+    'IPython.kernel.ipclusterapp'
+))
 
+ipengine_cmd_argv = pycmd2argv(get_ipython_module_path(
+    'IPython.kernel.ipengineapp'
+))
+
+ipcontroller_cmd_argv = pycmd2argv(get_ipython_module_path(
+    'IPython.kernel.ipcontrollerapp'
+))
 
 #-----------------------------------------------------------------------------
 # Base launchers and errors
@@ -333,7 +314,7 b' class LocalProcessLauncher(BaseLauncher):'
 class LocalControllerLauncher(LocalProcessLauncher):
     """Launch a controller as a regular external process."""
 
-    controller_cmd = List(find_controller_cmd(), config=True)
+    controller_cmd = List(ipcontroller_cmd_argv, config=True)
     # Command line arguments to ipcontroller.
     controller_args = List(['--log-to-file','--log-level', '40'], config=True)
 
@@ -351,7 +332,7 b' class LocalControllerLauncher(LocalProcessLauncher):'
 class LocalEngineLauncher(LocalProcessLauncher):
     """Launch a single engine as a regular externall process."""
 
-    engine_cmd = List(find_engine_cmd(), config=True)
+    engine_cmd = List(ipengine_cmd_argv, config=True)
     # Command line arguments for ipengine.
     engine_args = List(
         ['--log-to-file','--log-level', '40'], config=True
@@ -462,7 +443,7 b' class MPIExecLauncher(LocalProcessLauncher):'
 class MPIExecControllerLauncher(MPIExecLauncher):
     """Launch a controller using mpiexec."""
 
-    controller_cmd = List(find_controller_cmd(), config=True)
+    controller_cmd = List(ipcontroller_cmd_argv, config=True)
     # Command line arguments to ipcontroller.
     controller_args = List(['--log-to-file','--log-level', '40'], config=True)
     n = Int(1, config=False)
@@ -481,7 +462,7 b' class MPIExecControllerLauncher(MPIExecLauncher):'
 
 class MPIExecEngineSetLauncher(MPIExecLauncher):
 
-    engine_cmd = List(find_engine_cmd(), config=True)
+    engine_cmd = List(ipengine_cmd_argv, config=True)
     # Command line arguments for ipengine.
     engine_args = List(
         ['--log-to-file','--log-level', '40'], config=True
@@ -831,28 +812,10 b' class PBSEngineSetLauncher(PBSLauncher):'
 #-----------------------------------------------------------------------------
 
 
-def find_ipcluster_cmd():
-    """Find the command line ipcluster program in a cross platform way."""
-    if sys.platform == 'win32':
-        # This logic is needed because the ipcluster script doesn't
-        # always get installed in the same way or in the same location.
-        from IPython.kernel import ipclusterapp
-        script_location = ipclusterapp.__file__.replace('.pyc', '.py')
-        # The -u option here turns on unbuffered output, which is required
-        # on Win32 to prevent wierd conflict and problems with Twisted.
-        # Also, use sys.executable to make sure we are picking up the 
-        # right python exe.
-        cmd = [sys.executable, '-u', script_location]
-    else:
-        # ipcontroller has to be on the PATH in this case.
-        cmd = ['ipcluster']
-    return cmd
-
-
 class IPClusterLauncher(LocalProcessLauncher):
     """Launch the ipcluster program in an external process."""
 
-    ipcluster_cmd = List(find_ipcluster_cmd(), config=True)
+    ipcluster_cmd = List(ipcluster_cmd_argv, config=True)
     # Command line arguments to pass to ipcluster.
     ipcluster_args = List(
         ['--clean-logs', '--log-to-file', '--log-level', '40'], config=True)

IPython/kernel/map.py

@@ -21,7 +21,7 b' __docformat__ = "restructuredtext en"'
 
 import types
 
-from IPython.utils.genutils import flatten as genutil_flatten
+from IPython.utils.data import flatten as utils_flatten
 
 #-------------------------------------------------------------------------------
 # Figure out which array packages are present and their array types
@@ -87,7 +87,7 b' class Map:'
                 return m['module'].concatenate(listOfPartitions)
         # Next try for Python sequence types
         if isinstance(testObject, (types.ListType, types.TupleType)):
-            return genutil_flatten(listOfPartitions)
+            return utils_flatten(listOfPartitions)
         # If we have scalars, just return listOfPartitions
         return listOfPartitions
 

IPython/kernel/mapper.py

@@ -18,8 +18,7 b' __docformat__ = "restructuredtext en"'
 from types import FunctionType
 from zope.interface import Interface, implements
 from IPython.kernel.task import MapTask
-from IPython.kernel.twistedutil import DeferredList, gatherBoth
-from IPython.kernel.util import printer
+from IPython.kernel.twistedutil import gatherBoth
 from IPython.kernel.error import collect_exceptions
 
 #----------------------------------------------------------------------------

IPython/kernel/multiengine.py

@@ -27,24 +27,17 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
-from new import instancemethod
-from types import FunctionType
-
-from twisted.application import service
 from twisted.internet import defer, reactor
 from twisted.python import log, components, failure
-from zope.interface import Interface, implements, Attribute
+from zope.interface import Interface, implements
 
-from IPython.utils import growl
-from IPython.kernel.util import printer
 from IPython.kernel.twistedutil import gatherBoth
-from IPython.kernel import map as Map
 from IPython.kernel import error
 from IPython.kernel.pendingdeferred import PendingDeferredManager, two_phase
-from IPython.kernel.controllerservice import \
-    ControllerAdapterBase, \
-    ControllerService, \
+from IPython.kernel.controllerservice import (
+    ControllerAdapterBase,
     IControllerBase
+)
 
 
 #-------------------------------------------------------------------------------

IPython/kernel/multienginefc.py

@@ -22,12 +22,11 b' from types import FunctionType'
 
 from zope.interface import Interface, implements
 from twisted.internet import defer
-from twisted.python import components, failure, log
+from twisted.python import components, failure
 
 from foolscap import Referenceable
 
 from IPython.kernel import error 
-from IPython.kernel.util import printer
 from IPython.kernel import map as Map
 from IPython.kernel.parallelfunction import ParallelFunction
 from IPython.kernel.mapper import (
@@ -36,14 +35,15 b' from IPython.kernel.mapper import ('
     IMapper
 )
 from IPython.kernel.twistedutil import gatherBoth
-from IPython.kernel.multiengine import (MultiEngine,
+from IPython.kernel.multiengine import (
     IMultiEngine,
     IFullSynchronousMultiEngine,
     ISynchronousMultiEngine)
-from IPython.kernel.multiengineclient import wrapResultList
 from IPython.kernel.pendingdeferred import PendingDeferredManager
-from IPython.kernel.pickleutil import (can, canDict,
-    canSequence, uncan, uncanDict, uncanSequence)
+from IPython.kernel.pickleutil import (
+    canDict,
+    canSequence, uncanDict, uncanSequence
+)
 
 from IPython.kernel.clientinterfaces import (
     IFCClientInterfaceProvider, 

IPython/kernel/pbutil.py

@@ -19,7 +19,6 b' import cPickle as pickle'
 
 from twisted.python.failure import Failure
 from twisted.python import failure
-import threading, sys
 
 from IPython.kernel import pbconfig
 from IPython.kernel.error import PBMessageSizeError, UnpickleableException
@@ -58,7 +57,7 b' def unpackageFailure(r):'
                 result = pickle.loads(r[8:])
             except pickle.PickleError:
                 return failure.Failure( \
-                    FailureUnpickleable("Could not unpickle failure."))
+                    UnpickleableException("Could not unpickle failure."))
             else:
                 return result
     return r

IPython/kernel/pendingdeferred.py

@@ -22,15 +22,11 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
-from twisted.application import service
-from twisted.internet import defer, reactor
-from twisted.python import log, components, failure
-from zope.interface import Interface, implements, Attribute
+from twisted.internet import defer
+from twisted.python import failure
 
-from IPython.kernel.twistedutil import gatherBoth
 from IPython.kernel import error
 from IPython.external import guid
-from IPython.utils import growl
 
 class PendingDeferredManager(object):
     """A class to track pending deferreds.

IPython/kernel/pickleutil.py

@@ -16,7 +16,6 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 
 from types import FunctionType
-from twisted.python import log
 
 class CannedObject(object):
     pass

IPython/kernel/task.py

@@ -19,19 +19,18 b' __docformat__ = "restructuredtext en"'
 # Tell nose to skip the testing of this module
 __test__ = {}
 
-import copy, time
+import time
 from types import FunctionType
 
-import zope.interface as zi, string
+import zope.interface as zi
 from twisted.internet import defer, reactor
 from twisted.python import components, log, failure
 
-from IPython.kernel.util import printer
 from IPython.kernel import engineservice as es, error
 from IPython.kernel import controllerservice as cs
-from IPython.kernel.twistedutil import gatherBoth, DeferredList
+from IPython.kernel.twistedutil import DeferredList
 
-from IPython.kernel.pickleutil import can, uncan, CannedFunction
+from IPython.kernel.pickleutil import can, uncan
 
 #-----------------------------------------------------------------------------
 # Definition of the Task objects

IPython/kernel/taskclient.py

@@ -19,10 +19,10 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 
 from zope.interface import Interface, implements
-from twisted.python import components, log
+from twisted.python import components
 
 from IPython.kernel.twistedutil import blockingCallFromThread
-from IPython.kernel import task, error
+from IPython.kernel import task
 from IPython.kernel.mapper import (
     SynchronousTaskMapper,
     ITaskMapperFactory,

IPython/kernel/taskfc.py

@@ -19,17 +19,14 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 
 import cPickle as pickle
-import xmlrpclib, copy
 
 from zope.interface import Interface, implements
 from twisted.internet import defer
-from twisted.python import components, failure
+from twisted.python import components
 
 from foolscap import Referenceable
 
-from IPython.kernel.twistedutil import blockingCallFromThread
-from IPython.kernel import error, task as taskmodule, taskclient
-from IPython.kernel.pickleutil import can, uncan
+from IPython.kernel import task as taskmodule
 from IPython.kernel.clientinterfaces import (
     IFCClientInterfaceProvider, 
     IBlockingClientAdaptor

IPython/kernel/twistedutil.py

@@ -15,7 +15,7 b''
 #-----------------------------------------------------------------------------
 
 import os, sys
-import threading, Queue, atexit
+import threading, Queue
 
 import twisted
 from twisted.internet import defer, reactor

IPython/kernel/winhpcjob.py

@@ -23,12 +23,10 b' import re'
 import uuid
 
 from xml.etree import ElementTree as ET
-from xml.dom import minidom
 
 from IPython.core.component import Component
-from IPython.external import Itpl
 from IPython.utils.traitlets import (
-    Str, Int, List, Unicode, Instance,
+    Str, Int, List, Instance,
     Enum, Bool, CStr
 )
 

IPython/lib/backgroundjobs.py

@@ -31,7 +31,7 b' import sys'
 import threading
 
 from IPython.core.ultratb import AutoFormattedTB
-from IPython.utils.genutils import warn,error
+from IPython.utils.warn import warn, error
 
 class BackgroundJobManager:
     """Class to manage a pool of backgrounded threaded jobs.

IPython/lib/demo.py

@@ -176,7 +176,8 b' import shlex'
 import sys
 
 from IPython.utils.PyColorize import Parser
-from IPython.utils.genutils import marquee, file_read, file_readlines, Term
+from IPython.utils.io import file_read, file_readlines, Term
+from IPython.utils.text import marquee
 
 __all__ = ['Demo','IPythonDemo','LineDemo','IPythonLineDemo','DemoError']
 
@@ -543,7 +544,7 b' class ClearMixin(object):'
         """Method called before executing each block.
         
         This one simply clears the screen."""
-        from IPython.utils.platutils import term_clear
+        from IPython.utils.terminal import term_clear
         term_clear()
 
 class ClearDemo(ClearMixin,Demo):

IPython/lib/pylabtools.py

@@ -12,10 +12,12 b' Fernando Perez.'
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
+
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
-from IPython.utils.genutils import flag_calls
+
+from IPython.utils.decorators import flag_calls
 
 #-----------------------------------------------------------------------------
 # Main classes and functions

IPython/quarantine/InterpreterExec.py

@@ -50,7 +50,7 b' del InteractiveShell,prefilter_shell'
 
 # Provide pysh and further shell-oriented services
 import os,sys,shutil
-from IPython.utils.genutils import system,shell,getoutput,getoutputerror
+from IPython.utils.process import system,shell,getoutput,getoutputerror
 
 # Short aliases for getting shell output as a string and a list
 sout = getoutput

IPython/quarantine/ext_rescapture.py

@@ -10,6 +10,7 b' var = !ls'
 
 from IPython.core import ipapi
 from IPython.core.error import TryNext
+from IPython.utils.text import make_quoted_expr
 from IPython.utils.genutils import *
 
 ip = ipapi.get()

IPython/quarantine/ipy_greedycompleter.py

@@ -12,7 +12,7 b' do the same in default completer.'
 from IPython.core import ipapi
 from IPython.core.error import TryNext
 from IPython.utils import generics
-from IPython.utils.genutils import dir2
+from IPython.utils.dir2 import dir2
 
 def attr_matches(self, text):
     """Compute matches when text contains a dot.

IPython/quarantine/ipy_jot.py

@@ -16,6 +16,7 b' import pickleshare'
 import inspect,pickle,os,sys,textwrap
 from IPython.core.fakemodule import FakeModule
 from IPython.utils.ipstruct import Struct 
+from IPython.utils.warn import error
 
 
 def refresh_variables(ip, key=None):

IPython/quarantine/ipy_pydb.py

@@ -1,6 +1,6 b''
 import inspect
 from IPython.core import ipapi
-from IPython.utils.genutils import arg_split
+from IPython.utils.process import arg_split
 ip = ipapi.get()
 
 from IPython.core import debugger

IPython/quarantine/jobctrl.py

@@ -45,10 +45,9 b' from subprocess import *'
 import os,shlex,sys,time
 import threading,Queue
 
-from IPython.utils import genutils
-
 from IPython.core import ipapi
 from IPython.core.error import TryNext
+from IPython.utils.text import make_quoted_expr
 
 if os.name == 'nt':
     def kill_process(pid):
@@ -126,8 +125,8 b' def jobctrl_prefilter_f(self,line):'
         
         line = ip.expand_aliases(fn,rest)
         if not _jobq:
-            return 'get_ipython().startjob(%s)' % genutils.make_quoted_expr(line)
-        return 'get_ipython().jobq(%s)' % genutils.make_quoted_expr(line)
+            return 'get_ipython().startjob(%s)' % make_quoted_expr(line)
+        return 'get_ipython().jobq(%s)' % make_quoted_expr(line)
 
     raise TryNext
 

IPython/testing/__init__.py

@@ -1,6 +1,17 b''
 """Testing support (tools to test IPython itself).
 """
 
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Functions
+#-----------------------------------------------------------------------------
+
 # User-level entry point for testing
 def test():
     """Run the entire IPython test suite.

IPython/testing/_doctest26.py

@@ -4,6 +4,17 b' This is just so we can use 2.6 features when running in 2.5, the code below is'
 copied verbatim from the stdlib's collections and doctest modules.
 """
 
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
 from keyword import iskeyword as _iskeyword
 from operator import itemgetter as _itemgetter
 import sys as _sys

IPython/testing/_paramtestpy2.py

@@ -1,10 +1,17 b''
 """Implementation of the parametric test support for Python 2.x
 """
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
-# Stdlib
 import unittest
 from compiler.consts import CO_GENERATOR
 

IPython/testing/_paramtestpy3.py

@@ -3,11 +3,18 b''
 Thanks for the py3 version to Robert Collins, from the Testing in Python
 mailing list.
 """
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
-# Stdlib
 import unittest
 from unittest import TestSuite
 

IPython/testing/globalipapp.py

@@ -9,18 +9,22 b' done.'
 from __future__ import absolute_import
 
 #-----------------------------------------------------------------------------
-# Module imports
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
 #-----------------------------------------------------------------------------
 
-# From the standard library
 import __builtin__
 import commands
-import new
 import os
 import sys
 
 from . import tools
-from IPython.utils.genutils import Term
 
 #-----------------------------------------------------------------------------
 # Functions
@@ -35,7 +39,7 b' class py_file_finder(object):'
         self.test_filename = test_filename
         
     def __call__(self,name):
-        from IPython.utils.genutils import get_py_filename
+        from IPython.utils.path import get_py_filename
         try:
             return get_py_filename(name)
         except IOError:

IPython/testing/iptest.py

@@ -17,13 +17,19 b' will change in the future.'
 """
 
 #-----------------------------------------------------------------------------
-# Module imports
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib
 import os
 import os.path as path
-import platform
 import signal
 import sys
 import subprocess
@@ -50,11 +56,11 b' import nose.plugins.builtin'
 from nose.core import TestProgram
 
 # Our own imports
-from IPython.core import release
-from IPython.utils import genutils
-from IPython.utils.platutils import find_cmd, FindCmdError
+from IPython.utils.path import get_ipython_module_path
+from IPython.utils.process import find_cmd, pycmd2argv
+from IPython.utils.sysinfo import sys_info
+
 from IPython.testing import globalipapp
-from IPython.testing import tools
 from IPython.testing.plugin.ipdoctest import IPythonDoctest
 
 pjoin = path.join
@@ -124,7 +130,7 b" have['gobject'] = test_for('gobject')"
 def report():
     """Return a string with a summary report of test-related variables."""
 
-    out = [ genutils.sys_info() ]
+    out = [ sys_info() ]
 
     out.append('\nRunning from an installed IPython: %s\n' % INSTALLED)
 
@@ -198,18 +204,12 b' def make_exclude():'
     if not have['objc']:
         exclusions.append(ipjoin('frontend', 'cocoa'))
 
-    if not sys.platform == 'win32':
-        exclusions.append(ipjoin('utils', 'platutils_win32'))
-
     # These have to be skipped on win32 because the use echo, rm, cd, etc.
     # See ticket https://bugs.launchpad.net/bugs/366982
     if sys.platform == 'win32':
         exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
         exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
 
-    if not os.name == 'posix':
-        exclusions.append(ipjoin('utils', 'platutils_posix'))
-
     if not have['pexpect']:
         exclusions.extend([ipjoin('scripts', 'irunner'),
                            ipjoin('lib', 'irunner')])
@@ -256,19 +256,19 b' class IPTester(object):'
         p = os.path
         if runner == 'iptest':
             if INSTALLED:
-                self.runner = tools.cmd2argv(
-                    p.abspath(find_cmd('iptest'))) + sys.argv[1:]
+                iptest_app = get_ipython_module_path('IPython.testing.iptest')
+                self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
             else:
                 # Find our own 'iptest' script OS-level entry point.  Don't
                 # look system-wide, so we are sure we pick up *this one*.  And
                 # pass through to subprocess call our own sys.argv
                 ippath = p.abspath(p.join(p.dirname(__file__),'..','..'))
                 script = p.join(ippath, 'iptest.py')
-                self.runner = tools.cmd2argv(script) + sys.argv[1:]
+                self.runner = pycmd2argv(script) + sys.argv[1:]
                 
         else:
             # For trial, it needs to be installed system-wide
-            self.runner = tools.cmd2argv(p.abspath(find_cmd('trial')))
+            self.runner = pycmd2argv(p.abspath(find_cmd('trial')))
         if params is None:
             params = []
         if isinstance(params, str):

IPython/testing/ipunittest.py

@@ -31,7 +31,6 b' from __future__ import absolute_import'
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
-
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------

IPython/testing/mkdoctests.py

@@ -38,7 +38,7 b' import tempfile'
 
 # IPython-specific libraries
 from IPython.lib import irunner
-from IPython.utils.genutils import fatal
+from IPython.utils.warn import fatal
 
 class IndentOut(object):
     """A simple output stream that indents all output by a fixed amount.

IPython/testing/nosepatch.py

@@ -5,10 +5,25 b' Once this is fixed in upstream nose we can disable it.'
 
 Note: merely importing this module causes the monkeypatch to be applied."""
 
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
 import unittest
 import nose.loader
 from inspect import ismethod, isfunction
 
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
 def getTestCaseNames(self, testCaseClass):
     """Override to select with selector, unless
     config.getTestCaseNamesCompat is True

IPython/testing/parametric.py

@@ -5,10 +5,27 b" parametric code. We just need to double-check that the new code doesn't clash"
 with Twisted (we know it works with nose and unittest).
 """
 
-__all__ = ['parametric','Parametric']
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
 
 from twisted.trial.unittest import TestCase
 
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
+__all__ = ['parametric','Parametric']
+
+
 def partial(f, *partial_args, **partial_kwargs):
     """Generate a partial class method.
 
@@ -20,6 +37,7 b' def partial(f, *partial_args, **partial_kwargs):'
 
     return partial_func
 
+
 def parametric(f):
     """Mark f as a parametric test.
 
@@ -27,6 +45,7 b' def parametric(f):'
     f._parametric = True
     return classmethod(f)
 
+
 def Parametric(cls):
     """Register parametric tests with a class.
 
@@ -56,3 +75,4 b' def Parametric(cls):'
 
         # rename test generator so it isn't called again by nose
         test_gen.im_func.func_name = '__done_' + test_name
+

IPython/testing/tools.py

@@ -15,22 +15,22 b' Authors'
 - Fernando Perez <[e-mail]>
 """
 
-#*****************************************************************************
-#       Copyright (C) 2009 The IPython Development Team
+from __future__ import absolute_import
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#*****************************************************************************
+#-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
-# Required modules and packages
+# Imports
 #-----------------------------------------------------------------------------
-from __future__ import absolute_import
 
 import os
 import re
 import sys
-import tempfile
 
 try:
     # These tools are used by parts of the runtime, so we make the nose
@@ -41,7 +41,9 b' try:'
 except ImportError:
     has_nose = False
 
-from IPython.utils import genutils, platutils
+from IPython.utils.process import find_cmd, getoutputerror
+from IPython.utils.text import list_strings
+from IPython.utils.io import temp_pyfile
 
 from . import decorators as dec
 
@@ -107,7 +109,7 b' def full_path(startPath,files):'
     ['/a.txt']
     """
 
-    files = genutils.list_strings(files)
+    files = list_strings(files)
     base = os.path.split(startPath)[0]
     return [ os.path.join(base,f) for f in files ]
 
@@ -156,63 +158,6 b' def parse_test_output(txt):'
 parse_test_output.__test__ = False
 
 
-def cmd2argv(cmd):
-    r"""Take the path of a command and return a list (argv-style).
-
-    For a given path ``cmd``, this returns [cmd] if cmd's extension is .exe,
-    .com or .bat, and ['python', cmd] otherwise.
-
-    This is mostly a Windows utility, to deal with the fact that the scripts in
-    Windows get wrapped in .exe entry points, so we have to call them
-    differently.
-
-    Parameters
-    ----------
-    cmd : string
-      The path of the command.
-
-    Returns
-    -------
-    argv-style list.
-
-    Examples
-    --------
-    In [2]: cmd2argv('/usr/bin/ipython')
-    Out[2]: ['python', '/usr/bin/ipython']
-
-    In [3]: cmd2argv(r'C:\Python26\Scripts\ipython.exe')
-    Out[3]: ['C:\\Python26\\Scripts\\ipython.exe']
-    """
-    ext = os.path.splitext(cmd)[1]
-    if ext in ['.exe', '.com', '.bat']:
-        return [cmd]
-    else:
-        return ['python', cmd]
-    
-
-def temp_pyfile(src, ext='.py'):
-    """Make a temporary python file, return filename and filehandle.
-
-    Parameters
-    ----------
-    src : string or list of strings (no need for ending newlines if list)
-      Source code to be written to the file.
-
-    ext : optional, string
-      Extension for the generated file.
-
-    Returns
-    -------
-    (filename, open filehandle)
-      It is the caller's responsibility to close the open file and unlink it.
-    """
-    fname = tempfile.mkstemp(ext)[1]
-    f = open(fname,'w')
-    f.write(src)
-    f.flush()
-    return fname, f
-
-
 def default_argv():
     """Return a valid default argv for creating testing instances of ipython"""
 
@@ -256,7 +201,7 b' def ipexec(fname, options=None):'
     # suite can be run from the source tree without an installed IPython
     p = os.path
     if INSTALLED:
-        ipython_cmd = platutils.find_cmd('ipython')
+        ipython_cmd = find_cmd('ipython')
     else:
         ippath = p.abspath(p.join(p.dirname(__file__),'..','..'))
         ipython_script = p.join(ippath, 'ipython.py')
@@ -265,7 +210,7 b' def ipexec(fname, options=None):'
     full_fname = p.join(test_dir, fname)
     full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname)
     #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg
-    return genutils.getoutputerror(full_cmd)
+    return getoutputerror(full_cmd)
 
 
 def ipexec_validate(fname, expected_out, expected_err='',

IPython/testing/util.py

@@ -1,23 +1,24 b''
 # encoding: utf-8
 """This file contains utility classes for performing tests with Deferreds.
 """
-__docformat__ = "restructuredtext en"
-#-------------------------------------------------------------------------------
-#       Copyright (C) 2005  Fernando Perez <[e-mail]>
-#                           Brian E Granger <[e-mail]>
-#                           Benjamin Ragan-Kelley <[e-mail]>
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
 from twisted.trial import unittest
 from twisted.internet import defer
 
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
 class DeferredTestCase(unittest.TestCase):
     
     def assertDeferredEquals(self, deferred, expectedResult,

IPython/utils/generics.py

@@ -1,8 +1,9 b''
+# encoding: utf-8
 """Generic functions for extending IPython.
 
 See http://cheeseshop.python.org/pypi/simplegeneric.
 
-Here is an example from genutils.py::
+Here is an example from IPython.utils.text::
 
     def print_lsstring(arg):
         "Prettier (non-repr-like) and more informative printer for LSString"
@@ -12,19 +13,37 b' Here is an example from genutils.py::'
     print_lsstring = result_display.when_type(LSString)(print_lsstring)
 """
 
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
 from IPython.core.error import TryNext
 from IPython.external.simplegeneric import generic
 
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+
 @generic
 def result_display(result):
     """Print the result of computation."""
     raise TryNext
 
+
 @generic
 def inspect_object(obj):
     """Called when you do obj?"""
     raise TryNext
 
+
 @generic
 def complete_object(obj, prev_completions):
     """Custom completer dispatching for python objects.
@@ -41,3 +60,5 b' def complete_object(obj, prev_completions):'
     own_attrs + prev_completions.
     """
     raise TryNext
+
+

IPython/utils/importstring.py

@@ -1,4 +1,3 b''
-#!/usr/bin/env python
 # encoding: utf-8
 """
 A simple utility to import something by its string name.

IPython/utils/ipstruct.py

@@ -19,9 +19,7 b' Authors:'
 # Imports
 #-----------------------------------------------------------------------------
 
-import pprint
-
-from IPython.utils.genutils import list2dict2
+from IPython.utils.data import list2dict2
 
 __all__ = ['Struct']
 

IPython/utils/pickleshare.py

@@ -37,7 +37,6 b' from IPython.external.path import path as Path'
 import os,stat,time
 import cPickle as pickle
 import UserDict
-import warnings
 import glob
 
 def gethashfile(key):

IPython/utils/rlineimpl.py

@@ -1,10 +1,12 b''
 # -*- coding: utf-8 -*-
 """ Imports and provides the 'correct' version of readline for the platform.
 
-Readline is used throughout IPython as 'import IPython.utils.rlineimpl as readline'.
+Readline is used throughout IPython as::
+
+    import IPython.utils.rlineimpl as readline
 
 In addition to normal readline stuff, this module provides have_readline
-boolean and _outputfile variable used in genutils.
+boolean and _outputfile variable used in IPython.utils.
 """
 
 import sys

IPython/utils/syspathcontext.py

@@ -19,9 +19,12 b' Authors:'
 # Imports
 #-----------------------------------------------------------------------------
 
-
 import sys
 
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
 class appended_to_syspath(object):
     """A context for appending a directory to sys.path for a second."""
 

IPython/utils/tests/test_imports.py

@@ -3,21 +3,12 b''
 def test_import_coloransi():
     from IPython.utils import coloransi
 
-def test_import_DPyGetOpt():
-    from IPython.utils import DPyGetOpt
-
 def test_import_generics():
     from IPython.utils import generics
 
-def test_import_genutils():
-    from IPython.utils import genutils
-
 def test_import_ipstruct():
     from IPython.utils import ipstruct
 
-def test_import_platutils():
-    from IPython.utils import platutils
-
 def test_import_PyColorize():
     from IPython.utils import PyColorize
 
@@ -33,5 +24,3 b' def test_import_upgradedir():'
 def test_import_wildcard():
     from IPython.utils import wildcard
 
-def test_import_winconsole():
-    from IPython.utils import winconsole

IPython/utils/tests/test_notification.py

@@ -15,11 +15,7 b''
 
 import unittest
 
-from IPython.utils.notification import (
-    NotificationCenter,
-    NotificationError,
-    shared_center
-)
+from IPython.utils.notification import shared_center
 
 #-----------------------------------------------------------------------------
 # Support Classes

IPython/utils/tests/test_path.py

@@ -1,8 +1,5 b''
 # encoding: utf-8
-
-"""Tests for genutils.py"""
-
-__docformat__ = "restructuredtext en"
+"""Tests for IPython.utils.path.py"""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
@@ -15,27 +12,29 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-----------------------------------------------------------------------------
 
-# stdlib
 import os
 import shutil
 import sys
 import tempfile
-import unittest
 
-from cStringIO import StringIO
 from os.path import join, abspath, split
 
-# third-party
 import nose.tools as nt
 
 from nose import with_setup
-from nose.tools import raises
 
-# Our own
 import IPython
-from IPython.utils import genutils
 from IPython.testing import decorators as dec
-from IPython.testing.decorators import skipif, skip_if_not_win32
+from IPython.testing.decorators import skip_if_not_win32
+from IPython.utils.path import (
+    get_home_dir,
+    HomeDirError,
+    get_ipython_dir,
+    get_ipython_package_dir,
+    get_ipython_module_path,
+    filefind,
+    get_long_path_name
+)
 
 # Platform-dependent imports
 try:
@@ -88,7 +87,7 b' def setup_environment():'
     each testfunction needs a pristine environment.
     """
     global oldstuff, platformstuff
-    oldstuff = (env.copy(), os.name, genutils.get_home_dir, IPython.__file__)
+    oldstuff = (env.copy(), os.name, get_home_dir, IPython.__file__)
 
     if os.name == 'nt':
         platformstuff = (wreg.OpenKey, wreg.QueryValueEx,)
@@ -97,7 +96,7 b' def setup_environment():'
 def teardown_environment():
     """Restore things that were remebered by the setup_environment function
     """
-    (oldenv, os.name, genutils.get_home_dir, IPython.__file__,) = oldstuff
+    (oldenv, os.name, get_home_dir, IPython.__file__,) = oldstuff
         
     for key in env.keys():
         if key not in oldenv:
@@ -112,10 +111,6 b' def teardown_environment():'
 with_environment = with_setup(setup_environment, teardown_environment)
 
 
-#
-# Tests for get_home_dir
-#
-
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_1():
@@ -126,7 +121,7 b' def test_get_home_dir_1():'
     #fake filename for IPython.__init__
     IPython.__file__ = abspath(join(HOME_TEST_DIR, "Lib/IPython/__init__.py"))
     
-    home_dir = genutils.get_home_dir()
+    home_dir = get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
     
 @skip_if_not_win32
@@ -138,14 +133,14 b' def test_get_home_dir_2():'
     #fake filename for IPython.__init__
     IPython.__file__ = abspath(join(HOME_TEST_DIR, "Library.zip/IPython/__init__.py")).lower()
     
-    home_dir = genutils.get_home_dir()
+    home_dir = get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR).lower())
 
 @with_environment
 def test_get_home_dir_3():
     """Testcase $HOME is set, then use its value as home directory."""
     env["HOME"] = HOME_TEST_DIR
-    home_dir = genutils.get_home_dir()
+    home_dir = get_home_dir()
     nt.assert_equal(home_dir, env["HOME"])
 
 @with_environment
@@ -155,7 +150,7 b' def test_get_home_dir_4():'
     
     os.name = 'posix'
     if 'HOME' in env: del env['HOME']
-    nt.assert_raises(genutils.HomeDirError, genutils.get_home_dir)
+    nt.assert_raises(HomeDirError, get_home_dir)
         
 @skip_if_not_win32
 @with_environment
@@ -167,7 +162,7 b' def test_get_home_dir_5():'
     if 'HOME' in env: del env['HOME']
     env['HOMEDRIVE'], env['HOMEPATH'] = os.path.splitdrive(HOME_TEST_DIR)
 
-    home_dir = genutils.get_home_dir()
+    home_dir = get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
 
 @skip_if_not_win32
@@ -183,7 +178,7 b' def test_get_home_dir_6():'
     env['HOMEDRIVE'], env['HOMEPATH'] = os.path.abspath(TEST_FILE_PATH), "DOES NOT EXIST"
     env["USERPROFILE"] = abspath(HOME_TEST_DIR)
 
-    home_dir = genutils.get_home_dir()
+    home_dir = get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
 
 # Should we stub wreg fully so we can run the test on all platforms?
@@ -211,116 +206,55 b' def test_get_home_dir_7():'
     wreg.OpenKey = OpenKey
     wreg.QueryValueEx = QueryValueEx
 
-    home_dir = genutils.get_home_dir()
+    home_dir = get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
 
-#
-# Tests for get_ipython_dir
-#
 
 @with_environment
 def test_get_ipython_dir_1():
     """test_get_ipython_dir_1, Testcase to see if we can call get_ipython_dir without Exceptions."""
     env['IPYTHON_DIR'] = "someplace/.ipython"
-    ipdir = genutils.get_ipython_dir()
+    ipdir = get_ipython_dir()
     nt.assert_equal(ipdir, "someplace/.ipython")
 
 
 @with_environment
 def test_get_ipython_dir_2():
     """test_get_ipython_dir_2, Testcase to see if we can call get_ipython_dir without Exceptions."""
-    genutils.get_home_dir = lambda : "someplace"
+    get_home_dir = lambda : "someplace"
     os.name = "posix"
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
-    ipdir = genutils.get_ipython_dir()
+    ipdir = get_ipython_dir()
     nt.assert_equal(ipdir, os.path.join("someplace", ".ipython"))
 
-#
-# Tests for popkey
-#
-
-def test_popkey_1():
-    """test_popkey_1, Basic usage test of popkey
-    """
-    dct = dict(a=1, b=2, c=3)
-    nt.assert_equal(genutils.popkey(dct, "a"), 1)
-    nt.assert_equal(dct, dict(b=2, c=3))
-    nt.assert_equal(genutils.popkey(dct, "b"), 2)
-    nt.assert_equal(dct, dict(c=3))
-    nt.assert_equal(genutils.popkey(dct, "c"), 3)
-    nt.assert_equal(dct, dict())
-
-def test_popkey_2():
-    """test_popkey_2, Test to see that popkey of non occuring keys
-    generates a KeyError exception
-    """
-    dct = dict(a=1, b=2, c=3)
-    nt.assert_raises(KeyError, genutils.popkey, dct, "d")
-
-def test_popkey_3():
-    """test_popkey_3, Tests to see that popkey calls returns the correct value
-    and that the key/value was removed from the dict.
-    """
-    dct = dict(a=1, b=2, c=3)
-    nt.assert_equal(genutils.popkey(dct, "A", 13), 13)
-    nt.assert_equal(dct, dict(a=1, b=2, c=3))
-    nt.assert_equal(genutils.popkey(dct, "B", 14), 14)
-    nt.assert_equal(dct, dict(a=1, b=2, c=3))
-    nt.assert_equal(genutils.popkey(dct, "C", 15), 15)
-    nt.assert_equal(dct, dict(a=1, b=2, c=3))
-    nt.assert_equal(genutils.popkey(dct, "a"), 1)
-    nt.assert_equal(dct, dict(b=2, c=3))
-    nt.assert_equal(genutils.popkey(dct, "b"), 2)
-    nt.assert_equal(dct, dict(c=3))
-    nt.assert_equal(genutils.popkey(dct, "c"), 3)
-    nt.assert_equal(dct, dict())
-
 
 def test_filefind():
     """Various tests for filefind"""
     f = tempfile.NamedTemporaryFile()
     print 'fname:',f.name
-    alt_dirs = genutils.get_ipython_dir()
-    t = genutils.filefind(f.name,alt_dirs)
+    alt_dirs = get_ipython_dir()
+    t = filefind(f.name, alt_dirs)
     print 'found:',t
 
 
 def test_get_ipython_package_dir():
-    ipdir = genutils.get_ipython_package_dir()
+    ipdir = get_ipython_package_dir()
     nt.assert_true(os.path.isdir(ipdir))
 
+def test_get_ipython_module_path():
+    ipapp_path = get_ipython_module_path('IPython.core.ipapp')
+    nt.assert_true(os.path.isfile(ipapp_path))
 
-def test_tee_simple():
-    "Very simple check with stdout only"
-    chan = StringIO()
-    text = 'Hello'
-    tee = genutils.Tee(chan, channel='stdout')
-    print >> chan, text,
-    nt.assert_equal(chan.getvalue(), text)
[e-mail]_if_not_win32
+def test_get_long_path_name_win32():
+    p = get_long_path_name('c:\\docume~1')
+    nt.assert_equals(p,u'c:\\Documents and Settings') 
 
+    
[e-mail]_win32
+def test_get_long_path_name():
+    p = get_long_path_name('/usr/local')
+    nt.assert_equals(p,'/usr/local')
 
-class TeeTestCase(dec.ParametricTestCase):
 
-    def tchan(self, channel, check='close'):
-        trap = StringIO()
-        chan = StringIO()
-        text = 'Hello'
-        
-        std_ori = getattr(sys, channel)
-        setattr(sys, channel, trap)
-
-        tee = genutils.Tee(chan, channel=channel)
-        print >> chan, text,
-        setattr(sys, channel, std_ori)
-        trap_val = trap.getvalue()
-        nt.assert_equals(chan.getvalue(), text)
-        if check=='close':
-            tee.close()
-        else:
-            del tee
-
-    def test(self):
-        for chan in ['stdout', 'stderr']:
-            for check in ['close', 'del']:
-                yield self.tchan(chan, check)

IPython/utils/tests/test_process.py

@@ -14,12 +14,11 b' Tests for platutils.py'
 # Imports
 #-----------------------------------------------------------------------------
 
-import os
 import sys
 
 import nose.tools as nt
 
-from IPython.utils.platutils import find_cmd, FindCmdError, get_long_path_name
+from IPython.utils.process import find_cmd, FindCmdError
 from IPython.testing import decorators as dec
 
 #-----------------------------------------------------------------------------
@@ -60,13 +59,4 b' def test_find_cmd_fail():'
     nt.assert_raises(FindCmdError,find_cmd,'asdfasdf')
 
     
[e-mail]_if_not_win32
-def test_get_long_path_name_win32():
-    p = get_long_path_name('c:\\docume~1')
-    nt.assert_equals(p,u'c:\\Documents and Settings') 
 
-    
[e-mail]_win32
-def test_get_long_path_name():
-    p = get_long_path_name('/usr/local')
-    nt.assert_equals(p,'/usr/local')

IPython/utils/tests/test_traitlets.py

@@ -22,15 +22,11 b' Authors:'
 # Imports
 #-----------------------------------------------------------------------------
 
-import sys
-import os
-
-
 from unittest import TestCase
 
 from IPython.utils.traitlets import (
     HasTraitlets, MetaHasTraitlets, TraitletType, Any,
-    Int, Long, Float, Complex, Str, Unicode, Bool, TraitletError,
+    Int, Long, Float, Complex, Str, Unicode, TraitletError,
     Undefined, Type, This, Instance
 )
 

IPython/utils/upgradedir.py

@@ -11,7 +11,7 b' try:'
 except ImportError:
     from path import path
 
-import md5,pickle
+import md5, pickle
 
 def showdiff(old,new):
     import difflib

IPython/utils/wildcard.py

@@ -14,13 +14,10 b' Authors'
 #*****************************************************************************
 
 import __builtin__
-import exceptions
-import pdb
-import pprint
 import re
 import types
 
-from IPython.utils.genutils import dir2
+from IPython.utils.dir2 import dir2
 
 def create_typestr2type_dicts(dont_include_in_type2type2str=["lambda"]):
     """Return dictionaries mapping lower case typename to type objects, from

docs/examples/kernel/task_profiler.py

@@ -18,7 +18,7 b' overhead of a single task is about 0.001-0.01 seconds.'
 import random, sys
 from optparse import OptionParser
 
-from IPython.genutils import time
+from IPython.utils.timing import time
 from IPython.kernel import client
 
 def main():
@@ -51,7 +51,7 b' def main():'
     print tc.task_controller
     rc.block=True
     nengines = len(rc.get_ids())
-    rc.execute('from IPython.genutils import time')
+    rc.execute('from IPython.utils.timing import time')
 
     # the jobs should take a random time within a range
     times = [random.random()*(opts.tmax-opts.tmin)+opts.tmin for i in range(opts.n)]

docs/source/config/overview.txt

@@ -245,7 +245,7 b' this directory is determined by the following algorithm:'
 
 * If the ``--ipython-dir`` command line flag is given, its value is used.
 
-* If not, the value returned by :func:`IPython.utils.genutils.get_ipython_dir`
+* If not, the value returned by :func:`IPython.utils.path.get_ipython_dir`
   is used. This function will first look at the :envvar:`IPYTHON_DIR`
   environment variable and then default to the directory
   :file:`$HOME/.ipython`.

docs/source/development/doc_guide.txt

@@ -41,8 +41,8 b' A bit of Python code::'
 
 An interactive Python session::
 
-    >>> from IPython import genutils
-    >>> genutils.get_ipython_dir()
+    >>> from IPython.utils.path import get_ipython_dir
+    >>> get_ipython_dir()
     '/home/fperez/.ipython'
 
 An IPython session:

docs/source/interactive/shell.txt

@@ -166,7 +166,7 b' ipy_user_conf.py.'
 String lists
 ============
 
-String lists (IPython.genutils.SList) are handy way to process output
+String lists (IPython.utils.text.SList) are handy way to process output
 from system commands. They are produced by ``var = !cmd`` syntax.
 
 First, we acquire the output of 'ls -l'::

docs/sphinxext/docscrape.py

@@ -8,7 +8,7 b' import re'
 import pydoc
 from StringIO import StringIO
 from warnings import warn
-4
+
 class Reader(object):
     """A line-based string reader.
 

docs/sphinxext/ipython_directive.py

@@ -81,7 +81,7 b" matplotlib.use('Agg')"
 
 # Our own
 from IPython import Config, IPythonApp
-from IPython.utils.genutils import Term, Tee
+from IPython.utils.io import Term, Tee
 
 #-----------------------------------------------------------------------------
 # Globals
@@ -208,8 +208,9 b' class EmbeddedSphinxShell(object):'
         Term.cerr = self.cout
 
         # For debugging, so we can see normal output, use this:
-        #Term.cout = genutils.Tee(self.cout, channel='stdout') # dbg
-        #Term.cerr = genutils.Tee(self.cout, channel='stderr') # dbg
+        # from IPython.utils.io import Tee
+        #Term.cout = Tee(self.cout, channel='stdout') # dbg
+        #Term.cerr = Tee(self.cout, channel='stderr') # dbg
 
         # Create config object for IPython
         config = Config()

setup.py

@@ -47,7 +47,7 b" if os.path.exists('MANIFEST'): os.remove('MANIFEST')"
 from distutils.core import setup
 
 # Our own imports
-from IPython.utils.genutils import target_update
+from IPython.utils.path import target_update
 
 from setupbase import (
     setup_args,