Work to address the review comments on Fernando's branch.

* Added comment about Magic(object) (r1224)
* Moved InteractiveTB.set_mode from IPythonApp -> InteractiveShell (r1229)
* Moved pylabtools.py to IPython/lib (r1229)
* Cleaned up comments and copyrights in testing (r1233)
* Added comment about ip.shell._ofind (r1237)
* Removed "Bye." from quitter (r1240).
* Refactored and removed :mod:`IPython.utils.genutils` and
  :mod:`IPython.utils.platutils`. These modules have been replaced by topical
  focused modules in :mod:`IPython.utils`.
* Refactored tests in :mod:`IPython.utils.tests`.
* Moved :func:`IPython.testing.tools.temp_pyfile` to :mod:`IPython.utils.io`.
* Moved :func:`IPython.testing.tools.cmd2argv` to
  :func:`IPython.testing.tools.pycmd2argv` and documented the fact that this
  only works with Python based command line programs.
* Created a new :func:`IPython.utils.path.get_ipython_module_path` to use
  in finding paths to IPython modules.

IPython/utils/attic.py

@@ -0,0 +1,167
+# encoding: utf-8
+"""
+Older utilities that are not being used.
+
+WARNING: IF YOU NEED TO USE ONE OF THESE FUNCTIONS, PLEASE FIRST MOVE IT
+TO ANOTHER APPROPRIATE MODULE IN IPython.utils.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+import warnings
+
+from IPython.utils.warn import warn
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+def mutex_opts(dict,ex_op):
+    """Check for presence of mutually exclusive keys in a dict.
+
+    Call: mutex_opts(dict,[[op1a,op1b],[op2a,op2b]...]"""
+    for op1,op2 in ex_op:
+        if op1 in dict and op2 in dict:
+            raise ValueError,'\n*** ERROR in Arguments *** '\
+                  'Options '+op1+' and '+op2+' are mutually exclusive.'
+
+
+class EvalDict:
+    """
+    Emulate a dict which evaluates its contents in the caller's frame.
+
+    Usage:
+    >>> number = 19
+
+    >>> text = "python"
+
+    >>> print "%(text.capitalize())s %(number/9.0).1f rules!" % EvalDict()
+    Python 2.1 rules!
+    """
+
+    # This version is due to [e-mail] on c.l.py, and is basically a
+    # modified (shorter) version of:
+    # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66018 by
+    # Skip Montanaro ([e-mail]).
+
+    def __getitem__(self, name):
+        frame = sys._getframe(1)
+        return eval(name, frame.f_globals, frame.f_locals)
+
+EvalString = EvalDict  # for backwards compatibility
+
+
+def all_belong(candidates,checklist):
+    """Check whether a list of items ALL appear in a given list of options.
+
+    Returns a single 1 or 0 value."""
+
+    return 1-(0 in [x in checklist for x in candidates])
+
+
+def belong(candidates,checklist):
+    """Check whether a list of items appear in a given list of options.
+
+    Returns a list of 1 and 0, one for each candidate given."""
+
+    return [x in checklist for x in candidates]
+
+
+def with_obj(object, **args):
+    """Set multiple attributes for an object, similar to Pascal's with.
+
+    Example:
+    with_obj(jim,
+             born = 1960,
+             haircolour = 'Brown',
+             eyecolour = 'Green')
+
+    Credit: Greg Ewing, in
+    http://mail.python.org/pipermail/python-list/2001-May/040703.html.
+
+    NOTE: up until IPython 0.7.2, this was called simply 'with', but 'with'
+    has become a keyword for Python 2.5, so we had to rename it."""
+
+    object.__dict__.update(args)
+
+
+def map_method(method,object_list,*argseq,**kw):
+    """map_method(method,object_list,*args,**kw) -> list
+
+    Return a list of the results of applying the methods to the items of the
+    argument sequence(s).  If more than one sequence is given, the method is
+    called with an argument list consisting of the corresponding item of each
+    sequence. All sequences must be of the same length.
+
+    Keyword arguments are passed verbatim to all objects called.
+
+    This is Python code, so it's not nearly as fast as the builtin map()."""
+
+    out_list = []
+    idx = 0
+    for object in object_list:
+        try:
+            handler = getattr(object, method)
+        except AttributeError:
+            out_list.append(None)
+        else:
+            if argseq:
+                args = map(lambda lst:lst[idx],argseq)
+                #print 'ob',object,'hand',handler,'ar',args # dbg
+                out_list.append(handler(args,**kw))
+            else:
+                out_list.append(handler(**kw))
+        idx += 1
+    return out_list
+
+
+def import_fail_info(mod_name,fns=None):
+    """Inform load failure for a module."""
+
+    if fns == None:
+        warn("Loading of %s failed.\n" % (mod_name,))
+    else:
+        warn("Loading of %s from %s failed.\n" % (fns,mod_name))
+
+
+class NotGiven: pass
+
+def popkey(dct,key,default=NotGiven):
+    """Return dct[key] and delete dct[key].
+
+    If default is given, return it if dct[key] doesn't exist, otherwise raise
+    KeyError.  """
+
+    try:
+        val = dct[key]
+    except KeyError:
+        if default is NotGiven:
+            raise
+        else:
+            return default
+    else:
+        del dct[key]
+        return val
+
+
+def wrap_deprecated(func, suggest = '<nothing>'):
+    def newFunc(*args, **kwargs):
+        warnings.warn("Call to deprecated function %s, use %s instead" %
+                      ( func.__name__, suggest),
+                      category=DeprecationWarning,
+                      stacklevel = 2)
+        return func(*args, **kwargs)
+    return newFunc
+
+

IPython/utils/cursesimport.py

@@ -0,0 +1,31
+# encoding: utf-8
+"""
+See if we have curses.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+# Curses and termios are Unix-only modules
+try:
+    import curses
+    # We need termios as well, so if its import happens to raise, we bail on
+    # using curses altogether.
+    import termios
+except ImportError:
+    use_curses = False
+else:
+    # Curses on Solaris may not be complete, so we can't use it there
+    use_curses = hasattr(curses,'initscr')
  No newline at end of file

IPython/utils/data.py

@@ -0,0 +1,106
+# encoding: utf-8
+"""Utilities for working with data structures like lists, dicts and tuples.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import types
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def uniq_stable(elems):
+    """uniq_stable(elems) -> list
+
+    Return from an iterable, a list of all the unique elements in the input,
+    but maintaining the order in which they first appear.
+
+    A naive solution to this problem which just makes a dictionary with the
+    elements as keys fails to respect the stability condition, since
+    dictionaries are unsorted by nature.
+
+    Note: All elements in the input must be valid dictionary keys for this
+    routine to work, as it internally uses a dictionary for efficiency
+    reasons."""
+
+    unique = []
+    unique_dict = {}
+    for nn in elems:
+        if nn not in unique_dict:
+            unique.append(nn)
+            unique_dict[nn] = None
+    return unique
+
+
+def sort_compare(lst1, lst2, inplace=1):
+    """Sort and compare two lists.
+
+    By default it does it in place, thus modifying the lists. Use inplace = 0
+    to avoid that (at the cost of temporary copy creation)."""
+    if not inplace:
+        lst1 = lst1[:]
+        lst2 = lst2[:]
+    lst1.sort(); lst2.sort()
+    return lst1 == lst2
+
+
+def list2dict(lst):
+    """Takes a list of (key,value) pairs and turns it into a dict."""
+
+    dic = {}
+    for k,v in lst: dic[k] = v
+    return dic
+
+
+def list2dict2(lst, default=''):
+    """Takes a list and turns it into a dict.
+    Much slower than list2dict, but more versatile. This version can take
+    lists with sublists of arbitrary length (including sclars)."""
+
+    dic = {}
+    for elem in lst:
+        if type(elem) in (types.ListType,types.TupleType):
+            size = len(elem)
+            if  size == 0:
+                pass
+            elif size == 1:
+                dic[elem] = default
+            else:
+                k,v = elem[0], elem[1:]
+                if len(v) == 1: v = v[0]
+                dic[k] = v
+        else:
+            dic[elem] = default
+    return dic
+
+
+def flatten(seq):
+    """Flatten a list of lists (NOT recursive, only works for 2d lists)."""
+
+    return [x for subseq in seq for x in subseq]
+
+
+def get_slice(seq, start=0, stop=None, step=1):
+    """Get a slice of a sequence with variable step. Specify start,stop,step."""
+    if stop == None:
+        stop = len(seq)
+    item = lambda i: seq[i]
+    return map(item,xrange(start,stop,step))
+
+
+def chop(seq, size):
+    """Chop a sequence into chunks of the given size."""
+    chunk = lambda i: seq[i:i+size]
+    return map(chunk,xrange(0,len(seq),size))
+
+

IPython/utils/decorators.py

@@ -0,0 +1,46
+# encoding: utf-8
+"""Decorators that don't go anywhere else.
+
+This module contains misc. decorators that don't really go with another module
+in :mod:`IPython.utils`. Beore putting something here please see if it should
+go into another topical module in :mod:`IPython.utils`.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def flag_calls(func):
+    """Wrap a function to detect and flag when it gets called.
+
+    This is a decorator which takes a function and wraps it in a function with
+    a 'called' attribute. wrapper.called is initialized to False.
+
+    The wrapper.called attribute is set to False right before each call to the
+    wrapped function, so if the call fails it remains False.  After the call
+    completes, wrapper.called is set to True and the output is returned.
+
+    Testing for truth in wrapper.called allows you to determine if a call to
+    func() was attempted and succeeded."""
+
+    def wrapper(*args,**kw):
+        wrapper.called = False
+        out = func(*args,**kw)
+        wrapper.called = True
+        return out
+
+    wrapper.called = False
+    wrapper.__doc__ = func.__doc__
+    return wrapper
+

IPython/utils/dir2.py

@@ -0,0 +1,82
+# encoding: utf-8
+"""A fancy version of Python's builtin :func:`dir` function.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def get_class_members(cls):
+    ret = dir(cls)
+    if hasattr(cls,'__bases__'):
+        for base in cls.__bases__:
+            ret.extend(get_class_members(base))
+    return ret
+
+
+def dir2(obj):
+    """dir2(obj) -> list of strings
+
+    Extended version of the Python builtin dir(), which does a few extra
+    checks, and supports common objects with unusual internals that confuse
+    dir(), such as Traits and PyCrust.
+
+    This version is guaranteed to return only a list of true strings, whereas
+    dir() returns anything that objects inject into themselves, even if they
+    are later not really valid for attribute access (many extension libraries
+    have such bugs).
+    """
+
+    # Start building the attribute list via dir(), and then complete it
+    # with a few extra special-purpose calls.
+    words = dir(obj)
+
+    if hasattr(obj,'__class__'):
+        words.append('__class__')
+        words.extend(get_class_members(obj.__class__))
+    #if '__base__' in words: 1/0
+
+    # Some libraries (such as traits) may introduce duplicates, we want to
+    # track and clean this up if it happens
+    may_have_dupes = False
+
+    # this is the 'dir' function for objects with Enthought's traits
+    if hasattr(obj, 'trait_names'):
+        try:
+            words.extend(obj.trait_names())
+            may_have_dupes = True
+        except TypeError:
+            # This will happen if `obj` is a class and not an instance.
+            pass
+
+    # Support for PyCrust-style _getAttributeNames magic method.
+    if hasattr(obj, '_getAttributeNames'):
+        try:
+            words.extend(obj._getAttributeNames())
+            may_have_dupes = True
+        except TypeError:
+            # `obj` is a class and not an instance.  Ignore
+            # this error.
+            pass
+
+    if may_have_dupes:
+        # eliminate possible duplicates, as some traits may also
+        # appear as normal attributes in the dir() call.
+        words = list(set(words))
+        words.sort()
+
+    # filter out non-string attributes which may be stuffed by dir() calls
+    # and poor coding in third-party modules
+    return [w for w in words if isinstance(w, basestring)]
+

IPython/utils/doctestreload.py

@@ -0,0 +1,75
+# encoding: utf-8
+"""
+A utility for handling the reloading of doctest.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def dhook_wrap(func,*a,**k):
+    """Wrap a function call in a sys.displayhook controller.
+
+    Returns a wrapper around func which calls func, with all its arguments and
+    keywords unmodified, using the default sys.displayhook.  Since IPython
+    modifies sys.displayhook, it breaks the behavior of certain systems that
+    rely on the default behavior, notably doctest.
+    """
+
+    def f(*a,**k):
+
+        dhook_s = sys.displayhook
+        sys.displayhook = sys.__displayhook__
+        try:
+            out = func(*a,**k)
+        finally:
+            sys.displayhook = dhook_s
+
+        return out
+
+    f.__doc__ = func.__doc__
+    return f
+
+
+def doctest_reload():
+    """Properly reload doctest to reuse it interactively.
+
+    This routine:
+
+      - imports doctest but does NOT reload it (see below).
+
+      - resets its global 'master' attribute to None, so that multiple uses of
+      the module interactively don't produce cumulative reports.
+
+      - Monkeypatches its core test runner method to protect it from IPython's
+      modified displayhook.  Doctest expects the default displayhook behavior
+      deep down, so our modification breaks it completely.  For this reason, a
+      hard monkeypatch seems like a reasonable solution rather than asking
+      users to manually use a different doctest runner when under IPython.
+
+    Notes
+    -----
+
+    This function *used to* reload doctest, but this has been disabled because
+    reloading doctest unconditionally can cause massive breakage of other
+    doctest-dependent modules already in memory, such as those for IPython's
+    own testing system.  The name wasn't changed to avoid breaking people's
+    code, but the reload call isn't actually made anymore."""
+
+    import doctest
+    doctest.master = None
+    doctest.DocTestRunner.run = dhook_wrap(doctest.DocTestRunner.run)
+

IPython/utils/frame.py

@@ -0,0 +1,85
+# encoding: utf-8
+"""
+Utilities for working with stack frames.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def extract_vars(*names,**kw):
+    """Extract a set of variables by name from another frame.
+
+    :Parameters:
+      - `*names`: strings
+        One or more variable names which will be extracted from the caller's
+    frame.
+
+    :Keywords:
+      - `depth`: integer (0)
+        How many frames in the stack to walk when looking for your variables.
+
+
+    Examples:
+
+        In [2]: def func(x):
+           ...:     y = 1
+           ...:     print extract_vars('x','y')
+           ...:
+
+        In [3]: func('hello')
+        {'y': 1, 'x': 'hello'}
+    """
+
+    depth = kw.get('depth',0)
+    
+    callerNS = sys._getframe(depth+1).f_locals
+    return dict((k,callerNS[k]) for k in names)
+
+
+def extract_vars_above(*names):
+    """Extract a set of variables by name from another frame.
+
+    Similar to extractVars(), but with a specified depth of 1, so that names
+    are exctracted exactly from above the caller.
+
+    This is simply a convenience function so that the very common case (for us)
+    of skipping exactly 1 frame doesn't have to construct a special dict for
+    keyword passing."""
+
+    callerNS = sys._getframe(2).f_locals
+    return dict((k,callerNS[k]) for k in names)
+
+
+def debugx(expr,pre_msg=''):
+    """Print the value of an expression from the caller's frame.
+
+    Takes an expression, evaluates it in the caller's frame and prints both
+    the given expression and the resulting value (as well as a debug mark
+    indicating the name of the calling function.  The input must be of a form
+    suitable for eval().
+
+    An optional message can be passed, which will be prepended to the printed
+    expr->value pair."""
+
+    cf = sys._getframe(1)
+    print '[DBG:%s] %s%s -> %r' % (cf.f_code.co_name,pre_msg,expr,
+                                   eval(expr,cf.f_globals,cf.f_locals))
+
+
+# deactivate it by uncommenting the following line, which makes it a no-op
+#def debugx(expr,pre_msg=''): pass
+

IPython/utils/io.py

@@ -0,0 +1,292
+# encoding: utf-8
+"""
+IO related utilities.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+import tempfile
+
+from IPython.external.Itpl import itpl, printpl
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+class IOStream:
+
+    def __init__(self,stream,fallback):
+        if not hasattr(stream,'write') or not hasattr(stream,'flush'):
+            stream = fallback
+        self.stream = stream
+        self._swrite = stream.write
+        self.flush = stream.flush
+
+    def write(self,data):
+        try:
+            self._swrite(data)
+        except:
+            try:
+                # print handles some unicode issues which may trip a plain
+                # write() call.  Attempt to emulate write() by using a
+                # trailing comma
+                print >> self.stream, data,
+            except:
+                # if we get here, something is seriously broken.
+                print >> sys.stderr, \
+                      'ERROR - failed to write data to stream:', self.stream
+
+    def writeln(self, data):
+        self.write(data)
+        self.write('\n')        
+                
+    def close(self):
+        pass
+
+
+class IOTerm:
+    """ Term holds the file or file-like objects for handling I/O operations.
+
+    These are normally just sys.stdin, sys.stdout and sys.stderr but for
+    Windows they can can replaced to allow editing the strings before they are
+    displayed."""
+
+    # In the future, having IPython channel all its I/O operations through
+    # this class will make it easier to embed it into other environments which
+    # are not a normal terminal (such as a GUI-based shell)
+    def __init__(self,cin=None,cout=None,cerr=None):
+        self.cin  = IOStream(cin,sys.stdin)
+        self.cout = IOStream(cout,sys.stdout)
+        self.cerr = IOStream(cerr,sys.stderr)
+
+
+# Global variable to be used for all I/O
+Term = IOTerm()
+
+
+import IPython.utils.rlineimpl as readline
+# Remake Term to use the readline i/o facilities
+if sys.platform == 'win32' and readline.have_readline:
+
+    Term = IOTerm(cout=readline._outputfile,cerr=readline._outputfile)
+
+
+class Tee(object):
+    """A class to duplicate an output stream to stdout/err.
+
+    This works in a manner very similar to the Unix 'tee' command.
+
+    When the object is closed or deleted, it closes the original file given to
+    it for duplication.
+    """
+    # Inspired by:
+    # http://mail.python.org/pipermail/python-list/2007-May/442737.html
+
+    def __init__(self, file_or_name, mode=None, channel='stdout'):
+        """Construct a new Tee object.
+
+        Parameters
+        ----------
+        file_or_name : filename or open filehandle (writable)
+          File that will be duplicated
+
+        mode : optional, valid mode for open().
+          If a filename was give, open with this mode.
+
+        channel : str, one of ['stdout', 'stderr']  
+        """
+        if channel not in ['stdout', 'stderr']:
+            raise ValueError('Invalid channel spec %s' % channel)
+        
+        if hasattr(file, 'write') and hasattr(file, 'seek'):
+            self.file = file_or_name
+        else:
+            self.file = open(file_or_name, mode)
+        self.channel = channel
+        self.ostream = getattr(sys, channel)
+        setattr(sys, channel, self)
+        self._closed = False
+
+    def close(self):
+        """Close the file and restore the channel."""
+        self.flush()
+        setattr(sys, self.channel, self.ostream)
+        self.file.close()
+        self._closed = True
+
+    def write(self, data):
+        """Write data to both channels."""
+        self.file.write(data)
+        self.ostream.write(data)
+        self.ostream.flush()
+
+    def flush(self):
+        """Flush both channels."""
+        self.file.flush()
+        self.ostream.flush()
+
+    def __del__(self):
+        if not self._closed:
+            self.close()
+
+
+def file_read(filename):
+    """Read a file and close it.  Returns the file source."""
+    fobj = open(filename,'r');
+    source = fobj.read();
+    fobj.close()
+    return source
+
+
+def file_readlines(filename):
+    """Read a file and close it.  Returns the file source using readlines()."""
+    fobj = open(filename,'r');
+    lines = fobj.readlines();
+    fobj.close()
+    return lines
+
+
+def raw_input_multi(header='', ps1='==> ', ps2='..> ',terminate_str = '.'):
+    """Take multiple lines of input.
+
+    A list with each line of input as a separate element is returned when a
+    termination string is entered (defaults to a single '.'). Input can also
+    terminate via EOF (^D in Unix, ^Z-RET in Windows).
+
+    Lines of input which end in \\ are joined into single entries (and a
+    secondary continuation prompt is issued as long as the user terminates
+    lines with \\). This allows entering very long strings which are still
+    meant to be treated as single entities.
+    """
+
+    try:
+        if header:
+            header += '\n'
+