Merging a number of recent bug fixes in preparation for the 0.10 release.

No new features were introduced, but many test failures and a few bugs were
fixed.  In particular, closes:

https://bugs.launchpad.net/ipython/+bug/362137

IPython/frontend/tests/test_prefilterfrontend.py

@@ -15,6 +15,7 b' __docformat__ = "restructuredtext en"'
 from copy import copy, deepcopy
 from cStringIO import StringIO
 import string
+import sys
 
 from nose.tools import assert_equal
 
@@ -23,22 +24,6 b' from IPython.ipapi import get as get_ipython0'
 from IPython.testing.plugin.ipdoctest import default_argv
 
 
-def safe_deepcopy(d):
-    """ Deep copy every key of the given dict, when possible. Elsewhere
-        do a copy.
-    """
-    copied_d = dict()
-    for key, value in d.iteritems():
-        try:
-            copied_d[key] = deepcopy(value)
-        except:
-            try:
-                copied_d[key] = copy(value)
-            except:
-                copied_d[key] = value
-    return copied_d
-
-
 class TestPrefilterFrontEnd(PrefilterFrontEnd):
     
     input_prompt_template = string.Template('')
@@ -72,17 +57,34 b' def isolate_ipython0(func):'
         with arguments.
     """
     def my_func():
-        iplib = get_ipython0()
-        if iplib is None:
+        ip0 = get_ipython0()
+        if ip0 is None:
             return func()
-        ipython0 = iplib.IP
-        global_ns = safe_deepcopy(ipython0.user_global_ns)
-        user_ns = safe_deepcopy(ipython0.user_ns)
+        # We have a real ipython running...
+        user_ns = ip0.IP.user_ns
+        user_global_ns = ip0.IP.user_global_ns
+
+        # Previously the isolation was attempted with a deep copy of the user
+        # dicts, but we found cases where this didn't work correctly. I'm not
+        # quite sure why, but basically it did damage the user namespace, such
+        # that later tests stopped working correctly.  Instead we use a simpler
+        # approach, just computing the list of added keys to the namespace and
+        # eliminating those afterwards.  Existing keys that may have been
+        # modified remain modified.  So far this has proven to be robust.
+
+        # Compute set of old local/global keys
+        old_locals = set(user_ns.keys())
+        old_globals = set(user_global_ns.keys())
         try:
             out = func()
         finally:
-            ipython0.user_ns = user_ns
-            ipython0.user_global_ns = global_ns
+            # Find new keys, and if any, remove them
+            new_locals = set(user_ns.keys()) - old_locals
+            new_globals = set(user_global_ns.keys()) - old_globals
+            for k in new_locals:
+                del user_ns[k]
+            for k in new_globals:
+                del user_global_ns[k]
         # Undo the hack at creation of PrefilterFrontEnd
         from IPython import iplib
         iplib.InteractiveShell.isthreaded = False
@@ -97,7 +99,7 b' def test_execution():'
     """ Test execution of a command.
     """
     f = TestPrefilterFrontEnd()
-    f.input_buffer = 'print 1'
+    f.input_buffer = 'print(1)'
     f._on_enter()
     out_value = f.out.getvalue()
     assert_equal(out_value, '1\n')
@@ -228,7 +230,14 b' def test_completion_indexing():'
     f._on_enter()
     f.input_buffer = 'a[0].'
     f.complete_current_input()
-    assert_equal(f.input_buffer, 'a[0].__')
+
+    if sys.version_info[:2] >= (2,6):
+        # In Python 2.6, ints picked up a few non __ methods, so now there are
+        # no completions.
+        assert_equal(f.input_buffer, 'a[0].')
+    else:
+        # Right answer for 2.4/2.5
+        assert_equal(f.input_buffer, 'a[0].__')
 
 
 @isolate_ipython0
@@ -238,8 +247,13 b' def test_completion_equal():'
     f = TestPrefilterFrontEnd()
     f.input_buffer = 'a=1.'
     f.complete_current_input()
-    assert_equal(f.input_buffer, 'a=1.__')
-
+    if sys.version_info[:2] >= (2,6):
+        # In Python 2.6, ints picked up a few non __ methods, so now there are
+        # no completions.
+        assert_equal(f.input_buffer, 'a=1.')
+    else:
+        # Right answer for 2.4/2.5
+        assert_equal(f.input_buffer, 'a=1.__')
 
 
 if __name__ == '__main__':

IPython/genutils.py

@@ -880,7 +880,7 b' def doctest_reload():'
 
     This routine:
 
-      - reloads doctest
+      - imports doctest but does NOT reload it (see below).
 
       - resets its global 'master' attribute to None, so that multiple uses of
       the module interactively don't produce cumulative reports.
@@ -889,20 +889,20 b' def doctest_reload():'
       modified displayhook.  Doctest expects the default displayhook behavior
       deep down, so our modification breaks it completely.  For this reason, a
       hard monkeypatch seems like a reasonable solution rather than asking
-      users to manually use a different doctest runner when under IPython."""
+      users to manually use a different doctest runner when under IPython.
 
-    import doctest
-    reload(doctest)
-    doctest.master=None
+    Note
+    ----
 
-    try:
-        doctest.DocTestRunner
-    except AttributeError:
-        # This is only for python 2.3 compatibility, remove once we move to
-        # 2.4 only.
-        pass
-    else:
-        doctest.DocTestRunner.run = dhook_wrap(doctest.DocTestRunner.run)
+    This function *used to* reload doctest, but this has been disabled because
+    reloading doctest unconditionally can cause massive breakage of other
+    doctest-dependent modules already in memory, such as those for IPython's
+    own testing system.  The name wasn't changed to avoid breaking people's
+    code, but the reload call isn't actually made anymore."""
+
+    import doctest
+    doctest.master = None
+    doctest.DocTestRunner.run = dhook_wrap(doctest.DocTestRunner.run)
 
 #----------------------------------------------------------------------------
 class HomeDirError(Error):

IPython/kernel/core/ultraTB.py

@@ -90,7 +90,7 b' from inspect import getsourcefile, getfile, getmodule,\\'
 
 # IPython's own modules
 # Modified pdb which doesn't damage IPython's readline handling
-from IPython import Debugger, PyColorize
+from IPython import Debugger, PyColorize, ipapi
 from IPython.ipstruct import Struct
 from IPython.excolors import exception_colors
 from IPython.genutils import Term,uniq_stable,error,info
@@ -268,9 +268,7 b' def _formatTracebackLines(lnum, index, lines, Colors, lvals=None,scheme=None):'
     # This lets us get fully syntax-highlighted tracebacks.
     if scheme is None:
         try:
-            # Again, reference to a global __IPYTHON__ that doesn't exist.
-            # XXX
-            scheme = __IPYTHON__.rc.colors
+            scheme = ipapi.get().IP.rc.colors
         except:
             scheme = DEFAULT_SCHEME
     _line_format = _parser.format2
@@ -489,14 +487,10 b' class ListTB(TBTools):'
             else:
                 list.append('%s\n' % str(stype))
 
-        # This is being commented out for now as the __IPYTHON__ variable
-        # referenced here is not resolved and causes massive test failures
-        # and errors. B. Granger, 04/2009. XXX
-        # See https://bugs.launchpad.net/bugs/362137
-        # # vds:>>
-        # if have_filedata:
-        #     __IPYTHON__.hooks.synchronize_with_editor(filename, lineno, 0)
-        # # vds:<<
+        # vds:>>
+        if have_filedata:
+            ipapi.get().IP.hooks.synchronize_with_editor(filename, lineno, 0)
+        # vds:<<
 
         return list
 
@@ -810,17 +804,13 b' class VerboseTB(TBTools):'
                 value = text_repr(getattr(evalue, name))
                 exception.append('\n%s%s = %s' % (indent, name, value))
 
-        # This is being commented out for now as the __IPYTHON__ variable
-        # referenced here is not resolved and causes massive test failures
-        # and errors. B. Granger, 04/2009. XXX
-        # See https://bugs.launchpad.net/bugs/362137
-        # # vds: >>
-        # if records:
-        #      filepath, lnum = records[-1][1:3]
-        #      #print "file:", str(file), "linenb", str(lnum) # dbg
-        #      filepath = os.path.abspath(filepath)
-        #      __IPYTHON__.hooks.synchronize_with_editor(filepath, lnum, 0)
-        # # vds: <<
+        # vds: >>
+        if records:
+             filepath, lnum = records[-1][1:3]
+             #print "file:", str(file), "linenb", str(lnum) # dbg
+             filepath = os.path.abspath(filepath)
+             ipapi.get().IP.hooks.synchronize_with_editor(filepath, lnum, 0)
+        # vds: <<
                 
         # return all our info assembled as a single string
         return '%s\n\n%s\n%s' % (head,'\n'.join(frames),''.join(exception[0]) )

IPython/testing/iptest.py

@@ -50,6 +50,7 b' def test_for(mod):'
 
 have_curses = test_for('_curses')
 have_wx = test_for('wx')
+have_wx_aui = test_for('wx.aui')
 have_zi = test_for('zope.interface')
 have_twisted = test_for('twisted')
 have_foolscap = test_for('foolscap')
@@ -65,14 +66,15 b" EXCLUDE = [pjoin('IPython', 'external'),"
            pjoin('IPython_doctest_plugin'),
            pjoin('IPython', 'Gnuplot'),
            pjoin('IPython', 'Extensions', 'ipy_'),
-           pjoin('IPython', 'Extensions', 'clearcmd'),
+           pjoin('IPython', 'Extensions', 'PhysicalQInput'),
            pjoin('IPython', 'Extensions', 'PhysicalQInteractive'),
+           pjoin('IPython', 'Extensions', 'InterpreterPasteInput'),
            pjoin('IPython', 'Extensions', 'scitedirector'),
            pjoin('IPython', 'Extensions', 'numeric_formats'),
            pjoin('IPython', 'testing', 'attic'),
            pjoin('IPython', 'testing', 'tutils'),
            pjoin('IPython', 'testing', 'tools'),
-           pjoin('IPython', 'testing', 'mkdoctests')
+           pjoin('IPython', 'testing', 'mkdoctests'),
            ]
 
 if not have_wx:
@@ -80,6 +82,9 b' if not have_wx:'
     EXCLUDE.append(pjoin('IPython', 'gui'))
     EXCLUDE.append(pjoin('IPython', 'frontend', 'wx'))
 
+if not have_wx_aui:
+    EXCLUDE.append(pjoin('IPython', 'gui', 'wx', 'wxIPython'))
+
 if not have_objc:
     EXCLUDE.append(pjoin('IPython', 'frontend', 'cocoa'))
 
@@ -258,7 +263,7 b' def run_iptestall():'
     t_start = time.time()
     for name,runner in runners.iteritems():
         print '*'*77
-        print 'IPython test set:',name
+        print 'IPython test group:',name
         res = runner.run()
         if res:
             failed[name] = res
@@ -269,14 +274,14 b' def run_iptestall():'
     # summarize results
     print
     print '*'*77
-    print 'Ran %s test sets in %.3fs' % (nrunners, t_tests)
+    print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
     print
     if not failed:
         print 'OK'
     else:
         # If anything went wrong, point out what command to rerun manually to
         # see the actual errors and individual summary
-        print 'ERROR - %s out of %s test sets failed.' % (nfail, nrunners)
+        print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
         for name in failed:
             failed_runner = runners[name]
             print '-'*40
@@ -297,4 +302,4 b' def main():'
 
 
 if __name__ == '__main__':
-    main()
  No newline at end of file
+    main()

IPython/testing/plugin/ipdoctest.py

@@ -207,6 +207,15 b' def start_ipython():'
     _ip.IP.magic_run_ori = _ip.IP.magic_run
     _ip.IP.magic_run = im
 
+    # XXX - For some very bizarre reason, the loading of %history by default is
+    # failing.  This needs to be fixed later, but for now at least this ensures
+    # that tests that use %hist run to completion.
+    from IPython import history
+    history.init_ipython(_ip)
+    if not hasattr(_ip.IP,'magic_history'):
+        raise RuntimeError("Can't load magics, aborting")
+
+
 # The start call MUST be made here.  I'm not sure yet why it doesn't work if
 # it is made later, at plugin initialization time, but in all my tests, that's
 # the case.

IPython/tests/test_magic.py

@@ -25,14 +25,14 b' def test_rehashx():'
     _ip.magic('rehashx')
     # Practically ALL ipython development systems will have more than 10 aliases
 
-    assert len(_ip.IP.alias_table) > 10
+    yield (nt.assert_true, len(_ip.IP.alias_table) > 10)
     for key, val in _ip.IP.alias_table.items():
         # we must strip dots from alias names
-        assert '.' not in key
+        nt.assert_true('.' not in key)
 
     # rehashx must fill up syscmdlist
     scoms = _ip.db['syscmdlist']
-    assert len(scoms) > 10
+    yield (nt.assert_true, len(scoms) > 10)
 
 
 def doctest_hist_f():
@@ -42,7 +42,8 b' def doctest_hist_f():'
 
     In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
 
-    In [11]: %history -n -f $tfile 3
+    In [11]: %hist -n -f $tfile 3
+
     """
 
 
@@ -51,9 +52,12 b' def doctest_hist_r():'
 
     XXX - This test is not recording the output correctly.  Not sure why...
 
+    In [20]: 'hist' in _ip.IP.lsmagic()
+    Out[20]: True
+
     In [6]: x=1
 
-    In [7]: hist -n -r 2
+    In [7]: %hist -n -r 2
     x=1  # random
     hist -n -r 2  # random
     """
@@ -94,12 +98,13 b' def test_shist():'
     
 @dec.skipif_not_numpy
 def test_numpy_clear_array_undec():
+    from IPython.Extensions import clearcmd
+
     _ip.ex('import numpy as np')
     _ip.ex('a = np.empty(2)')
-    
-    yield nt.assert_true,'a' in _ip.user_ns
+    yield (nt.assert_true, 'a' in _ip.user_ns)
     _ip.magic('clear array')
-    yield nt.assert_false,'a' in _ip.user_ns
+    yield (nt.assert_false, 'a' in _ip.user_ns)
     
 
 @dec.skip()

IPython/ultraTB.py

@@ -90,7 +90,7 b' from inspect import getsourcefile, getfile, getmodule,\\'
 
 # IPython's own modules
 # Modified pdb which doesn't damage IPython's readline handling
-from IPython import Debugger, PyColorize
+from IPython import Debugger, PyColorize, ipapi
 from IPython.ipstruct import Struct
 from IPython.excolors import exception_colors
 from IPython.genutils import Term,uniq_stable,error,info
@@ -268,7 +268,7 b' def _formatTracebackLines(lnum, index, lines, Colors, lvals=None,scheme=None):'
     # This lets us get fully syntax-highlighted tracebacks.
     if scheme is None:
         try:
-            scheme = __IPYTHON__.rc.colors
+            scheme = ipapi.get().IP.rc.colors
         except:
             scheme = DEFAULT_SCHEME
     _line_format = _parser.format2
@@ -489,7 +489,7 b' class ListTB(TBTools):'
 
         # vds:>>
         if have_filedata:
-            __IPYTHON__.hooks.synchronize_with_editor(filename, lineno, 0)
+            ipapi.get().IP.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds:<<
 
         return list
@@ -809,7 +809,7 b' class VerboseTB(TBTools):'
              filepath, lnum = records[-1][1:3]
              #print "file:", str(file), "linenb", str(lnum) # dbg
              filepath = os.path.abspath(filepath)
-             __IPYTHON__.hooks.synchronize_with_editor(filepath, lnum, 0)
+             ipapi.get().IP.hooks.synchronize_with_editor(filepath, lnum, 0)
         # vds: <<
                 
         # return all our info assembled as a single string

docs/examples/kernel/task2.py

@@ -3,6 +3,8 b''
 
 from IPython.kernel import client
 import time
+import sys
+flush = sys.stdout.flush
 
 tc = client.TaskClient()
 mec = client.MultiEngineClient()
@@ -16,6 +18,7 b' for i in range(6):'
     time.sleep(1.0)
     print "Queue status (vebose=False)"
     print tc.queue_status()
+    flush()
     
 for i in range(24):
     tc.run(client.StringTask('time.sleep(1)'))
@@ -24,12 +27,14 b' for i in range(6):'
     time.sleep(1.0)
     print "Queue status (vebose=True)"
     print tc.queue_status(True)
+    flush()
 
 for i in range(12):
     tc.run(client.StringTask('time.sleep(2)'))
 
 print "Queue status (vebose=True)"
 print tc.queue_status(True)
+flush()
 
 qs = tc.queue_status(True)
 sched = qs['scheduled']
@@ -41,4 +46,5 b' for i in range(6):'
     time.sleep(1.0)
     print "Queue status (vebose=True)"
     print tc.queue_status(True)
+    flush()