Merging -r 1177 from lp:ipython with fixes and resolutions.

The main conflicts I had to fix were in ultratb.  I have removed
the ultraTB.py in IPython/kernel/core.  Now IPython/core/ultratb.py
is being used everywhere.  Also I have protected the calls to ipapi.get
to see if None is returned.  This happens when trial IPython.kernel
is run.

IPython/core/tests/test_magic.py

@@ -25,14 +25,14 b' def test_rehashx():'
     _ip.magic('rehashx')
     # Practically ALL ipython development systems will have more than 10 aliases
 
-    assert len(_ip.IP.alias_table) > 10
+    yield (nt.assert_true, len(_ip.IP.alias_table) > 10)
     for key, val in _ip.IP.alias_table.items():
         # we must strip dots from alias names
-        assert '.' not in key
+        nt.assert_true('.' not in key)
 
     # rehashx must fill up syscmdlist
     scoms = _ip.db['syscmdlist']
-    assert len(scoms) > 10
+    yield (nt.assert_true, len(scoms) > 10)
 
 
 def doctest_hist_f():
@@ -42,7 +42,8 b' def doctest_hist_f():'
 
     In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
 
-    In [11]: %history -n -f $tfile 3
+    In [11]: %hist -n -f $tfile 3
+
     """
 
 
@@ -51,9 +52,12 b' def doctest_hist_r():'
 
     XXX - This test is not recording the output correctly.  Not sure why...
 
+    In [20]: 'hist' in _ip.IP.lsmagic()
+    Out[20]: True
+
     In [6]: x=1
 
-    In [7]: hist -n -r 2
+    In [7]: %hist -n -r 2
     x=1  # random
     hist -n -r 2  # random
     """
@@ -94,12 +98,13 b' def test_shist():'
     
 @dec.skipif_not_numpy
 def test_numpy_clear_array_undec():
+    from IPython.extensions import clearcmd
+
     _ip.ex('import numpy as np')
     _ip.ex('a = np.empty(2)')
-    
-    yield nt.assert_true,'a' in _ip.user_ns
+    yield (nt.assert_true, 'a' in _ip.user_ns)
     _ip.magic('clear array')
-    yield nt.assert_false,'a' in _ip.user_ns
+    yield (nt.assert_false, 'a' in _ip.user_ns)
     
 
 @dec.skip()

IPython/core/ultratb.py

@@ -91,7 +91,7 b' from inspect import getsourcefile, getfile, getmodule,\\'
 # IPython's own modules
 # Modified pdb which doesn't damage IPython's readline handling
 from IPython.utils import PyColorize
-from IPython.core import debugger
+from IPython.core import debugger, ipapi
 from IPython.utils.ipstruct import Struct
 from IPython.core.excolors import exception_colors
 from IPython.utils.genutils import Term,uniq_stable,error,info
@@ -268,10 +268,12 b' def _formatTracebackLines(lnum, index, lines, Colors, lvals=None,scheme=None):'
 
     # This lets us get fully syntax-highlighted tracebacks.
     if scheme is None:
-        try:
-            scheme = __IPYTHON__.rc.colors
-        except:
+        ipinst = ipapi.get()
+        if ipinst is not None:
+            scheme = ipinst.IP.rc.colors
+        else:
             scheme = DEFAULT_SCHEME
+
     _line_format = _parser.format2
 
     for line in lines:
@@ -490,7 +492,9 b' class ListTB(TBTools):'
 
         # vds:>>
         if have_filedata:
-            __IPYTHON__.hooks.synchronize_with_editor(filename, lineno, 0)
+            ipinst = ipapi.get()
+            if ipinst is not None:
+                ipinst.IP.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds:<<
 
         return list
@@ -810,7 +814,9 b' class VerboseTB(TBTools):'
              filepath, lnum = records[-1][1:3]
              #print "file:", str(file), "linenb", str(lnum) # dbg
              filepath = os.path.abspath(filepath)
-             __IPYTHON__.hooks.synchronize_with_editor(filepath, lnum, 0)
+             ipinst = ipapi.get()
+             if ipinst is not None:
+                 ipinst.IP.hooks.synchronize_with_editor(filepath, lnum, 0)
         # vds: <<
                 
         # return all our info assembled as a single string

IPython/frontend/tests/test_prefilterfrontend.py

@@ -15,6 +15,7 b' __docformat__ = "restructuredtext en"'
 from copy import copy, deepcopy
 from cStringIO import StringIO
 import string
+import sys
 
 from nose.tools import assert_equal
 
@@ -23,22 +24,6 b' from IPython.core.ipapi import get as get_ipython0'
 from IPython.testing.plugin.ipdoctest import default_argv
 
 
-def safe_deepcopy(d):
-    """ Deep copy every key of the given dict, when possible. Elsewhere
-        do a copy.
-    """
-    copied_d = dict()
-    for key, value in d.iteritems():
-        try:
-            copied_d[key] = deepcopy(value)
-        except:
-            try:
-                copied_d[key] = copy(value)
-            except:
-                copied_d[key] = value
-    return copied_d
-
-
 class TestPrefilterFrontEnd(PrefilterFrontEnd):
     
     input_prompt_template = string.Template('')
@@ -72,17 +57,34 b' def isolate_ipython0(func):'
         with arguments.
     """
     def my_func():
-        iplib = get_ipython0()
-        if iplib is None:
+        ip0 = get_ipython0()
+        if ip0 is None:
             return func()
-        ipython0 = iplib.IP
-        global_ns = safe_deepcopy(ipython0.user_global_ns)
-        user_ns = safe_deepcopy(ipython0.user_ns)
+        # We have a real ipython running...
+        user_ns = ip0.IP.user_ns
+        user_global_ns = ip0.IP.user_global_ns
+
+        # Previously the isolation was attempted with a deep copy of the user
+        # dicts, but we found cases where this didn't work correctly. I'm not
+        # quite sure why, but basically it did damage the user namespace, such
+        # that later tests stopped working correctly.  Instead we use a simpler
+        # approach, just computing the list of added keys to the namespace and
+        # eliminating those afterwards.  Existing keys that may have been
+        # modified remain modified.  So far this has proven to be robust.
+
+        # Compute set of old local/global keys
+        old_locals = set(user_ns.keys())
+        old_globals = set(user_global_ns.keys())
         try:
             out = func()
         finally:
-            ipython0.user_ns = user_ns
-            ipython0.user_global_ns = global_ns
+            # Find new keys, and if any, remove them
+            new_locals = set(user_ns.keys()) - old_locals
+            new_globals = set(user_global_ns.keys()) - old_globals
+            for k in new_locals:
+                del user_ns[k]
+            for k in new_globals:
+                del user_global_ns[k]
         # Undo the hack at creation of PrefilterFrontEnd
         from IPython.core import iplib
         iplib.InteractiveShell.isthreaded = False
@@ -97,7 +99,7 b' def test_execution():'
     """ Test execution of a command.
     """
     f = TestPrefilterFrontEnd()
-    f.input_buffer = 'print 1'
+    f.input_buffer = 'print(1)'
     f._on_enter()
     out_value = f.out.getvalue()
     assert_equal(out_value, '1\n')
@@ -228,6 +230,13 b' def test_completion_indexing():'
     f._on_enter()
     f.input_buffer = 'a[0].'
     f.complete_current_input()
+
+    if sys.version_info[:2] >= (2,6):
+        # In Python 2.6, ints picked up a few non __ methods, so now there are
+        # no completions.
+        assert_equal(f.input_buffer, 'a[0].')
+    else:
+        # Right answer for 2.4/2.5
     assert_equal(f.input_buffer, 'a[0].__')
 
 
@@ -238,10 +247,15 b' def test_completion_equal():'
     f = TestPrefilterFrontEnd()
     f.input_buffer = 'a=1.'
     f.complete_current_input()
+    if sys.version_info[:2] >= (2,6):
+        # In Python 2.6, ints picked up a few non __ methods, so now there are
+        # no completions.
+        assert_equal(f.input_buffer, 'a=1.')
+    else:
+        # Right answer for 2.4/2.5
     assert_equal(f.input_buffer, 'a=1.__')
 
 
-
 if __name__ == '__main__':
     test_magic()
     test_help()

IPython/kernel/contexts.py

@@ -26,7 +26,7 b' import sys'
 
 from twisted.internet.error import ConnectionRefusedError
 
-from IPython.ultraTB import _fixed_getinnerframes, findsource
+from IPython.core.ultratb import _fixed_getinnerframes, findsource
 from IPython.core import ipapi
 
 from IPython.kernel import error

IPython/kernel/core/interpreter.py

@@ -29,7 +29,7 b' import sys'
 import traceback
 
 # Local imports.
-from IPython.kernel.core import ultraTB
+from IPython.core import ultratb
 from IPython.kernel.core.display_trap import DisplayTrap
 from IPython.kernel.core.macro import Macro
 from IPython.kernel.core.prompts import CachedOutput
@@ -167,9 +167,9 b' class Interpreter(object):'
             formatters=self.traceback_formatters)
 
         # This is used temporarily for reformating exceptions in certain
-        # cases.  It will go away once the ultraTB stuff is ported
+        # cases.  It will go away once the ultratb stuff is ported
         # to ipython1
-        self.tbHandler = ultraTB.FormattedTB(color_scheme='NoColor',
+        self.tbHandler = ultratb.FormattedTB(color_scheme='NoColor',
                                                  mode='Context',
                                                  tb_offset=2)
 

IPython/testing/iptest.py

@@ -50,6 +50,7 b' def test_for(mod):'
 
 have_curses = test_for('_curses')
 have_wx = test_for('wx')
+have_wx_aui = test_for('wx.aui')
 have_zi = test_for('zope.interface')
 have_twisted = test_for('twisted')
 have_foolscap = test_for('foolscap')
@@ -67,8 +68,9 b' def make_exclude():'
                pjoin('IPython', 'frontend', 'process', 'winprocess.py'),
                pjoin('IPython_doctest_plugin'),
                pjoin('IPython', 'extensions', 'ipy_'),
-               pjoin('IPython', 'extensions', 'clearcmd'),
+               pjoin('IPython', 'extensions', 'PhysicalQInput'),
                pjoin('IPython', 'extensions', 'PhysicalQInteractive'),
+               pjoin('IPython', 'extensions', 'InterpreterPasteInput'),
                pjoin('IPython', 'extensions', 'scitedirector'),
                pjoin('IPython', 'extensions', 'numeric_formats'),
                pjoin('IPython', 'testing', 'attic'),
@@ -81,6 +83,9 b' def make_exclude():'
         EXCLUDE.append(pjoin('IPython', 'gui'))
         EXCLUDE.append(pjoin('IPython', 'frontend', 'wx'))
 
+    if not have_wx_aui:
+        EXCLUDE.append(pjoin('IPython', 'gui', 'wx', 'wxIPython'))
+
     if not have_objc:
         EXCLUDE.append(pjoin('IPython', 'frontend', 'cocoa'))
 
@@ -111,6 +116,7 b' def make_exclude():'
 
     return EXCLUDE
 
+
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
@@ -237,7 +243,7 b' def run_iptestall():'
     t_start = time.time()
     for name,runner in runners.iteritems():
         print '*'*77
-        print 'IPython test set:', name
+        print 'IPython test group:',name
         res = runner.run()
         if res:
             failed[name] = res
@@ -248,14 +254,14 b' def run_iptestall():'
     # summarize results
     print
     print '*'*77
-    print 'Ran %s test sets in %.3fs' % (nrunners, t_tests)
+    print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
     print
     if not failed:
         print 'OK'
     else:
         # If anything went wrong, point out what command to rerun manually to
         # see the actual errors and individual summary
-        print 'ERROR - %s out of %s test sets failed.' % (nfail, nrunners)
+        print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
         for name in failed:
             failed_runner = runners[name]
             print '-'*40

IPython/testing/plugin/ipdoctest.py

@@ -208,6 +208,15 b' def start_ipython():'
     _ip.IP.magic_run_ori = _ip.IP.magic_run
     _ip.IP.magic_run = im
 
+    # XXX - For some very bizarre reason, the loading of %history by default is
+    # failing.  This needs to be fixed later, but for now at least this ensures
+    # that tests that use %hist run to completion.
+    from IPython.core import history
+    history.init_ipython(_ip)
+    if not hasattr(_ip.IP,'magic_history'):
+        raise RuntimeError("Can't load magics, aborting")
+
+
 # The start call MUST be made here.  I'm not sure yet why it doesn't work if
 # it is made later, at plugin initialization time, but in all my tests, that's
 # the case.

IPython/utils/genutils.py

@@ -881,7 +881,7 b' def doctest_reload():'
 
     This routine:
 
-      - reloads doctest
+      - imports doctest but does NOT reload it (see below).
 
       - resets its global 'master' attribute to None, so that multiple uses of
       the module interactively don't produce cumulative reports.
@@ -890,19 +890,19 b' def doctest_reload():'
       modified displayhook.  Doctest expects the default displayhook behavior
       deep down, so our modification breaks it completely.  For this reason, a
       hard monkeypatch seems like a reasonable solution rather than asking
-      users to manually use a different doctest runner when under IPython."""
+      users to manually use a different doctest runner when under IPython.
+
+    Note
+    ----
+
+    This function *used to* reload doctest, but this has been disabled because
+    reloading doctest unconditionally can cause massive breakage of other
+    doctest-dependent modules already in memory, such as those for IPython's
+    own testing system.  The name wasn't changed to avoid breaking people's
+    code, but the reload call isn't actually made anymore."""
 
     import doctest
-    reload(doctest)
     doctest.master=None
-
-    try:
-        doctest.DocTestRunner
-    except AttributeError:
-        # This is only for python 2.3 compatibility, remove once we move to
-        # 2.4 only.
-        pass
-    else:
         doctest.DocTestRunner.run = dhook_wrap(doctest.DocTestRunner.run)
 
 #----------------------------------------------------------------------------

docs/examples/kernel/task2.py

@@ -3,6 +3,8 b''
 
 from IPython.kernel import client
 import time
+import sys
+flush = sys.stdout.flush
 
 tc = client.TaskClient()
 mec = client.MultiEngineClient()
@@ -16,6 +18,7 b' for i in range(6):'
     time.sleep(1.0)
     print "Queue status (vebose=False)"
     print tc.queue_status()
+    flush()
     
 for i in range(24):
     tc.run(client.StringTask('time.sleep(1)'))
@@ -24,12 +27,14 b' for i in range(6):'
     time.sleep(1.0)
     print "Queue status (vebose=True)"
     print tc.queue_status(True)
+    flush()
 
 for i in range(12):
     tc.run(client.StringTask('time.sleep(2)'))
 
 print "Queue status (vebose=True)"
 print tc.queue_status(True)
+flush()
 
 qs = tc.queue_status(True)
 sched = qs['scheduled']
@@ -41,4 +46,5 b' for i in range(6):'
     time.sleep(1.0)
     print "Queue status (vebose=True)"
     print tc.queue_status(True)
+    flush()
 

docs/source/interactive/reference.txt

@@ -1329,8 +1329,9 b' For stand-alone use of the feature in your programs which do not use'
 IPython at all, put the following lines toward the top of your 'main'
 routine::
 
-    import sys,IPython.ultraTB  
-    sys.excepthook = IPython.ultraTB.FormattedTB(mode='Verbose',
+    import sys
+    from IPython.core import ultratb
+    sys.excepthook = ultratb.FormattedTB(mode='Verbose',
     color_scheme='Linux', call_pdb=1)
 
 The mode keyword can be either 'Verbose' or 'Plain', giving either very