More work addressing review comments for Fernando's branch.

* :mod:`IPython.testing.globalipapp` now directly creates a
  :class:`~IPython.core.iplib.InteractiveShell` instance by passing it a
  configuration object, rather than creating an IPython application.
* Updated everything in :mod:`IPython.frontend` and :mod:`IPython.gui` to use
  raw :class:`~IPython.core.iplib.InteractiveShell directly rather than
  creating an IPython application.
* Updated the IPython sphinx extension to use raw
  :class:`~IPython.core.iplib.InteractiveShell directly rather than creating
  an IPython application.
* Removed code from :mod:`IPython.extensions.pretty` that called
  :func:`get_ipython` (r1271).
* Addressed comment on (r1284) about holding refs to deferreds in
  :mod:`IPython.kernel.ipclusterapp`.
* Removed :mod:`IPython.kernel` from list of modules tested by nose in
  :mod:`IPython.testing.iptest`. (r1318)

IPython/core/iplib.py

@@ -191,64 +191,6 b' class SeparateStr(Str):'
         return super(SeparateStr, self).validate(obj, value)
 
 
-def make_user_namespaces(user_ns=None, user_global_ns=None):
-    """Return a valid local and global user interactive namespaces.
-
-    This builds a dict with the minimal information needed to operate as a
-    valid IPython user namespace, which you can pass to the various
-    embedding classes in ipython. The default implementation returns the
-    same dict for both the locals and the globals to allow functions to
-    refer to variables in the namespace. Customized implementations can
-    return different dicts. The locals dictionary can actually be anything
-    following the basic mapping protocol of a dict, but the globals dict
-    must be a true dict, not even a subclass. It is recommended that any
-    custom object for the locals namespace synchronize with the globals
-    dict somehow.
-
-    Raises TypeError if the provided globals namespace is not a true dict.
-
-    Parameters
-    ----------
-    user_ns : dict-like, optional
-        The current user namespace. The items in this namespace should
-        be included in the output. If None, an appropriate blank
-        namespace should be created.
-    user_global_ns : dict, optional
-        The current user global namespace. The items in this namespace
-        should be included in the output. If None, an appropriate
-        blank namespace should be created.
-
-    Returns
-    -------
-        A pair of dictionary-like object to be used as the local namespace
-        of the interpreter and a dict to be used as the global namespace.
-    """
-
-
-    # We must ensure that __builtin__ (without the final 's') is always
-    # available and pointing to the __builtin__ *module*.  For more details:
-    # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
-
-    if user_ns is None:
-        # Set __name__ to __main__ to better match the behavior of the
-        # normal interpreter.
-        user_ns = {'__name__'     :'__main__',
-                   '__builtin__' : __builtin__,
-                   '__builtins__' : __builtin__,
-                  }
-    else:
-        user_ns.setdefault('__name__','__main__')
-        user_ns.setdefault('__builtin__',__builtin__)
-        user_ns.setdefault('__builtins__',__builtin__)
-
-    if user_global_ns is None:
-        user_global_ns = user_ns
-    if type(user_global_ns) is not dict:
-        raise TypeError("user_global_ns must be a true dict; got %r"
-            % type(user_global_ns))
-
-    return user_ns, user_global_ns
-
 #-----------------------------------------------------------------------------
 # Main IPython class
 #-----------------------------------------------------------------------------
@@ -880,7 +822,7 b' class InteractiveShell(Component, Magic):'
         # These routines return properly built dicts as needed by the rest of
         # the code, and can also be used by extension writers to generate
         # properly initialized namespaces.
-        user_ns, user_global_ns = make_user_namespaces(user_ns, user_global_ns)
+        user_ns, user_global_ns = self.make_user_namespaces(user_ns, user_global_ns)
 
         # Assign namespaces
         # This is the namespace where all normal user variables live
@@ -940,6 +882,64 b' class InteractiveShell(Component, Magic):'
         self.ns_refs_table = [ user_ns, user_global_ns, self.user_config_ns,
                                self.internal_ns, self._main_ns_cache ]
 
+    def make_user_namespaces(self, user_ns=None, user_global_ns=None):
+        """Return a valid local and global user interactive namespaces.
+
+        This builds a dict with the minimal information needed to operate as a
+        valid IPython user namespace, which you can pass to the various
+        embedding classes in ipython. The default implementation returns the
+        same dict for both the locals and the globals to allow functions to
+        refer to variables in the namespace. Customized implementations can
+        return different dicts. The locals dictionary can actually be anything
+        following the basic mapping protocol of a dict, but the globals dict
+        must be a true dict, not even a subclass. It is recommended that any
+        custom object for the locals namespace synchronize with the globals
+        dict somehow.
+
+        Raises TypeError if the provided globals namespace is not a true dict.
+
+        Parameters
+        ----------
+        user_ns : dict-like, optional
+            The current user namespace. The items in this namespace should
+            be included in the output. If None, an appropriate blank
+            namespace should be created.
+        user_global_ns : dict, optional
+            The current user global namespace. The items in this namespace
+            should be included in the output. If None, an appropriate
+            blank namespace should be created.
+
+        Returns
+        -------
+            A pair of dictionary-like object to be used as the local namespace
+            of the interpreter and a dict to be used as the global namespace.
+        """
+
+
+        # We must ensure that __builtin__ (without the final 's') is always
+        # available and pointing to the __builtin__ *module*.  For more details:
+        # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
+
+        if user_ns is None:
+            # Set __name__ to __main__ to better match the behavior of the
+            # normal interpreter.
+            user_ns = {'__name__'     :'__main__',
+                       '__builtin__' : __builtin__,
+                       '__builtins__' : __builtin__,
+                      }
+        else:
+            user_ns.setdefault('__name__','__main__')
+            user_ns.setdefault('__builtin__',__builtin__)
+            user_ns.setdefault('__builtins__',__builtin__)
+
+        if user_global_ns is None:
+            user_global_ns = user_ns
+        if type(user_global_ns) is not dict:
+            raise TypeError("user_global_ns must be a true dict; got %r"
+                % type(user_global_ns))
+
+        return user_ns, user_global_ns
+
     def init_sys_modules(self):
         # We need to insert into sys.modules something that looks like a
         # module but which accesses the IPython namespace, for shelve and

IPython/extensions/pretty.py

@@ -128,9 +128,8 b' class PrettyResultDisplay(Component):'
 #-----------------------------------------------------------------------------
 
 
-def load_ipython_extension(ip=None):
+def load_ipython_extension(ip):
     """Load the extension in IPython as a hook."""
-    if ip is None: ip = get_ipython()
     global _loaded
     if not _loaded:
         prd = PrettyResultDisplay(ip, name='pretty_result_display')

IPython/frontend/linefrontendbase.py

@@ -39,9 +39,10 b' def common_prefix(strings):'
 
     return prefix
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Base class for the line-oriented front ends
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+
 class LineFrontEndBase(FrontEndBase):
     """ Concrete implementation of the FrontEndBase class. This is meant
     to be the base class behind all the frontend that are line-oriented,

IPython/frontend/prefilterfrontend.py

@@ -26,7 +26,7 b' import os'
 import re
 import __builtin__
 
-from IPython.core.ipapp import IPythonApp
+from IPython.core.iplib import InteractiveShell
 from IPython.kernel.core.redirector_output_trap import RedirectorOutputTrap
 
 from IPython.kernel.core.sync_traceback_trap import SyncTracebackTrap
@@ -50,9 +50,10 b' def mk_system_call(system_call_function, command):'
     my_system_call.__doc__ = "Calls %s" % command
     return my_system_call
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Frontend class using ipython0 to do the prefiltering. 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+
 class PrefilterFrontEnd(LineFrontEndBase):
     """ Class that uses ipython0 to do prefilter the input, do the
     completion and the magics.
@@ -65,19 +66,13 b' class PrefilterFrontEnd(LineFrontEndBase):'
 
     debug = False
     
-    def __init__(self, ipython0=None, argv=None, *args, **kwargs):
+    def __init__(self, ipython0=None, *args, **kwargs):
         """ Parameters
             ----------
 
             ipython0: an optional ipython0 instance to use for command
             prefiltering and completion.
-
-            argv : list, optional
-              Used as the instance's argv value.  If not given, [] is used.
         """
-        if argv is None:
-            argv = ['--no-banner']
-
         LineFrontEndBase.__init__(self, *args, **kwargs)
         self.shell.output_trap = RedirectorOutputTrap(
                             out_callback=self.write,
@@ -90,22 +85,19 b' class PrefilterFrontEnd(LineFrontEndBase):'
         # Start the ipython0 instance:
         self.save_output_hooks()
         if ipython0 is None:
-            # Instanciate an IPython0 interpreter to be able to use the
+            # Instanciate an IPython0 InteractiveShell to be able to use the
             # prefiltering.
             # Suppress all key input, to avoid waiting
             def my_rawinput(x=None):
                 return '\n'
             old_rawinput = __builtin__.raw_input
             __builtin__.raw_input = my_rawinput
-            ipython0 = IPythonApp(argv=argv, 
-                                  user_ns=self.shell.user_ns,
-                                  user_global_ns=self.shell.user_global_ns)
-            ipython0.initialize()
+            ipython0 = InteractiveShell(
+                parent=None, user_ns=self.shell.user_ns,
+                user_global_ns=self.shell.user_global_ns
+            )
             __builtin__.raw_input = old_rawinput
-        # XXX This will need to be updated as we refactor things, but for now,
-        # the .shell attribute of the ipythonapp instance conforms to the old
-        # api.
-        self.ipython0 = ipython0.shell
+        self.ipython0 = ipython0
         # Set the pager:
         self.ipython0.set_hook('show_in_pager', 
                     lambda s, string: self.write("\n" + string))

IPython/frontend/tests/test_asyncfrontendbase.py

@@ -1,14 +1,11 b''
 # encoding: utf-8
-
 """This file contains unittests for the asyncfrontendbase module."""
 
-__docformat__ = "restructuredtext en"
-                                                                            
 # Tell nose to skip this module
 __test__ = {}
 
 #---------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team                         
+#  Copyright (C) 2008-2009  The IPython Development Team                         
 #                                                                           
 #  Distributed under the terms of the BSD License.  The full license is in  
 #  the file COPYING, distributed as part of this software.                  

IPython/frontend/tests/test_prefilterfrontend.py

@@ -21,7 +21,6 b' from nose.tools import assert_equal'
 
 from IPython.frontend.prefilterfrontend import PrefilterFrontEnd
 from IPython.testing.globalipapp import get_ipython
-from IPython.testing.tools import default_argv
 
 #-----------------------------------------------------------------------------
 # Support utilities
@@ -35,7 +34,7 b' class TestPrefilterFrontEnd(PrefilterFrontEnd):'
 
     def __init__(self):
         self.out = StringIO()
-        PrefilterFrontEnd.__init__(self,argv=default_argv())
+        PrefilterFrontEnd.__init__(self)
         # Some more code for isolation (yeah, crazy)
         self._on_enter()
         self.out.flush()

IPython/frontend/wx/console_widget.py

@@ -135,9 +135,10 b' else:'
                 }
  
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # The console widget class
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+
 class ConsoleWidget(editwindow.EditWindow):
     """ Specialized styled text control view for console-like workflow.
 

IPython/frontend/wx/ipythonx.py

@@ -47,7 +47,7 b' class IPythonXController(WxController):'
         self._input_state = 'subprocess'
         self.write('\n', refresh=False)
         self.capture_output()
-        self.ipython0.shell.exit()
+        self.ipython0.exit()
         self.release_output()
         if not self.ipython0.exit_now:
             wx.CallAfter(self.new_prompt,

IPython/gui/wx/ipshell_nonblocking.py

@@ -23,8 +23,7 b' import os'
 import locale
 from thread_ex import ThreadEx
 
-import IPython
-from IPython.core import iplib, ipapp
+from IPython.core import iplib
 from IPython.utils.io import Term
 
 ##############################################################################
@@ -88,12 +87,10 b' class NonBlockingIPShell(object):'
           via raise_exc()
     '''
 
-    def __init__(self, argv=[], user_ns={}, user_global_ns=None,
+    def __init__(self, user_ns={}, user_global_ns=None,
                  cin=None, cout=None, cerr=None,
                  ask_exit_handler=None):
         '''
-        @param argv: Command line options for IPython
-        @type argv: list
         @param user_ns: User namespace.
         @type user_ns: dictionary
         @param user_global_ns: User global namespace.
@@ -111,7 +108,7 b' class NonBlockingIPShell(object):'
         '''
         #ipython0 initialisation
         self._IP = None
-        self.init_ipython0(argv, user_ns, user_global_ns,
+        self.init_ipython0(user_ns, user_global_ns,
                           cin, cout, cerr,
                           ask_exit_handler)
         
@@ -131,7 +128,7 b' class NonBlockingIPShell(object):'
         self._help_text = None
         self._add_button = None
 
-    def init_ipython0(self, argv=[], user_ns={}, user_global_ns=None,
+    def init_ipython0(self, user_ns={}, user_global_ns=None,
                      cin=None, cout=None, cerr=None,
                      ask_exit_handler=None):
         ''' Initialize an ipython0 instance '''
@@ -151,17 +148,12 b' class NonBlockingIPShell(object):'
 
         #Hack to save sys.displayhook, because ipython seems to overwrite it...
         self.sys_displayhook_ori = sys.displayhook
-
-        ipython0 = ipapp.IPythonApp(argv,user_ns=user_ns,
-                                    user_global_ns=user_global_ns)
-        ipython0.initialize()
-        self._IP = ipython0.shell
-        
-        ## self._IP = IPython.shell.make_IPython(
-        ##                             argv,user_ns=user_ns,
-        ##                             user_global_ns=user_global_ns,
-        ##                             embedded=True,
-        ##                             shell_class=IPython.shell.InteractiveShell)
+        ipython0 = iplib.InteractiveShell(
+            parent=None, config=None,
+            user_ns=user_ns,
+            user_global_ns=user_global_ns
+        )
+        self._IP = ipython0
 
         #we save ipython0 displayhook and we restore sys.displayhook
         self.displayhook = sys.displayhook
@@ -185,12 +177,10 b' class NonBlockingIPShell(object):'
         #we replace the help command
         self._IP.user_ns['help'] = _Helper(self._pager_help)
 
-        #we disable cpase magic... until we found a way to use it properly.
-        from IPython.core import ipapi
-        ip = ipapi.get()
+        #we disable cpaste magic... until we found a way to use it properly.
         def bypass_magic(self, arg):
             print '%this magic is currently disabled.'
-        ip.define_magic('cpaste', bypass_magic)
+        ipython0.define_magic('cpaste', bypass_magic)
 
         import __builtin__
         __builtin__.raw_input = self._raw_input

IPython/gui/wx/ipython_history.py

@@ -11,6 +11,7 b' __author__ = "Laurent Dufrechou"'
 __email__   = "laurent.dufrechou _at_ gmail.com"
 __license__ = "BSD"
 #-----------------------------------------
+
 class IPythonHistoryPanel(wx.Panel):
 
     def __init__(self, parent,flt_empty=True,

IPython/kernel/ipclusterapp.py

@@ -361,8 +361,11 b' class IPClusterApp(ApplicationWithClusterDir):'
                 log.msg('Unexpected error in ipcluster:')
                 log.msg(r.getTraceback())
             log.msg("IPython cluster: stopping")
-            self.stop_engines()
-            self.stop_controller()
+            # These return deferreds. We are not doing anything with them
+            # but we are holding refs to them as a reminder that they 
+            # do return deferreds.
+            d1 = self.stop_engines()
+            d2 = self.stop_controller()
             # Wait a few seconds to let things shut down.
             reactor.callLater(4.0, reactor.stop)
 

IPython/lib/deepreload.py

@@ -1,15 +1,18 b''
 # -*- coding: utf-8 -*-
 """
 A module to change reload() so that it acts recursively.
-To enable it type:
-    >>> import __builtin__, deepreload
-    >>> __builtin__.reload = deepreload.reload
+To enable it type::
 
-You can then disable it with:
-    >>> __builtin__.reload = deepreload.original_reload
+    import __builtin__, deepreload
+    __builtin__.reload = deepreload.reload
     
-Alternatively, you can add a dreload builtin alongside normal reload with:
-    >>> __builtin__.dreload = deepreload.reload
+You can then disable it with::
+
+    __builtin__.reload = deepreload.original_reload
+    
+Alternatively, you can add a dreload builtin alongside normal reload with::
+
+    __builtin__.dreload = deepreload.reload
     
 This code is almost entirely based on knee.py from the standard library.
 """

IPython/testing/globalipapp.py

@@ -103,6 +103,7 b' def get_ipython():'
     # This will get replaced by the real thing once we start IPython below
     return start_ipython()
 
+
 def start_ipython():
     """Start a global IPython shell, which we need for IPython-specific syntax.
     """
@@ -114,7 +115,7 b' def start_ipython():'
     start_ipython.already_called = True
 
     # Ok,  first time we're called, go ahead
-    from IPython.core import ipapp, iplib
+    from IPython.core import iplib
     
     def xsys(cmd):
         """Execute a command and print its output.
@@ -132,26 +133,27 b' def start_ipython():'
     _main = sys.modules.get('__main__')
 
     # Create custom argv and namespaces for our IPython to be test-friendly
-    argv = tools.default_argv()
-    user_ns, global_ns = iplib.make_user_namespaces(ipnsdict(), {})
+    config = tools.default_config()
     
     # Create and initialize our test-friendly IPython instance.
-    ip = ipapp.IPythonApp(argv, user_ns=user_ns, user_global_ns=global_ns)
-    ip.initialize()
+    shell = iplib.InteractiveShell(
+        parent=None, config=config, 
+        user_ns=ipnsdict(), user_global_ns={}
+    )
 
     # A few more tweaks needed for playing nicely with doctests...
     
     # These traps are normally only active for interactive use, set them
     # permanently since we'll be mocking interactive sessions.
-    ip.shell.builtin_trap.set()
+    shell.builtin_trap.set()
 
     # Set error printing to stdout so nose can doctest exceptions
-    ip.shell.InteractiveTB.out_stream = 'stdout'
+    shell.InteractiveTB.out_stream = 'stdout'
     
     # Modify the IPython system call with one that uses getoutput, so that we
     # can capture subcommands and print them to Python's stdout, otherwise the
     # doctest machinery would miss them.
-    ip.shell.system = xsys
+    shell.system = xsys
 
     # IPython is ready, now clean up some global state...
     
@@ -164,7 +166,7 b' def start_ipython():'
     # So that ipython magics and aliases can be doctested (they work by making
     # a call into a global _ip object).  Also make the top-level get_ipython
     # now return this without recursively calling here again.
-    _ip = ip.shell
+    _ip = shell
     get_ipython = _ip.get_ipython
     __builtin__._ip = _ip
     __builtin__.get_ipython = get_ipython

IPython/testing/iptest.py

@@ -343,9 +343,9 b' def make_runners():'
 
     # And add twisted ones if conditions are met
     if have['zope.interface'] and have['twisted'] and have['foolscap']:
-        # Note that we list the kernel here, though the bulk of it is
-        # twisted-based, because nose picks up doctests that twisted doesn't.
-        nose_pkg_names.append('kernel')
+        # We only list IPython.kernel for testing using twisted.trial as
+        # nose and twisted.trial have conflicts that make the testing system
+        # unstable.
         trial_pkg_names.append('kernel')
 
     # For debugging this code, only load quick stuff

IPython/testing/tools.py

@@ -41,6 +41,7 b' try:'
 except ImportError:
     has_nose = False
 
+from IPython.config.loader import Config
 from IPython.utils.process import find_cmd, getoutputerror
 from IPython.utils.text import list_strings
 from IPython.utils.io import temp_pyfile
@@ -167,6 +168,15 b' def default_argv():'
             '--autocall=0']
     
 
+def default_config():
+    """Return a config object with good defaults for testing."""
+    config = Config()
+    config.InteractiveShell.colors = 'NoColor'
+    config.InteractiveShell.term_title = False,
+    config.InteractiveShell.autocall = 0
+    return config
+
+
 def ipexec(fname, options=None):
     """Utility to call 'ipython filename'.
 

IPython/utils/path.py

@@ -249,9 +249,7 b' def get_ipython_dir():'
     and the adds .ipython to the end of the path.
     """
     ipdir_def = '.ipython'
-    print get_home_dir
     home_dir = get_home_dir()
-    print home_dir
     #import pdb; pdb.set_trace()  # dbg
     ipdir = os.environ.get(
         'IPYTHON_DIR', os.environ.get(

docs/source/development/testing.txt

@@ -244,7 +244,7 b' and an example of ``# all-random``::'
 
 When writing docstrings, you can use the ``@skip_doctest`` decorator to
 indicate that a docstring should *not* be treated as a doctest at all.  The
-difference betwee ``# all-random`` and ``@skip_doctest`` is that the former
+difference between ``# all-random`` and ``@skip_doctest`` is that the former
 executes the example but ignores output, while the latter doesn't execute any
 code.  ``@skip_doctest`` should be used for docstrings whose examples are
 purely informational.

docs/sphinxext/ipython_directive.py

@@ -59,12 +59,9 b' Authors'
 
 # Stdlib
 import cStringIO
-import imp
 import os
 import re
-import shutil
 import sys
-import warnings
 
 # To keep compatibility with various python versions
 try:
@@ -80,8 +77,8 b' from docutils.parsers.rst import directives'
 matplotlib.use('Agg')
 
 # Our own
-from IPython import Config, IPythonApp
-from IPython.utils.io import Term, Tee
+from IPython import Config, InteractiveShell
+from IPython.utils.io import Term
 
 #-----------------------------------------------------------------------------
 # Globals
@@ -222,15 +219,11 b' class EmbeddedSphinxShell(object):'
         config.InteractiveShell.autoindent = False
         config.InteractiveShell.colors = 'NoColor'
 
-        # Merge global config which can be used to override.
-        config._merge(CONFIG)
-
         # Create and initialize ipython, but don't start its mainloop
-        IP = IPythonApp(override_config=config)
-        IP.initialize()
+        IP = InteractiveShell(parent=None, config=config)
 
         # Store a few parts of IPython we'll need.
-        self.IP = IP.shell
+        self.IP = IP
         self.user_ns = self.IP.user_ns
         self.user_global_ns = self.IP.user_global_ns