Continuing work on ipcontroller.

IPython/config/default/ipcontroller_config.py

@@ -0,0 +1,60 b''
+from IPython.config.loader import Config
+
+c = get_config()
+
+#-----------------------------------------------------------------------------
+# Global configuration
+#-----------------------------------------------------------------------------
+
+c.Global.logfile = ''
+c.Global.import_statement = ''
+c.Global.reuse_furls = False
+
+
+#-----------------------------------------------------------------------------
+# Configure the client services
+#-----------------------------------------------------------------------------
+
+c.FCClientServiceFactory.ip = ''
+c.FCClientServiceFactory.port = 0 
+c.FCClientServiceFactory.location = ''
+c.FCClientServiceFactory.secure =  True
+c.FCClientServiceFactory.cert_file = 'ipcontroller-client.pem'
+
+c.FCClientServiceFactory.Interfaces.Task.interface_chain = [
+    'IPython.kernel.task.ITaskController',
+    'IPython.kernel.taskfc.IFCTaskController'
+]
+c.FCClientServiceFactory.Interfaces.Task.furl_file = 'ipcontroller-tc.furl'
+
+c.FCClientServiceFactory.Interfaces.MultiEngine.interface_chain = [
+    'IPython.kernel.multiengine.IMultiEngine',
+    'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
+]
+c.FCClientServiceFactory.Interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl'
+
+
+#-----------------------------------------------------------------------------
+# Configure the engine services
+#-----------------------------------------------------------------------------
+
+c.FCEngineServiceFactory.ip = ''
+c.FCEngineServiceFactory.port = 0
+c.FCEngineServiceFactory.location = ''
+c.FCEngineServiceFactory.secure = True
+c.FCEngineServiceFactory.cert_file = 'ipcontroller-engine.pem'
+
+engine_config = Config()
+engine_config.furl_file = 
+c.Global.engine_furl_file = 'ipcontroller-engine.furl'
+c.Global.engine_fc_interface = 'IPython.kernel.enginefc.IFCControllerBase'
+
+
+
+
+
+CLIENT_INTERFACES = dict(
+    TASK = dict(FURL_FILE = 'ipcontroller-tc.furl'),
+    MULTIENGINE = dict(FURLFILE='ipcontroller-mec.furl')
+)
+

IPython/config/default/ipengine_config.py

@@ -0,0 +1,19 b''
+c = get_config()
+
+c.MPI.default = 'mpi4py'
+
+c.MPI.mpi4py = """from mpi4py import MPI as mpi
+mpi.size = mpi.COMM_WORLD.Get_size()
+mpi.rank = mpi.COMM_WORLD.Get_rank()
+"""
+
+c.MPI.pytrilinos = """from PyTrilinos import Epetra
+class SimpleStruct:
+pass
+mpi = SimpleStruct()
+mpi.rank = 0
+mpi.size = 0
+"""
+
+c.Global.logfile = ''
+c.Global.furl_file = 'ipcontroller-engine.furl'

IPython/kernel/configobjfactory.py

@@ -0,0 +1,75 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+A class for creating a Twisted service that is configured using IPython's
+configuration system.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import zope.interface as zi
+
+from IPython.core.component import Component
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+class IConfiguredObjectFactory(zi.Interface):
+    """I am a component that creates a configured object.
+
+    This class is useful if you want to configure a class that is not a 
+    subclass of :class:`IPython.core.component.Component`.
+    """
+
+    def __init__(config):
+        """Get ready to configure the object using config."""
+
+    def create():
+        """Return an instance of the configured object."""
+
+
+class ConfiguredObjectFactory(Component):
+
+    zi.implements(IConfiguredObjectFactory)
+
+    def __init__(self, config):
+        super(ConfiguredObjectFactory, self).__init__(None, config=config)
+
+    def create(self):
+        raise NotImplementedError('create must be implemented in a subclass')
+
+
+class IAdaptedConfiguredObjectFactory(zi.Interface):
+    """I am a component that adapts and configures an object.
+
+    This class is useful if you have the adapt a instance and configure it.
+    """
+
+    def __init__(config, adaptee=None):
+        """Get ready to adapt adaptee and then configure it using config."""
+
+    def create():
+        """Return an instance of the adapted and configured object."""
+
+
+class AdaptedConfiguredObjectFactory(Component):
+
+    # zi.implements(IAdaptedConfiguredObjectFactory)
+
+    def __init__(self, config, adaptee):
+        super(AdaptedConfiguredObjectFactory, self).__init__(None, config=config)
+        self.adaptee = adaptee
+
+    def create(self):
+        raise NotImplementedError('create must be implemented in a subclass')
  No newline at end of file

IPython/kernel/scripts/ipcontrollerapp.py

@@ -0,0 +1,266 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+The IPython controller application
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import copy
+import os
+import sys
+
+from twisted.application import service
+from twisted.internet import reactor, defer
+from twisted.python import log
+
+from IPython.config.loader import Config, NoConfigDefault
+
+from IPython.core.application import Application, IPythonArgParseConfigLoader
+from IPython.core import release
+
+from IPython.utils.traitlets import Int, Str, Bool, Instance
+from IPython.utils.importstring import import_item
+
+from IPython.kernel import controllerservice
+from IPython.kernel.configobjfactory import (
+    ConfiguredObjectFactory,
+    AdaptedConfiguredObjectFactory
+)
+
+from IPython.kernel.fcutil import FCServiceFactory
+
+#-----------------------------------------------------------------------------
+# Components for creating services
+#-----------------------------------------------------------------------------
+
+
+# The default client interfaces for FCClientServiceFactory.Interfaces
+default_client_interfaces = Config()
+default_client_interfaces.Task.interface_chain = [
+    'IPython.kernel.task.ITaskController',
+    'IPython.kernel.taskfc.IFCTaskController'
+]
+default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
+default_client_interfaces.MultiEngine.interface_chain = [
+    'IPython.kernel.multiengine.IMultiEngine',
+    'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
+]
+default_client_interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl'
+
+# Make this a dict we can pass to Config.__init__ for the default
+default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))
+
+
+
+# The default engine interfaces for FCEngineServiceFactory.Interfaces
+default_engine_interfaces = Config()
+default_engine_interfaces.Default.interface_chain = [
+    'IPython.kernel.enginefc.IFCControllerBase'
+]
+default_engine_interfaces.Default.furl_file = 'ipcontroller-engine.furl'
+
+# Make this a dict we can pass to Config.__init__ for the default
+default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
+
+
+
+class FCClientServiceFactory(FCServiceFactory):
+    """A Foolscap implementation of the client services."""
+
+    cert_file = Str('ipcontroller-client.pem', config=True)
+    Interfaces = Instance(klass=Config, kw=default_client_interfaces,
+                          allow_none=False, config=True)
+
+
+class FCEngineServiceFactory(FCServiceFactory):
+    """A Foolscap implementation of the engine services."""
+
+    cert_file = Str('ipcontroller-engine.pem', config=True)
+    interfaces = Instance(klass=dict, kw=default_engine_interfaces,
+                          allow_none=False, config=True)
+
+
+#-----------------------------------------------------------------------------
+# The main application
+#-----------------------------------------------------------------------------
+
+
+cl_args = (
+    # Client config
+    (('--client-ip',), dict(
+        type=str, dest='FCClientServiceFactory.ip', default=NoConfigDefault,
+        help='The IP address or hostname the controller will listen on for client connections.',
+        metavar='FCClientServiceFactory.ip')
+    ),
+    (('--client-port',), dict(
+        type=int, dest='FCClientServiceFactory.port', default=NoConfigDefault,
+        help='The port the controller will listen on for client connections.',
+        metavar='FCClientServiceFactory.port')
+    ),
+    (('--client-location',), dict(
+        type=str, dest='FCClientServiceFactory.location', default=NoConfigDefault,
+        help='The hostname or ip that clients should connect to.',
+        metavar='FCClientServiceFactory.location')
+    ),
+    (('-x',), dict(
+        action='store_false', dest='FCClientServiceFactory.secure', default=NoConfigDefault,
+        help='Turn off all client security.')
+    ),
+    (('--client-cert-file',), dict(
+        type=str, dest='FCClientServiceFactory.cert_file', default=NoConfigDefault,
+        help='File to store the client SSL certificate in.',
+        metavar='FCClientServiceFactory.cert_file')
+    ),
+    (('--task-furl-file',), dict(
+        type=str, dest='FCClientServiceFactory.Interfaces.Task.furl_file', default=NoConfigDefault,
+        help='File to store the FURL in for task clients to connect with.',
+        metavar='FCClientServiceFactory.Interfaces.Task.furl_file')
+    ),
+    (('--multiengine-furl-file',), dict(
+        type=str, dest='FCClientServiceFactory.Interfaces.MultiEngine.furl_file', default=NoConfigDefault,
+        help='File to store the FURL in for multiengine clients to connect with.',
+        metavar='FCClientServiceFactory.Interfaces.MultiEngine.furl_file')
+    ),
+    # Engine config
+    (('--engine-ip',), dict(
+        type=str, dest='FCEngineServiceFactory.ip', default=NoConfigDefault,
+        help='The IP address or hostname the controller will listen on for engine connections.',
+        metavar='FCEngineServiceFactory.ip')
+    ),
+    (('--engine-port',), dict(
+        type=int, dest='FCEngineServiceFactory.port', default=NoConfigDefault,
+        help='The port the controller will listen on for engine connections.',
+        metavar='FCEngineServiceFactory.port')
+    ),
+    (('--engine-location',), dict(
+        type=str, dest='FCEngineServiceFactory.location', default=NoConfigDefault,
+        help='The hostname or ip that engines should connect to.',
+        metavar='FCEngineServiceFactory.location')
+    ),
+    (('-y',), dict(
+        action='store_false', dest='FCEngineServiceFactory.secure', default=NoConfigDefault,
+        help='Turn off all engine security.')
+    ),
+    (('--engine-cert-file',), dict(
+        type=str, dest='FCEngineServiceFactory.cert_file', default=NoConfigDefault,
+        help='File to store the client SSL certificate in.',
+        metavar='FCEngineServiceFactory.cert_file')
+    ),
+    (('--engine-furl-file',), dict(
+        type=str, dest='FCEngineServiceFactory.Interfaces.Default.furl_file', default=NoConfigDefault,
+        help='File to store the FURL in for engines to connect with.',
+        metavar='FCEngineServiceFactory.Interfaces.Default.furl_file')
+    ),
+    # Global config
+    (('-l','--logfile'), dict(
+        type=str, dest='Global.logfile', default=NoConfigDefault,
+        help='Log file name (default is stdout)',
+        metavar='Global.logfile')
+    ),
+    (('-r',), dict(
+        action='store_true', dest='Global.reuse_furls', default=NoConfigDefault,
+        help='Try to reuse all FURL files.')
+    )
+)
+
+
+class IPControllerAppCLConfigLoader(IPythonArgParseConfigLoader):
+
+    arguments = cl_args
+
+
+_default_config_file_name = 'ipcontroller_config.py'
+
+class IPControllerApp(Application):
+
+    name = 'ipcontroller'
+    config_file_name = _default_config_file_name
+
+    def create_default_config(self):
+        super(IPControllerApp, self).create_default_config()
+        self.default_config.Global.logfile = ''
+        self.default_config.Global.reuse_furls = False
+        self.default_config.Global.import_statements = []
+
+    def create_command_line_config(self):
+        """Create and return a command line config loader."""
+
+        return IPControllerAppCLConfigLoader(
+            description="Start an IPython controller",
+            version=release.version)
+
+    def construct(self):
+        # I am a little hesitant to put these into InteractiveShell itself.
+        # But that might be the place for them
+        sys.path.insert(0, '')
+
+        self.start_logging()
+        self.import_statements()
+        self.reuse_furls()
+
+        # Create the service hierarchy
+        self.main_service = service.MultiService()
+        # The controller service
+        controller_service = controllerservice.ControllerService()
+        controller_service.setServiceParent(self.main_service)
+        # The client tub and all its refereceables
+        csfactory = FCClientServiceFactory(self.master_config, controller_service)
+        client_service = csfactory.create()
+        client_service.setServiceParent(self.main_service)
+        # The engine tub
+        esfactory = FCEngineServiceFactory(self.master_config, controller_service)
+        engine_service = esfactory.create()
+        engine_service.setServiceParent(self.main_service)
+
+    def start_logging(self):
+        logfile = self.master_config.Global.logfile
+        if logfile:
+            logfile = logfile + str(os.getpid()) + '.log'
+            try:
+                openLogFile = open(logfile, 'w')
+            except:
+                openLogFile = sys.stdout
+        else:
+            openLogFile = sys.stdout
+        log.startLogging(openLogFile)
+
+    def import_statements(self):
+        statements = self.master_config.Global.import_statements
+        for s in statements:
+            try:
+                exec s in globals(), locals()
+            except:
+                log.msg("Error running import statement: %s" % s)
+
+    def reuse_furls(self):
+        # This logic might need to be moved into the components
+        # Delete old furl files unless the reuse_furls is set
+        reuse = self.master_config.Global.reuse_furls
+        # if not reuse:
+        #     paths = (
+        #         self.master_config.FCEngineServiceFactory.Interfaces.Default.furl_file,
+        #         self.master_config.FCClientServiceFactory.Interfaces.Task.furl_file,
+        #         self.master_config.FCClientServiceFactory.Interfaces.MultiEngine.furl_file
+        #     )
+        #     for p in paths:
+        #         if os.path.isfile(p):
+        #             os.remove(p)
+
+    def start_app(self):
+        # Start the controller service and set things running
+        self.main_service.startService()
+        reactor.run()
+
+if __name__ == '__main__':
+    app = IPControllerApp()
+    app.start()

docs/examples/core/new-embed.py

@@ -0,0 +1,17 b''
+# This shows how to use the new top-level embed function.  It is a simpler
+# API that manages the creation of the embedded shell.
+
+from IPython import embed
+
+a = 10
+b = 20
+
+embed('First time')
+
+c = 30
+d = 40
+
+try:
+    raise Exception('adsfasdf')
+except:
+    embed('The second time')

docs/source/development/ipgraph.txt

@@ -0,0 +1,59 b''
+====================================================
+Notes on code execution in :class:`InteractiveShell`
+====================================================
+
+Overview
+========
+
+This section contains information and notes about the code execution 
+system in :class:`InteractiveShell`.  This system needs to be refactored
+and we are keeping notes about this process here.
+
+Current design
+==============
+
+Here is a script that shows the relationships between the various
+methods in :class:`InteractiveShell` that manage code execution::
+
+    import networkx as nx
+    import matplotlib.pyplot as plt
+
+    exec_init_cmd = 'exec_init_cmd'
+    interact = 'interact'
+    runlines = 'runlines'
+    runsource = 'runsource'
+    runcode = 'runcode'
+    push_line = 'push_line'
+    mainloop = 'mainloop'
+    embed_mainloop = 'embed_mainloop'
+    ri = 'raw_input'
+    prefilter = 'prefilter'
+
+    g = nx.DiGraph()
+
+    g.add_node(exec_init_cmd)
+    g.add_node(interact)
+    g.add_node(runlines)
+    g.add_node(runsource)
+    g.add_node(push_line)
+    g.add_node(mainloop)
+    g.add_node(embed_mainloop)
+    g.add_node(ri)
+    g.add_node(prefilter)
+
+    g.add_edge(exec_init_cmd, push_line)
+    g.add_edge(exec_init_cmd, prefilter)
+    g.add_edge(mainloop, exec_init_cmd)
+    g.add_edge(mainloop, interact)
+    g.add_edge(embed_mainloop, interact)
+    g.add_edge(interact, ri)
+    g.add_edge(interact, push_line)
+    g.add_edge(push_line, runsource)
+    g.add_edge(runlines, push_line)
+    g.add_edge(runlines, prefilter)
+    g.add_edge(runsource, runcode)
+    g.add_edge(ri, prefilter)
+
+    nx.draw_spectral(g, node_size=100, alpha=0.6, node_color='r',
+                     font_size=10, node_shape='o')
+    plt.show()

docs/source/development/index.txt

@@ -15,4 +15,5 b" IPython developer's guide"
    roadmap.txt
    reorg.txt
    notification_blueprint.txt
+   ipgraph.txt