diff --git a/IPython/config/default/ipcontroller_config.py b/IPython/config/default/ipcontroller_config.py new file mode 100644 index 0000000..c4e0d8f --- /dev/null +++ b/IPython/config/default/ipcontroller_config.py @@ -0,0 +1,60 @@ +from IPython.config.loader import Config + +c = get_config() + +#----------------------------------------------------------------------------- +# Global configuration +#----------------------------------------------------------------------------- + +c.Global.logfile = '' +c.Global.import_statement = '' +c.Global.reuse_furls = False + + +#----------------------------------------------------------------------------- +# Configure the client services +#----------------------------------------------------------------------------- + +c.FCClientServiceFactory.ip = '' +c.FCClientServiceFactory.port = 0 +c.FCClientServiceFactory.location = '' +c.FCClientServiceFactory.secure = True +c.FCClientServiceFactory.cert_file = 'ipcontroller-client.pem' + +c.FCClientServiceFactory.Interfaces.Task.interface_chain = [ + 'IPython.kernel.task.ITaskController', + 'IPython.kernel.taskfc.IFCTaskController' +] +c.FCClientServiceFactory.Interfaces.Task.furl_file = 'ipcontroller-tc.furl' + +c.FCClientServiceFactory.Interfaces.MultiEngine.interface_chain = [ + 'IPython.kernel.multiengine.IMultiEngine', + 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine' +] +c.FCClientServiceFactory.Interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl' + + +#----------------------------------------------------------------------------- +# Configure the engine services +#----------------------------------------------------------------------------- + +c.FCEngineServiceFactory.ip = '' +c.FCEngineServiceFactory.port = 0 +c.FCEngineServiceFactory.location = '' +c.FCEngineServiceFactory.secure = True +c.FCEngineServiceFactory.cert_file = 'ipcontroller-engine.pem' + +engine_config = Config() +engine_config.furl_file = +c.Global.engine_furl_file = 'ipcontroller-engine.furl' +c.Global.engine_fc_interface = 'IPython.kernel.enginefc.IFCControllerBase' + + + + + +CLIENT_INTERFACES = dict( + TASK = dict(FURL_FILE = 'ipcontroller-tc.furl'), + MULTIENGINE = dict(FURLFILE='ipcontroller-mec.furl') +) + diff --git a/IPython/config/default/ipengine_config.py b/IPython/config/default/ipengine_config.py new file mode 100644 index 0000000..f86a407 --- /dev/null +++ b/IPython/config/default/ipengine_config.py @@ -0,0 +1,19 @@ +c = get_config() + +c.MPI.default = 'mpi4py' + +c.MPI.mpi4py = """from mpi4py import MPI as mpi +mpi.size = mpi.COMM_WORLD.Get_size() +mpi.rank = mpi.COMM_WORLD.Get_rank() +""" + +c.MPI.pytrilinos = """from PyTrilinos import Epetra +class SimpleStruct: +pass +mpi = SimpleStruct() +mpi.rank = 0 +mpi.size = 0 +""" + +c.Global.logfile = '' +c.Global.furl_file = 'ipcontroller-engine.furl' diff --git a/IPython/kernel/configobjfactory.py b/IPython/kernel/configobjfactory.py new file mode 100644 index 0000000..3d8cd24 --- /dev/null +++ b/IPython/kernel/configobjfactory.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# encoding: utf-8 +""" +A class for creating a Twisted service that is configured using IPython's +configuration system. +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2008-2009 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +import zope.interface as zi + +from IPython.core.component import Component + +#----------------------------------------------------------------------------- +# Code +#----------------------------------------------------------------------------- + + +class IConfiguredObjectFactory(zi.Interface): + """I am a component that creates a configured object. + + This class is useful if you want to configure a class that is not a + subclass of :class:`IPython.core.component.Component`. + """ + + def __init__(config): + """Get ready to configure the object using config.""" + + def create(): + """Return an instance of the configured object.""" + + +class ConfiguredObjectFactory(Component): + + zi.implements(IConfiguredObjectFactory) + + def __init__(self, config): + super(ConfiguredObjectFactory, self).__init__(None, config=config) + + def create(self): + raise NotImplementedError('create must be implemented in a subclass') + + +class IAdaptedConfiguredObjectFactory(zi.Interface): + """I am a component that adapts and configures an object. + + This class is useful if you have the adapt a instance and configure it. + """ + + def __init__(config, adaptee=None): + """Get ready to adapt adaptee and then configure it using config.""" + + def create(): + """Return an instance of the adapted and configured object.""" + + +class AdaptedConfiguredObjectFactory(Component): + + # zi.implements(IAdaptedConfiguredObjectFactory) + + def __init__(self, config, adaptee): + super(AdaptedConfiguredObjectFactory, self).__init__(None, config=config) + self.adaptee = adaptee + + def create(self): + raise NotImplementedError('create must be implemented in a subclass') \ No newline at end of file diff --git a/IPython/kernel/scripts/ipcontrollerapp.py b/IPython/kernel/scripts/ipcontrollerapp.py new file mode 100644 index 0000000..3d85dd2 --- /dev/null +++ b/IPython/kernel/scripts/ipcontrollerapp.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python +# encoding: utf-8 +""" +The IPython controller application +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2008-2009 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +import copy +import os +import sys + +from twisted.application import service +from twisted.internet import reactor, defer +from twisted.python import log + +from IPython.config.loader import Config, NoConfigDefault + +from IPython.core.application import Application, IPythonArgParseConfigLoader +from IPython.core import release + +from IPython.utils.traitlets import Int, Str, Bool, Instance +from IPython.utils.importstring import import_item + +from IPython.kernel import controllerservice +from IPython.kernel.configobjfactory import ( + ConfiguredObjectFactory, + AdaptedConfiguredObjectFactory +) + +from IPython.kernel.fcutil import FCServiceFactory + +#----------------------------------------------------------------------------- +# Components for creating services +#----------------------------------------------------------------------------- + + +# The default client interfaces for FCClientServiceFactory.Interfaces +default_client_interfaces = Config() +default_client_interfaces.Task.interface_chain = [ + 'IPython.kernel.task.ITaskController', + 'IPython.kernel.taskfc.IFCTaskController' +] +default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl' +default_client_interfaces.MultiEngine.interface_chain = [ + 'IPython.kernel.multiengine.IMultiEngine', + 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine' +] +default_client_interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl' + +# Make this a dict we can pass to Config.__init__ for the default +default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items())) + + + +# The default engine interfaces for FCEngineServiceFactory.Interfaces +default_engine_interfaces = Config() +default_engine_interfaces.Default.interface_chain = [ + 'IPython.kernel.enginefc.IFCControllerBase' +] +default_engine_interfaces.Default.furl_file = 'ipcontroller-engine.furl' + +# Make this a dict we can pass to Config.__init__ for the default +default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items())) + + + +class FCClientServiceFactory(FCServiceFactory): + """A Foolscap implementation of the client services.""" + + cert_file = Str('ipcontroller-client.pem', config=True) + Interfaces = Instance(klass=Config, kw=default_client_interfaces, + allow_none=False, config=True) + + +class FCEngineServiceFactory(FCServiceFactory): + """A Foolscap implementation of the engine services.""" + + cert_file = Str('ipcontroller-engine.pem', config=True) + interfaces = Instance(klass=dict, kw=default_engine_interfaces, + allow_none=False, config=True) + + +#----------------------------------------------------------------------------- +# The main application +#----------------------------------------------------------------------------- + + +cl_args = ( + # Client config + (('--client-ip',), dict( + type=str, dest='FCClientServiceFactory.ip', default=NoConfigDefault, + help='The IP address or hostname the controller will listen on for client connections.', + metavar='FCClientServiceFactory.ip') + ), + (('--client-port',), dict( + type=int, dest='FCClientServiceFactory.port', default=NoConfigDefault, + help='The port the controller will listen on for client connections.', + metavar='FCClientServiceFactory.port') + ), + (('--client-location',), dict( + type=str, dest='FCClientServiceFactory.location', default=NoConfigDefault, + help='The hostname or ip that clients should connect to.', + metavar='FCClientServiceFactory.location') + ), + (('-x',), dict( + action='store_false', dest='FCClientServiceFactory.secure', default=NoConfigDefault, + help='Turn off all client security.') + ), + (('--client-cert-file',), dict( + type=str, dest='FCClientServiceFactory.cert_file', default=NoConfigDefault, + help='File to store the client SSL certificate in.', + metavar='FCClientServiceFactory.cert_file') + ), + (('--task-furl-file',), dict( + type=str, dest='FCClientServiceFactory.Interfaces.Task.furl_file', default=NoConfigDefault, + help='File to store the FURL in for task clients to connect with.', + metavar='FCClientServiceFactory.Interfaces.Task.furl_file') + ), + (('--multiengine-furl-file',), dict( + type=str, dest='FCClientServiceFactory.Interfaces.MultiEngine.furl_file', default=NoConfigDefault, + help='File to store the FURL in for multiengine clients to connect with.', + metavar='FCClientServiceFactory.Interfaces.MultiEngine.furl_file') + ), + # Engine config + (('--engine-ip',), dict( + type=str, dest='FCEngineServiceFactory.ip', default=NoConfigDefault, + help='The IP address or hostname the controller will listen on for engine connections.', + metavar='FCEngineServiceFactory.ip') + ), + (('--engine-port',), dict( + type=int, dest='FCEngineServiceFactory.port', default=NoConfigDefault, + help='The port the controller will listen on for engine connections.', + metavar='FCEngineServiceFactory.port') + ), + (('--engine-location',), dict( + type=str, dest='FCEngineServiceFactory.location', default=NoConfigDefault, + help='The hostname or ip that engines should connect to.', + metavar='FCEngineServiceFactory.location') + ), + (('-y',), dict( + action='store_false', dest='FCEngineServiceFactory.secure', default=NoConfigDefault, + help='Turn off all engine security.') + ), + (('--engine-cert-file',), dict( + type=str, dest='FCEngineServiceFactory.cert_file', default=NoConfigDefault, + help='File to store the client SSL certificate in.', + metavar='FCEngineServiceFactory.cert_file') + ), + (('--engine-furl-file',), dict( + type=str, dest='FCEngineServiceFactory.Interfaces.Default.furl_file', default=NoConfigDefault, + help='File to store the FURL in for engines to connect with.', + metavar='FCEngineServiceFactory.Interfaces.Default.furl_file') + ), + # Global config + (('-l','--logfile'), dict( + type=str, dest='Global.logfile', default=NoConfigDefault, + help='Log file name (default is stdout)', + metavar='Global.logfile') + ), + (('-r',), dict( + action='store_true', dest='Global.reuse_furls', default=NoConfigDefault, + help='Try to reuse all FURL files.') + ) +) + + +class IPControllerAppCLConfigLoader(IPythonArgParseConfigLoader): + + arguments = cl_args + + +_default_config_file_name = 'ipcontroller_config.py' + +class IPControllerApp(Application): + + name = 'ipcontroller' + config_file_name = _default_config_file_name + + def create_default_config(self): + super(IPControllerApp, self).create_default_config() + self.default_config.Global.logfile = '' + self.default_config.Global.reuse_furls = False + self.default_config.Global.import_statements = [] + + def create_command_line_config(self): + """Create and return a command line config loader.""" + + return IPControllerAppCLConfigLoader( + description="Start an IPython controller", + version=release.version) + + def construct(self): + # I am a little hesitant to put these into InteractiveShell itself. + # But that might be the place for them + sys.path.insert(0, '') + + self.start_logging() + self.import_statements() + self.reuse_furls() + + # Create the service hierarchy + self.main_service = service.MultiService() + # The controller service + controller_service = controllerservice.ControllerService() + controller_service.setServiceParent(self.main_service) + # The client tub and all its refereceables + csfactory = FCClientServiceFactory(self.master_config, controller_service) + client_service = csfactory.create() + client_service.setServiceParent(self.main_service) + # The engine tub + esfactory = FCEngineServiceFactory(self.master_config, controller_service) + engine_service = esfactory.create() + engine_service.setServiceParent(self.main_service) + + def start_logging(self): + logfile = self.master_config.Global.logfile + if logfile: + logfile = logfile + str(os.getpid()) + '.log' + try: + openLogFile = open(logfile, 'w') + except: + openLogFile = sys.stdout + else: + openLogFile = sys.stdout + log.startLogging(openLogFile) + + def import_statements(self): + statements = self.master_config.Global.import_statements + for s in statements: + try: + exec s in globals(), locals() + except: + log.msg("Error running import statement: %s" % s) + + def reuse_furls(self): + # This logic might need to be moved into the components + # Delete old furl files unless the reuse_furls is set + reuse = self.master_config.Global.reuse_furls + # if not reuse: + # paths = ( + # self.master_config.FCEngineServiceFactory.Interfaces.Default.furl_file, + # self.master_config.FCClientServiceFactory.Interfaces.Task.furl_file, + # self.master_config.FCClientServiceFactory.Interfaces.MultiEngine.furl_file + # ) + # for p in paths: + # if os.path.isfile(p): + # os.remove(p) + + def start_app(self): + # Start the controller service and set things running + self.main_service.startService() + reactor.run() + +if __name__ == '__main__': + app = IPControllerApp() + app.start() diff --git a/docs/examples/core/new-embed.py b/docs/examples/core/new-embed.py new file mode 100644 index 0000000..f0eddf5 --- /dev/null +++ b/docs/examples/core/new-embed.py @@ -0,0 +1,17 @@ +# This shows how to use the new top-level embed function. It is a simpler +# API that manages the creation of the embedded shell. + +from IPython import embed + +a = 10 +b = 20 + +embed('First time') + +c = 30 +d = 40 + +try: + raise Exception('adsfasdf') +except: + embed('The second time') diff --git a/docs/source/development/index.txt b/docs/source/development/index.txt index 5b41f8d..4671263 100644 --- a/docs/source/development/index.txt +++ b/docs/source/development/index.txt @@ -15,4 +15,5 @@ IPython developer's guide roadmap.txt reorg.txt notification_blueprint.txt + ipgraph.txt diff --git a/docs/source/development/ipgraph.txt b/docs/source/development/ipgraph.txt new file mode 100644 index 0000000..f0b10fe --- /dev/null +++ b/docs/source/development/ipgraph.txt @@ -0,0 +1,59 @@ +==================================================== +Notes on code execution in :class:`InteractiveShell` +==================================================== + +Overview +======== + +This section contains information and notes about the code execution +system in :class:`InteractiveShell`. This system needs to be refactored +and we are keeping notes about this process here. + +Current design +============== + +Here is a script that shows the relationships between the various +methods in :class:`InteractiveShell` that manage code execution:: + + import networkx as nx + import matplotlib.pyplot as plt + + exec_init_cmd = 'exec_init_cmd' + interact = 'interact' + runlines = 'runlines' + runsource = 'runsource' + runcode = 'runcode' + push_line = 'push_line' + mainloop = 'mainloop' + embed_mainloop = 'embed_mainloop' + ri = 'raw_input' + prefilter = 'prefilter' + + g = nx.DiGraph() + + g.add_node(exec_init_cmd) + g.add_node(interact) + g.add_node(runlines) + g.add_node(runsource) + g.add_node(push_line) + g.add_node(mainloop) + g.add_node(embed_mainloop) + g.add_node(ri) + g.add_node(prefilter) + + g.add_edge(exec_init_cmd, push_line) + g.add_edge(exec_init_cmd, prefilter) + g.add_edge(mainloop, exec_init_cmd) + g.add_edge(mainloop, interact) + g.add_edge(embed_mainloop, interact) + g.add_edge(interact, ri) + g.add_edge(interact, push_line) + g.add_edge(push_line, runsource) + g.add_edge(runlines, push_line) + g.add_edge(runlines, prefilter) + g.add_edge(runsource, runcode) + g.add_edge(ri, prefilter) + + nx.draw_spectral(g, node_size=100, alpha=0.6, node_color='r', + font_size=10, node_shape='o') + plt.show()