IPython/kernel/ipcontrollerapp.py

#!/usr/bin/env python
# encoding: utf-8
"""
The IPython controller application.
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2009  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from __future__ import with_statement

import copy
import os
import sys

from twisted.application import service
from twisted.internet import reactor
from twisted.python import log

from IPython.config.loader import Config, NoConfigDefault
from IPython.core import release
from IPython.core.application import Application
from IPython.kernel import controllerservice
from IPython.kernel.clusterdir import ApplicationWithClusterDir
from IPython.kernel.fcutil import FCServiceFactory
from IPython.utils.traitlets import Str, Instance, Unicode

#-----------------------------------------------------------------------------
# Default interfaces
#-----------------------------------------------------------------------------

# The default client interfaces for FCClientServiceFactory.interfaces
default_client_interfaces = Config()
default_client_interfaces.Task.interface_chain = [
    'IPython.kernel.task.ITaskController',
    'IPython.kernel.taskfc.IFCTaskController'
]

default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'

default_client_interfaces.MultiEngine.interface_chain = [
    'IPython.kernel.multiengine.IMultiEngine',
    'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
]

default_client_interfaces.MultiEngine.furl_file = u'ipcontroller-mec.furl'

# Make this a dict we can pass to Config.__init__ for the default
default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))



# The default engine interfaces for FCEngineServiceFactory.interfaces
default_engine_interfaces = Config()
default_engine_interfaces.Default.interface_chain = [
    'IPython.kernel.enginefc.IFCControllerBase'
]

default_engine_interfaces.Default.furl_file = u'ipcontroller-engine.furl'

# Make this a dict we can pass to Config.__init__ for the default
default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))


#-----------------------------------------------------------------------------
# Service factories
#-----------------------------------------------------------------------------


class FCClientServiceFactory(FCServiceFactory):
    """A Foolscap implementation of the client services."""

    cert_file = Unicode(u'ipcontroller-client.pem', config=True)
    interfaces = Instance(klass=Config, kw=default_client_interfaces,
                          allow_none=False, config=True)


class FCEngineServiceFactory(FCServiceFactory):
    """A Foolscap implementation of the engine services."""

    cert_file = Unicode(u'ipcontroller-engine.pem', config=True)
    interfaces = Instance(klass=dict, kw=default_engine_interfaces,
                          allow_none=False, config=True)


#-----------------------------------------------------------------------------
# The main application
#-----------------------------------------------------------------------------


cl_args = (
    # Client config
    (('--client-ip',), dict(
        type=str, dest='FCClientServiceFactory.ip', 
        help='The IP address or hostname the controller will listen on for '
        'client connections.',
        metavar='FCClientServiceFactory.ip')
    ),
    (('--client-port',), dict(
        type=int, dest='FCClientServiceFactory.port', 
        help='The port the controller will listen on for client connections. '
        'The default is to use 0, which will autoselect an open port.',
        metavar='FCClientServiceFactory.port')
    ),
    (('--client-location',), dict(
        type=str, dest='FCClientServiceFactory.location',
        help='The hostname or IP that clients should connect to. This does '
        'not control which interface the controller listens on. Instead, this '
        'determines the hostname/IP that is listed in the FURL, which is how '
        'clients know where to connect. Useful if the controller is listening '
        'on multiple interfaces.',
        metavar='FCClientServiceFactory.location')
    ),
    # Engine config
    (('--engine-ip',), dict(
        type=str, dest='FCEngineServiceFactory.ip', 
        help='The IP address or hostname the controller will listen on for '
        'engine connections.',
        metavar='FCEngineServiceFactory.ip')
    ),
    (('--engine-port',), dict(
        type=int, dest='FCEngineServiceFactory.port',
        help='The port the controller will listen on for engine connections. '
        'The default is to use 0, which will autoselect an open port.',
        metavar='FCEngineServiceFactory.port')
    ),
    (('--engine-location',), dict(
        type=str, dest='FCEngineServiceFactory.location',
        help='The hostname or IP that engines should connect to. This does '
        'not control which interface the controller listens on. Instead, this '
        'determines the hostname/IP that is listed in the FURL, which is how '
        'engines know where to connect. Useful if the controller is listening '
        'on multiple interfaces.',
        metavar='FCEngineServiceFactory.location')
    ),
    # Global config
    (('--log-to-file',), dict(
        action='store_true', dest='Global.log_to_file',
        help='Log to a file in the log directory (default is stdout)')
    ),
    (('-r','--reuse-furls'), dict(
        action='store_true', dest='Global.reuse_furls',
        help='Try to reuse all FURL files. If this is not set all FURL files '
        'are deleted before the controller starts. This must be set if '
        'specific ports are specified by --engine-port or --client-port.')
    ),
    (('--no-secure',), dict(
        action='store_false', dest='Global.secure',
        help='Turn off SSL encryption for all connections.')
    ),
    (('--secure',), dict(
        action='store_true', dest='Global.secure',
        help='Turn off SSL encryption for all connections.')
    )
)


_description = """Start the IPython controller for parallel computing.

The IPython controller provides a gateway between the IPython engines and
clients. The controller needs to be started before the engines and can be
configured using command line options or using a cluster directory. Cluster
directories contain config, log and security files and are usually located in
your .ipython directory and named as "cluster_<profile>". See the --profile
and --cluster-dir options for details.
"""

default_config_file_name = u'ipcontroller_config.py'


class IPControllerApp(ApplicationWithClusterDir):

    name = u'ipcontroller'
    description = _description
    config_file_name = default_config_file_name
    auto_create_cluster_dir = True
    cl_arguments = Application.cl_arguments + cl_args

    def create_default_config(self):
        super(IPControllerApp, self).create_default_config()
        self.default_config.Global.reuse_furls = False
        self.default_config.Global.secure = True
        self.default_config.Global.import_statements = []
        self.default_config.Global.clean_logs = True

    def post_load_command_line_config(self):
        # Now setup reuse_furls
        c = self.command_line_config
        if hasattr(c.Global, 'reuse_furls'):
            c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls
            c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls
            del c.Global.reuse_furls
        if hasattr(c.Global, 'secure'):
            c.FCClientServiceFactory.secure = c.Global.secure
            c.FCEngineServiceFactory.secure = c.Global.secure
            del c.Global.secure

    def construct(self):
        # This is the working dir by now.
        sys.path.insert(0, '')

        self.start_logging()
        self.import_statements()

        # Create the service hierarchy
        self.main_service = service.MultiService()
        # The controller service
        controller_service = controllerservice.ControllerService()
        controller_service.setServiceParent(self.main_service)
        # The client tub and all its refereceables
        csfactory = FCClientServiceFactory(self.master_config, controller_service)
        client_service = csfactory.create()
        client_service.setServiceParent(self.main_service)
        # The engine tub
        esfactory = FCEngineServiceFactory(self.master_config, controller_service)
        engine_service = esfactory.create()
        engine_service.setServiceParent(self.main_service)

    def import_statements(self):
        statements = self.master_config.Global.import_statements
        for s in statements:
            try:
                log.msg("Executing statement: '%s'" % s)
                exec s in globals(), locals()
            except:
                log.msg("Error running statement: %s" % s)

    def start_app(self):
        # Start the controller service.
        self.main_service.startService()
        # Write the .pid file overwriting old ones. This allow multiple
        # controllers to clober each other. But Windows is not cleaning
        # these up properly.
        self.write_pid_file(overwrite=True)
        # Add a trigger to delete the .pid file upon shutting down.
        reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
        reactor.run()


def launch_new_instance():
    """Create and run the IPython controller"""
    app = IPControllerApp()
    app.start()


if __name__ == '__main__':
    launch_new_instance()