|
|
#!/usr/bin/env python
|
|
|
# encoding: utf-8
|
|
|
"""
|
|
|
The IPython controller application.
|
|
|
"""
|
|
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
# Copyright (C) 2008-2009 The IPython Development Team
|
|
|
#
|
|
|
# Distributed under the terms of the BSD License. The full license is in
|
|
|
# the file COPYING, distributed as part of this software.
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
# Imports
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
|
|
from __future__ import with_statement
|
|
|
|
|
|
import copy
|
|
|
import sys
|
|
|
|
|
|
from twisted.application import service
|
|
|
from twisted.internet import reactor
|
|
|
from twisted.python import log
|
|
|
|
|
|
from IPython.config.loader import Config
|
|
|
from IPython.kernel import controllerservice
|
|
|
from IPython.kernel.clusterdir import (
|
|
|
ApplicationWithClusterDir,
|
|
|
ClusterDirConfigLoader
|
|
|
)
|
|
|
from IPython.kernel.fcutil import FCServiceFactory, FURLError
|
|
|
from IPython.utils.traitlets import Instance, Unicode
|
|
|
|
|
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
# Module level variables
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
#: The default config file name for this application
|
|
|
default_config_file_name = u'ipcontroller_config.py'
|
|
|
|
|
|
|
|
|
_description = """Start the IPython controller for parallel computing.
|
|
|
|
|
|
The IPython controller provides a gateway between the IPython engines and
|
|
|
clients. The controller needs to be started before the engines and can be
|
|
|
configured using command line options or using a cluster directory. Cluster
|
|
|
directories contain config, log and security files and are usually located in
|
|
|
your .ipython directory and named as "cluster_<profile>". See the --profile
|
|
|
and --cluster-dir options for details.
|
|
|
"""
|
|
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
# Default interfaces
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
|
|
# The default client interfaces for FCClientServiceFactory.interfaces
|
|
|
default_client_interfaces = Config()
|
|
|
default_client_interfaces.Task.interface_chain = [
|
|
|
'IPython.kernel.task.ITaskController',
|
|
|
'IPython.kernel.taskfc.IFCTaskController'
|
|
|
]
|
|
|
|
|
|
default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
|
|
|
|
|
|
default_client_interfaces.MultiEngine.interface_chain = [
|
|
|
'IPython.kernel.multiengine.IMultiEngine',
|
|
|
'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
|
|
|
]
|
|
|
|
|
|
default_client_interfaces.MultiEngine.furl_file = u'ipcontroller-mec.furl'
|
|
|
|
|
|
# Make this a dict we can pass to Config.__init__ for the default
|
|
|
default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))
|
|
|
|
|
|
|
|
|
|
|
|
# The default engine interfaces for FCEngineServiceFactory.interfaces
|
|
|
default_engine_interfaces = Config()
|
|
|
default_engine_interfaces.Default.interface_chain = [
|
|
|
'IPython.kernel.enginefc.IFCControllerBase'
|
|
|
]
|
|
|
|
|
|
default_engine_interfaces.Default.furl_file = u'ipcontroller-engine.furl'
|
|
|
|
|
|
# Make this a dict we can pass to Config.__init__ for the default
|
|
|
default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
|
|
|
|
|
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
# Service factories
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
class FCClientServiceFactory(FCServiceFactory):
|
|
|
"""A Foolscap implementation of the client services."""
|
|
|
|
|
|
cert_file = Unicode(u'ipcontroller-client.pem', config=True)
|
|
|
interfaces = Instance(klass=Config, kw=default_client_interfaces,
|
|
|
allow_none=False, config=True)
|
|
|
|
|
|
|
|
|
class FCEngineServiceFactory(FCServiceFactory):
|
|
|
"""A Foolscap implementation of the engine services."""
|
|
|
|
|
|
cert_file = Unicode(u'ipcontroller-engine.pem', config=True)
|
|
|
interfaces = Instance(klass=dict, kw=default_engine_interfaces,
|
|
|
allow_none=False, config=True)
|
|
|
|
|
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
# Command line options
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
class IPControllerAppConfigLoader(ClusterDirConfigLoader):
|
|
|
|
|
|
def _add_arguments(self):
|
|
|
super(IPControllerAppConfigLoader, self)._add_arguments()
|
|
|
paa = self.parser.add_argument
|
|
|
# Client config
|
|
|
paa('--client-ip',
|
|
|
type=str, dest='FCClientServiceFactory.ip',
|
|
|
help='The IP address or hostname the controller will listen on for '
|
|
|
'client connections.',
|
|
|
metavar='FCClientServiceFactory.ip')
|
|
|
paa('--client-port',
|
|
|
type=int, dest='FCClientServiceFactory.port',
|
|
|
help='The port the controller will listen on for client connections. '
|
|
|
'The default is to use 0, which will autoselect an open port.',
|
|
|
metavar='FCClientServiceFactory.port')
|
|
|
paa('--client-location',), dict(
|
|
|
type=str, dest='FCClientServiceFactory.location',
|
|
|
help='The hostname or IP that clients should connect to. This does '
|
|
|
'not control which interface the controller listens on. Instead, this '
|
|
|
'determines the hostname/IP that is listed in the FURL, which is how '
|
|
|
'clients know where to connect. Useful if the controller is listening '
|
|
|
'on multiple interfaces.',
|
|
|
metavar='FCClientServiceFactory.location')
|
|
|
# Engine config
|
|
|
paa('--engine-ip',
|
|
|
type=str, dest='FCEngineServiceFactory.ip',
|
|
|
help='The IP address or hostname the controller will listen on for '
|
|
|
'engine connections.',
|
|
|
metavar='FCEngineServiceFactory.ip')
|
|
|
paa('--engine-port',
|
|
|
type=int, dest='FCEngineServiceFactory.port',
|
|
|
help='The port the controller will listen on for engine connections. '
|
|
|
'The default is to use 0, which will autoselect an open port.',
|
|
|
metavar='FCEngineServiceFactory.port')
|
|
|
paa('--engine-location',
|
|
|
type=str, dest='FCEngineServiceFactory.location',
|
|
|
help='The hostname or IP that engines should connect to. This does '
|
|
|
'not control which interface the controller listens on. Instead, this '
|
|
|
'determines the hostname/IP that is listed in the FURL, which is how '
|
|
|
'engines know where to connect. Useful if the controller is listening '
|
|
|
'on multiple interfaces.',
|
|
|
metavar='FCEngineServiceFactory.location')
|
|
|
# Global config
|
|
|
paa('--log-to-file',
|
|
|
action='store_true', dest='Global.log_to_file',
|
|
|
help='Log to a file in the log directory (default is stdout)')
|
|
|
paa('-r','--reuse-furls',
|
|
|
action='store_true', dest='Global.reuse_furls',
|
|
|
help='Try to reuse all FURL files. If this is not set all FURL files '
|
|
|
'are deleted before the controller starts. This must be set if '
|
|
|
'specific ports are specified by --engine-port or --client-port.')
|
|
|
paa('--no-secure',
|
|
|
action='store_false', dest='Global.secure',
|
|
|
help='Turn off SSL encryption for all connections.')
|
|
|
paa('--secure',
|
|
|
action='store_true', dest='Global.secure',
|
|
|
help='Turn off SSL encryption for all connections.')
|
|
|
|
|
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
# The main application
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
class IPControllerApp(ApplicationWithClusterDir):
|
|
|
|
|
|
name = u'ipcontroller'
|
|
|
description = _description
|
|
|
command_line_loader = IPControllerAppConfigLoader
|
|
|
default_config_file_name = default_config_file_name
|
|
|
auto_create_cluster_dir = True
|
|
|
|
|
|
def create_default_config(self):
|
|
|
super(IPControllerApp, self).create_default_config()
|
|
|
# Don't set defaults for Global.secure or Global.reuse_furls
|
|
|
# as those are set in a component.
|
|
|
self.default_config.Global.import_statements = []
|
|
|
self.default_config.Global.clean_logs = True
|
|
|
|
|
|
def pre_construct(self):
|
|
|
super(IPControllerApp, self).pre_construct()
|
|
|
c = self.master_config
|
|
|
# The defaults for these are set in FCClientServiceFactory and
|
|
|
# FCEngineServiceFactory, so we only set them here if the global
|
|
|
# options have be set to override the class level defaults.
|
|
|
if hasattr(c.Global, 'reuse_furls'):
|
|
|
c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls
|
|
|
c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls
|
|
|
del c.Global.reuse_furls
|
|
|
if hasattr(c.Global, 'secure'):
|
|
|
c.FCClientServiceFactory.secure = c.Global.secure
|
|
|
c.FCEngineServiceFactory.secure = c.Global.secure
|
|
|
del c.Global.secure
|
|
|
|
|
|
def construct(self):
|
|
|
# This is the working dir by now.
|
|
|
sys.path.insert(0, '')
|
|
|
|
|
|
self.start_logging()
|
|
|
self.import_statements()
|
|
|
|
|
|
# Create the service hierarchy
|
|
|
self.main_service = service.MultiService()
|
|
|
# The controller service
|
|
|
controller_service = controllerservice.ControllerService()
|
|
|
controller_service.setServiceParent(self.main_service)
|
|
|
# The client tub and all its refereceables
|
|
|
try:
|
|
|
csfactory = FCClientServiceFactory(config=self.master_config, adaptee=controller_service)
|
|
|
except FURLError, e:
|
|
|
log.err(e)
|
|
|
self.exit(0)
|
|
|
client_service = csfactory.create()
|
|
|
client_service.setServiceParent(self.main_service)
|
|
|
# The engine tub
|
|
|
try:
|
|
|
esfactory = FCEngineServiceFactory(config=self.master_config, adaptee=controller_service)
|
|
|
except FURLError, e:
|
|
|
log.err(e)
|
|
|
self.exit(0)
|
|
|
engine_service = esfactory.create()
|
|
|
engine_service.setServiceParent(self.main_service)
|
|
|
|
|
|
def import_statements(self):
|
|
|
statements = self.master_config.Global.import_statements
|
|
|
for s in statements:
|
|
|
try:
|
|
|
log.msg("Executing statement: '%s'" % s)
|
|
|
exec s in globals(), locals()
|
|
|
except:
|
|
|
log.msg("Error running statement: %s" % s)
|
|
|
|
|
|
def start_app(self):
|
|
|
# Start the controller service.
|
|
|
self.main_service.startService()
|
|
|
# Write the .pid file overwriting old ones. This allow multiple
|
|
|
# controllers to clober each other. But Windows is not cleaning
|
|
|
# these up properly.
|
|
|
self.write_pid_file(overwrite=True)
|
|
|
# Add a trigger to delete the .pid file upon shutting down.
|
|
|
reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
|
|
|
reactor.run()
|
|
|
|
|
|
|
|
|
def launch_new_instance():
|
|
|
"""Create and run the IPython controller"""
|
|
|
app = IPControllerApp()
|
|
|
app.start()
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
launch_new_instance()
|
|
|
|
