|
|
#!/usr/bin/env python
|
|
|
# encoding: utf-8
|
|
|
"""
|
|
|
The IPython controller application
|
|
|
"""
|
|
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
# Copyright (C) 2008-2009 The IPython Development Team
|
|
|
#
|
|
|
# Distributed under the terms of the BSD License. The full license is in
|
|
|
# the file COPYING, distributed as part of this software.
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
# Imports
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
|
|
import copy
|
|
|
import logging
|
|
|
import os
|
|
|
import sys
|
|
|
|
|
|
from twisted.application import service
|
|
|
from twisted.internet import reactor, defer
|
|
|
from twisted.python import log
|
|
|
|
|
|
from IPython.config.loader import Config, NoConfigDefault
|
|
|
|
|
|
from IPython.core.application import (
|
|
|
ApplicationWithDir,
|
|
|
AppWithDirArgParseConfigLoader
|
|
|
)
|
|
|
|
|
|
from IPython.core import release
|
|
|
|
|
|
from IPython.utils.traitlets import Int, Str, Bool, Instance
|
|
|
from IPython.utils.importstring import import_item
|
|
|
|
|
|
from IPython.kernel import controllerservice
|
|
|
from IPython.kernel.configobjfactory import (
|
|
|
ConfiguredObjectFactory,
|
|
|
AdaptedConfiguredObjectFactory
|
|
|
)
|
|
|
|
|
|
from IPython.kernel.fcutil import FCServiceFactory
|
|
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
# Default interfaces
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
# The default client interfaces for FCClientServiceFactory.interfaces
|
|
|
default_client_interfaces = Config()
|
|
|
default_client_interfaces.Task.interface_chain = [
|
|
|
'IPython.kernel.task.ITaskController',
|
|
|
'IPython.kernel.taskfc.IFCTaskController'
|
|
|
]
|
|
|
|
|
|
default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
|
|
|
|
|
|
default_client_interfaces.MultiEngine.interface_chain = [
|
|
|
'IPython.kernel.multiengine.IMultiEngine',
|
|
|
'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
|
|
|
]
|
|
|
|
|
|
default_client_interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl'
|
|
|
|
|
|
# Make this a dict we can pass to Config.__init__ for the default
|
|
|
default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))
|
|
|
|
|
|
|
|
|
|
|
|
# The default engine interfaces for FCEngineServiceFactory.interfaces
|
|
|
default_engine_interfaces = Config()
|
|
|
default_engine_interfaces.Default.interface_chain = [
|
|
|
'IPython.kernel.enginefc.IFCControllerBase'
|
|
|
]
|
|
|
|
|
|
default_engine_interfaces.Default.furl_file = 'ipcontroller-engine.furl'
|
|
|
|
|
|
# Make this a dict we can pass to Config.__init__ for the default
|
|
|
default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
|
|
|
|
|
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
# Service factories
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
class FCClientServiceFactory(FCServiceFactory):
|
|
|
"""A Foolscap implementation of the client services."""
|
|
|
|
|
|
cert_file = Str('ipcontroller-client.pem', config=True)
|
|
|
interfaces = Instance(klass=Config, kw=default_client_interfaces,
|
|
|
allow_none=False, config=True)
|
|
|
|
|
|
|
|
|
class FCEngineServiceFactory(FCServiceFactory):
|
|
|
"""A Foolscap implementation of the engine services."""
|
|
|
|
|
|
cert_file = Str('ipcontroller-engine.pem', config=True)
|
|
|
interfaces = Instance(klass=dict, kw=default_engine_interfaces,
|
|
|
allow_none=False, config=True)
|
|
|
|
|
|
|
|
|
#-----------------------------------------------------------------------------
|
|
|
# The main application
|
|
|
#-----------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
cl_args = (
|
|
|
# Client config
|
|
|
(('--client-ip',), dict(
|
|
|
type=str, dest='FCClientServiceFactory.ip', default=NoConfigDefault,
|
|
|
help='The IP address or hostname the controller will listen on for client connections.',
|
|
|
metavar='FCClientServiceFactory.ip')
|
|
|
),
|
|
|
(('--client-port',), dict(
|
|
|
type=int, dest='FCClientServiceFactory.port', default=NoConfigDefault,
|
|
|
help='The port the controller will listen on for client connections.',
|
|
|
metavar='FCClientServiceFactory.port')
|
|
|
),
|
|
|
(('--client-location',), dict(
|
|
|
type=str, dest='FCClientServiceFactory.location', default=NoConfigDefault,
|
|
|
help='The hostname or ip that clients should connect to.',
|
|
|
metavar='FCClientServiceFactory.location')
|
|
|
),
|
|
|
# Engine config
|
|
|
(('--engine-ip',), dict(
|
|
|
type=str, dest='FCEngineServiceFactory.ip', default=NoConfigDefault,
|
|
|
help='The IP address or hostname the controller will listen on for engine connections.',
|
|
|
metavar='FCEngineServiceFactory.ip')
|
|
|
),
|
|
|
(('--engine-port',), dict(
|
|
|
type=int, dest='FCEngineServiceFactory.port', default=NoConfigDefault,
|
|
|
help='The port the controller will listen on for engine connections.',
|
|
|
metavar='FCEngineServiceFactory.port')
|
|
|
),
|
|
|
(('--engine-location',), dict(
|
|
|
type=str, dest='FCEngineServiceFactory.location', default=NoConfigDefault,
|
|
|
help='The hostname or ip that engines should connect to.',
|
|
|
metavar='FCEngineServiceFactory.location')
|
|
|
),
|
|
|
# Global config
|
|
|
(('--log-to-file',), dict(
|
|
|
action='store_true', dest='Global.log_to_file', default=NoConfigDefault,
|
|
|
help='Log to a file in the log directory (default is stdout)')
|
|
|
),
|
|
|
(('-r','--reuse-furls'), dict(
|
|
|
action='store_true', dest='Global.reuse_furls', default=NoConfigDefault,
|
|
|
help='Try to reuse all FURL files.')
|
|
|
),
|
|
|
(('-ns','--no-security'), dict(
|
|
|
action='store_false', dest='Global.secure', default=NoConfigDefault,
|
|
|
help='Turn off SSL encryption for all connections.')
|
|
|
)
|
|
|
)
|
|
|
|
|
|
|
|
|
class IPControllerAppCLConfigLoader(AppWithDirArgParseConfigLoader):
|
|
|
|
|
|
arguments = cl_args
|
|
|
|
|
|
|
|
|
default_config_file_name = 'ipcontroller_config.py'
|
|
|
|
|
|
|
|
|
class IPControllerApp(ApplicationWithDir):
|
|
|
|
|
|
name = 'ipcontroller'
|
|
|
app_dir_basename = 'cluster'
|
|
|
description = 'Start the IPython controller for parallel computing.'
|
|
|
config_file_name = default_config_file_name
|
|
|
default_log_level = logging.WARN
|
|
|
|
|
|
def create_default_config(self):
|
|
|
super(IPControllerApp, self).create_default_config()
|
|
|
self.default_config.Global.reuse_furls = False
|
|
|
self.default_config.Global.secure = True
|
|
|
self.default_config.Global.import_statements = []
|
|
|
self.default_config.Global.log_dir_name = 'log'
|
|
|
self.default_config.Global.security_dir_name = 'security'
|
|
|
self.default_config.Global.log_to_file = False
|
|
|
|
|
|
def create_command_line_config(self):
|
|
|
"""Create and return a command line config loader."""
|
|
|
return IPControllerAppCLConfigLoader(
|
|
|
description=self.description,
|
|
|
version=release.version
|
|
|
)
|
|
|
|
|
|
def post_load_command_line_config(self):
|
|
|
# Now setup reuse_furls
|
|
|
if hasattr(self.command_line_config.Global, 'reuse_furls'):
|
|
|
self.command_line_config.FCClientServiceFactory.reuse_furls = \
|
|
|
self.command_line_config.Global.reuse_furls
|
|
|
self.command_line_config.FCEngineServiceFactory.reuse_furls = \
|
|
|
self.command_line_config.Global.reuse_furls
|
|
|
del self.command_line_config.Global.reuse_furls
|
|
|
if hasattr(self.command_line_config.Global, 'secure'):
|
|
|
self.command_line_config.FCClientServiceFactory.secure = \
|
|
|
self.command_line_config.Global.secure
|
|
|
self.command_line_config.FCEngineServiceFactory.secure = \
|
|
|
self.command_line_config.Global.secure
|
|
|
del self.command_line_config.Global.secure
|
|
|
|
|
|
def pre_construct(self):
|
|
|
# Now set the security_dir and log_dir and create them. We use
|
|
|
# the names an construct the absolute paths.
|
|
|
security_dir = os.path.join(self.master_config.Global.app_dir,
|
|
|
self.master_config.Global.security_dir_name)
|
|
|
log_dir = os.path.join(self.master_config.Global.app_dir,
|
|
|
self.master_config.Global.log_dir_name)
|
|
|
if not os.path.isdir(security_dir):
|
|
|
os.mkdir(security_dir, 0700)
|
|
|
else:
|
|
|
os.chmod(security_dir, 0700)
|
|
|
if not os.path.isdir(log_dir):
|
|
|
os.mkdir(log_dir, 0777)
|
|
|
|
|
|
self.security_dir = self.master_config.Global.security_dir = security_dir
|
|
|
self.log_dir = self.master_config.Global.log_dir = log_dir
|
|
|
|
|
|
def construct(self):
|
|
|
# I am a little hesitant to put these into InteractiveShell itself.
|
|
|
# But that might be the place for them
|
|
|
sys.path.insert(0, '')
|
|
|
|
|
|
self.start_logging()
|
|
|
self.import_statements()
|
|
|
|
|
|
# Create the service hierarchy
|
|
|
self.main_service = service.MultiService()
|
|
|
# The controller service
|
|
|
controller_service = controllerservice.ControllerService()
|
|
|
controller_service.setServiceParent(self.main_service)
|
|
|
# The client tub and all its refereceables
|
|
|
csfactory = FCClientServiceFactory(self.master_config, controller_service)
|
|
|
client_service = csfactory.create()
|
|
|
client_service.setServiceParent(self.main_service)
|
|
|
# The engine tub
|
|
|
esfactory = FCEngineServiceFactory(self.master_config, controller_service)
|
|
|
engine_service = esfactory.create()
|
|
|
engine_service.setServiceParent(self.main_service)
|
|
|
|
|
|
def start_logging(self):
|
|
|
if self.master_config.Global.log_to_file:
|
|
|
log_filename = self.name + '-' + str(os.getpid()) + '.log'
|
|
|
logfile = os.path.join(self.log_dir, log_filename)
|
|
|
open_log_file = open(logfile, 'w')
|
|
|
else:
|
|
|
open_log_file = sys.stdout
|
|
|
log.startLogging(open_log_file)
|
|
|
|
|
|
def import_statements(self):
|
|
|
statements = self.master_config.Global.import_statements
|
|
|
for s in statements:
|
|
|
try:
|
|
|
log.msg("Executing statement: '%s'" % s)
|
|
|
exec s in globals(), locals()
|
|
|
except:
|
|
|
log.msg("Error running statement: %s" % s)
|
|
|
|
|
|
def start_app(self):
|
|
|
# Start the controller service and set things running
|
|
|
self.main_service.startService()
|
|
|
reactor.run()
|
|
|
|
|
|
|
|
|
def launch_new_instance():
|
|
|
"""Create and run the IPython controller"""
|
|
|
app = IPControllerApp()
|
|
|
app.start()
|
|
|
|
