ipcontrollerapp.py
275 lines
| 10.2 KiB
| text/x-python
|
PythonLexer
Brian Granger
|
r2288 | #!/usr/bin/env python | ||
# encoding: utf-8 | ||||
""" | ||||
Brian Granger
|
r2301 | The IPython controller application. | ||
Brian Granger
|
r2288 | """ | ||
#----------------------------------------------------------------------------- | ||||
# Copyright (C) 2008-2009 The IPython Development Team | ||||
# | ||||
# Distributed under the terms of the BSD License. The full license is in | ||||
# the file COPYING, distributed as part of this software. | ||||
#----------------------------------------------------------------------------- | ||||
#----------------------------------------------------------------------------- | ||||
# Imports | ||||
#----------------------------------------------------------------------------- | ||||
import copy | ||||
import os | ||||
import sys | ||||
from twisted.application import service | ||||
Brian Granger
|
r2301 | from twisted.internet import reactor | ||
Brian Granger
|
r2288 | from twisted.python import log | ||
from IPython.config.loader import Config, NoConfigDefault | ||||
Brian Granger
|
r2301 | from IPython.kernel.clusterdir import ( | ||
ApplicationWithClusterDir, | ||||
AppWithClusterDirArgParseConfigLoader | ||||
Brian Granger
|
r2296 | ) | ||
Brian Granger
|
r2288 | from IPython.core import release | ||
Brian Granger
|
r2301 | from IPython.utils.traitlets import Str, Instance | ||
Brian Granger
|
r2288 | |||
from IPython.kernel import controllerservice | ||||
from IPython.kernel.fcutil import FCServiceFactory | ||||
#----------------------------------------------------------------------------- | ||||
Brian Granger
|
r2294 | # Default interfaces | ||
Brian Granger
|
r2288 | #----------------------------------------------------------------------------- | ||
Brian Granger
|
r2297 | # The default client interfaces for FCClientServiceFactory.interfaces | ||
Brian Granger
|
r2288 | default_client_interfaces = Config() | ||
default_client_interfaces.Task.interface_chain = [ | ||||
'IPython.kernel.task.ITaskController', | ||||
'IPython.kernel.taskfc.IFCTaskController' | ||||
] | ||||
Brian Granger
|
r2294 | |||
Brian Granger
|
r2288 | default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl' | ||
Brian Granger
|
r2297 | |||
Brian Granger
|
r2288 | default_client_interfaces.MultiEngine.interface_chain = [ | ||
'IPython.kernel.multiengine.IMultiEngine', | ||||
'IPython.kernel.multienginefc.IFCSynchronousMultiEngine' | ||||
] | ||||
Brian Granger
|
r2294 | |||
Brian Granger
|
r2288 | default_client_interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl' | ||
# Make this a dict we can pass to Config.__init__ for the default | ||||
default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items())) | ||||
Brian Granger
|
r2297 | # The default engine interfaces for FCEngineServiceFactory.interfaces | ||
Brian Granger
|
r2288 | default_engine_interfaces = Config() | ||
default_engine_interfaces.Default.interface_chain = [ | ||||
'IPython.kernel.enginefc.IFCControllerBase' | ||||
] | ||||
Brian Granger
|
r2294 | |||
Brian Granger
|
r2288 | default_engine_interfaces.Default.furl_file = 'ipcontroller-engine.furl' | ||
# Make this a dict we can pass to Config.__init__ for the default | ||||
default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items())) | ||||
Brian Granger
|
r2294 | #----------------------------------------------------------------------------- | ||
# Service factories | ||||
#----------------------------------------------------------------------------- | ||||
Brian Granger
|
r2288 | |||
class FCClientServiceFactory(FCServiceFactory): | ||||
"""A Foolscap implementation of the client services.""" | ||||
cert_file = Str('ipcontroller-client.pem', config=True) | ||||
Brian Granger
|
r2297 | interfaces = Instance(klass=Config, kw=default_client_interfaces, | ||
Brian Granger
|
r2288 | allow_none=False, config=True) | ||
class FCEngineServiceFactory(FCServiceFactory): | ||||
"""A Foolscap implementation of the engine services.""" | ||||
cert_file = Str('ipcontroller-engine.pem', config=True) | ||||
Brian Granger
|
r2297 | interfaces = Instance(klass=dict, kw=default_engine_interfaces, | ||
Brian Granger
|
r2288 | allow_none=False, config=True) | ||
#----------------------------------------------------------------------------- | ||||
# The main application | ||||
#----------------------------------------------------------------------------- | ||||
cl_args = ( | ||||
# Client config | ||||
(('--client-ip',), dict( | ||||
type=str, dest='FCClientServiceFactory.ip', default=NoConfigDefault, | ||||
Brian Granger
|
r2301 | help='The IP address or hostname the controller will listen on for ' | ||
'client connections.', | ||||
Brian Granger
|
r2288 | metavar='FCClientServiceFactory.ip') | ||
), | ||||
(('--client-port',), dict( | ||||
type=int, dest='FCClientServiceFactory.port', default=NoConfigDefault, | ||||
Brian Granger
|
r2301 | help='The port the controller will listen on for client connections. ' | ||
'The default is to use 0, which will autoselect an open port.', | ||||
Brian Granger
|
r2288 | metavar='FCClientServiceFactory.port') | ||
), | ||||
(('--client-location',), dict( | ||||
type=str, dest='FCClientServiceFactory.location', default=NoConfigDefault, | ||||
Brian Granger
|
r2301 | help='The hostname or IP that clients should connect to. This does ' | ||
'not control which interface the controller listens on. Instead, this ' | ||||
'determines the hostname/IP that is listed in the FURL, which is how ' | ||||
'clients know where to connect. Useful if the controller is listening ' | ||||
'on multiple interfaces.', | ||||
Brian Granger
|
r2288 | metavar='FCClientServiceFactory.location') | ||
), | ||||
# Engine config | ||||
(('--engine-ip',), dict( | ||||
type=str, dest='FCEngineServiceFactory.ip', default=NoConfigDefault, | ||||
Brian Granger
|
r2301 | help='The IP address or hostname the controller will listen on for ' | ||
'engine connections.', | ||||
Brian Granger
|
r2288 | metavar='FCEngineServiceFactory.ip') | ||
), | ||||
(('--engine-port',), dict( | ||||
type=int, dest='FCEngineServiceFactory.port', default=NoConfigDefault, | ||||
Brian Granger
|
r2301 | help='The port the controller will listen on for engine connections. ' | ||
'The default is to use 0, which will autoselect an open port.', | ||||
Brian Granger
|
r2288 | metavar='FCEngineServiceFactory.port') | ||
), | ||||
(('--engine-location',), dict( | ||||
type=str, dest='FCEngineServiceFactory.location', default=NoConfigDefault, | ||||
Brian Granger
|
r2301 | help='The hostname or IP that engines should connect to. This does ' | ||
'not control which interface the controller listens on. Instead, this ' | ||||
'determines the hostname/IP that is listed in the FURL, which is how ' | ||||
'engines know where to connect. Useful if the controller is listening ' | ||||
'on multiple interfaces.', | ||||
Brian Granger
|
r2288 | metavar='FCEngineServiceFactory.location') | ||
), | ||||
# Global config | ||||
Brian Granger
|
r2294 | (('--log-to-file',), dict( | ||
action='store_true', dest='Global.log_to_file', default=NoConfigDefault, | ||||
help='Log to a file in the log directory (default is stdout)') | ||||
Brian Granger
|
r2288 | ), | ||
Brian Granger
|
r2294 | (('-r','--reuse-furls'), dict( | ||
Brian Granger
|
r2288 | action='store_true', dest='Global.reuse_furls', default=NoConfigDefault, | ||
Brian Granger
|
r2301 | help='Try to reuse all FURL files. If this is not set all FURL files ' | ||
'are deleted before the controller starts. This must be set if ' | ||||
'specific ports are specified by --engine-port or --client-port.') | ||||
Brian Granger
|
r2297 | ), | ||
(('-ns','--no-security'), dict( | ||||
action='store_false', dest='Global.secure', default=NoConfigDefault, | ||||
help='Turn off SSL encryption for all connections.') | ||||
Brian Granger
|
r2296 | ) | ||
Brian Granger
|
r2288 | ) | ||
Brian Granger
|
r2301 | class IPControllerAppCLConfigLoader(AppWithClusterDirArgParseConfigLoader): | ||
Brian Granger
|
r2288 | |||
arguments = cl_args | ||||
Brian Granger
|
r2297 | default_config_file_name = 'ipcontroller_config.py' | ||
Brian Granger
|
r2288 | |||
Brian Granger
|
r2296 | |||
Brian Granger
|
r2301 | class IPControllerApp(ApplicationWithClusterDir): | ||
Brian Granger
|
r2288 | |||
name = 'ipcontroller' | ||||
Brian Granger
|
r2296 | description = 'Start the IPython controller for parallel computing.' | ||
Brian Granger
|
r2297 | config_file_name = default_config_file_name | ||
Brian Granger
|
r2303 | auto_create_cluster_dir = True | ||
Brian Granger
|
r2288 | |||
def create_default_config(self): | ||||
super(IPControllerApp, self).create_default_config() | ||||
self.default_config.Global.reuse_furls = False | ||||
Brian Granger
|
r2297 | self.default_config.Global.secure = True | ||
Brian Granger
|
r2288 | self.default_config.Global.import_statements = [] | ||
Brian Granger
|
r2294 | self.default_config.Global.log_to_file = False | ||
Brian Granger
|
r2297 | def create_command_line_config(self): | ||
"""Create and return a command line config loader.""" | ||||
return IPControllerAppCLConfigLoader( | ||||
description=self.description, | ||||
version=release.version | ||||
) | ||||
def post_load_command_line_config(self): | ||||
# Now setup reuse_furls | ||||
Brian Granger
|
r2301 | c = self.command_line_config | ||
if hasattr(c.Global, 'reuse_furls'): | ||||
c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls | ||||
c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls | ||||
del c.Global.reuse_furls | ||||
if hasattr(c.Global, 'secure'): | ||||
c.FCClientServiceFactory.secure = c.Global.secure | ||||
c.FCEngineServiceFactory.secure = c.Global.secure | ||||
del c.Global.secure | ||||
Brian Granger
|
r2297 | |||
Brian Granger
|
r2294 | def pre_construct(self): | ||
Brian Granger
|
r2301 | # The log and security dirs were set earlier, but here we put them | ||
# into the config and log them. | ||||
Brian Granger
|
r2299 | config = self.master_config | ||
Brian Granger
|
r2301 | sdir = self.cluster_dir_obj.security_dir | ||
self.security_dir = config.Global.security_dir = sdir | ||||
ldir = self.cluster_dir_obj.log_dir | ||||
self.log_dir = config.Global.log_dir = ldir | ||||
Brian Granger
|
r2302 | self.log.info("Cluster directory set to: %s" % self.cluster_dir) | ||
Brian Granger
|
r2299 | self.log.info("Log directory set to: %s" % self.log_dir) | ||
self.log.info("Security directory set to: %s" % self.security_dir) | ||||
Brian Granger
|
r2294 | |||
Brian Granger
|
r2288 | def construct(self): | ||
# I am a little hesitant to put these into InteractiveShell itself. | ||||
# But that might be the place for them | ||||
sys.path.insert(0, '') | ||||
self.start_logging() | ||||
self.import_statements() | ||||
Brian Granger
|
r2297 | |||
Brian Granger
|
r2288 | # Create the service hierarchy | ||
self.main_service = service.MultiService() | ||||
# The controller service | ||||
controller_service = controllerservice.ControllerService() | ||||
controller_service.setServiceParent(self.main_service) | ||||
# The client tub and all its refereceables | ||||
csfactory = FCClientServiceFactory(self.master_config, controller_service) | ||||
client_service = csfactory.create() | ||||
client_service.setServiceParent(self.main_service) | ||||
# The engine tub | ||||
esfactory = FCEngineServiceFactory(self.master_config, controller_service) | ||||
engine_service = esfactory.create() | ||||
engine_service.setServiceParent(self.main_service) | ||||
def start_logging(self): | ||||
Brian Granger
|
r2294 | if self.master_config.Global.log_to_file: | ||
log_filename = self.name + '-' + str(os.getpid()) + '.log' | ||||
logfile = os.path.join(self.log_dir, log_filename) | ||||
open_log_file = open(logfile, 'w') | ||||
Brian Granger
|
r2288 | else: | ||
Brian Granger
|
r2294 | open_log_file = sys.stdout | ||
log.startLogging(open_log_file) | ||||
Brian Granger
|
r2288 | |||
def import_statements(self): | ||||
statements = self.master_config.Global.import_statements | ||||
for s in statements: | ||||
try: | ||||
Brian Granger
|
r2297 | log.msg("Executing statement: '%s'" % s) | ||
Brian Granger
|
r2288 | exec s in globals(), locals() | ||
except: | ||||
Brian Granger
|
r2297 | log.msg("Error running statement: %s" % s) | ||
Brian Granger
|
r2288 | |||
def start_app(self): | ||||
# Start the controller service and set things running | ||||
self.main_service.startService() | ||||
reactor.run() | ||||
Brian Granger
|
r2296 | |||
def launch_new_instance(): | ||||
"""Create and run the IPython controller""" | ||||
Brian Granger
|
r2288 | app = IPControllerApp() | ||
app.start() | ||||
Brian Granger
|
r2302 | |||
if __name__ == '__main__': | ||||
launch_new_instance() | ||||