diff --git a/IPython/config/default/ipcontroller_config.py b/IPython/config/default/ipcontroller_config.py index c4e0d8f..216d855 100644 --- a/IPython/config/default/ipcontroller_config.py +++ b/IPython/config/default/ipcontroller_config.py @@ -6,10 +6,14 @@ c = get_config() # Global configuration #----------------------------------------------------------------------------- -c.Global.logfile = '' -c.Global.import_statement = '' +c.Global.log_to_file = False +c.Global.import_statements = [] c.Global.reuse_furls = False +# You shouldn't have to edit these +c.Global.log_dir_name = 'log' +c.Global.security_dir_name = 'security' + #----------------------------------------------------------------------------- # Configure the client services @@ -19,18 +23,23 @@ c.FCClientServiceFactory.ip = '' c.FCClientServiceFactory.port = 0 c.FCClientServiceFactory.location = '' c.FCClientServiceFactory.secure = True +c.FCClientServiceFactory.reuse_furls = False c.FCClientServiceFactory.cert_file = 'ipcontroller-client.pem' c.FCClientServiceFactory.Interfaces.Task.interface_chain = [ 'IPython.kernel.task.ITaskController', 'IPython.kernel.taskfc.IFCTaskController' ] +# This is just the filename of the furl file. The path is always the +# security dir of the cluster directory. c.FCClientServiceFactory.Interfaces.Task.furl_file = 'ipcontroller-tc.furl' c.FCClientServiceFactory.Interfaces.MultiEngine.interface_chain = [ 'IPython.kernel.multiengine.IMultiEngine', 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine' ] +# This is just the filename of the furl file. The path is always the +# security dir of the cluster directory. c.FCClientServiceFactory.Interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl' @@ -42,19 +51,17 @@ c.FCEngineServiceFactory.ip = '' c.FCEngineServiceFactory.port = 0 c.FCEngineServiceFactory.location = '' c.FCEngineServiceFactory.secure = True +c.FCEngineServiceFactory.reuse_furls = False c.FCEngineServiceFactory.cert_file = 'ipcontroller-engine.pem' -engine_config = Config() -engine_config.furl_file = -c.Global.engine_furl_file = 'ipcontroller-engine.furl' -c.Global.engine_fc_interface = 'IPython.kernel.enginefc.IFCControllerBase' - +c.FCEngineServiceFactory.Intefaces.Default.interface_chain = [ + 'IPython.kernel.enginefc.IFCControllerBase' +] +# This is just the filename of the furl file. The path is always the +# security dir of the cluster directory. +c.FCEngineServiceFactory.Intefaces.Default.furl_file = 'ipcontroller-engine.furl' -CLIENT_INTERFACES = dict( - TASK = dict(FURL_FILE = 'ipcontroller-tc.furl'), - MULTIENGINE = dict(FURLFILE='ipcontroller-mec.furl') -) diff --git a/IPython/core/application.py b/IPython/core/application.py index ee9ada8..41fea47 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -46,7 +46,7 @@ class IPythonArgParseConfigLoader(ArgParseConfigLoader): """Default command line options for IPython based applications.""" def _add_other_arguments(self): - self.parser.add_argument('-ipythondir', '--ipythondir', + self.parser.add_argument('-ipythondir', '--ipython-dir', dest='Global.ipythondir',type=str, help='Set to override default location of Global.ipythondir.', default=NoConfigDefault, @@ -77,6 +77,8 @@ class Application(object): config_file_name = 'ipython_config.py' name = 'ipython' + default_log_level = logging.WARN + def __init__(self): self.init_logger() @@ -85,7 +87,7 @@ class Application(object): def init_logger(self): self.log = logging.getLogger(self.__class__.__name__) # This is used as the default until the command line arguments are read. - self.log.setLevel(logging.WARN) + self.log.setLevel(self.default_log_level) self._log_handler = logging.StreamHandler() self._log_formatter = logging.Formatter("[%(name)s] %(message)s") self._log_handler.setFormatter(self._log_formatter) @@ -102,16 +104,23 @@ class Application(object): def start(self): """Start the application.""" self.attempt(self.create_default_config) + self.log_default_config() + self.set_default_config_log_level() self.attempt(self.pre_load_command_line_config) self.attempt(self.load_command_line_config, action='abort') + self.set_command_line_config_log_level() self.attempt(self.post_load_command_line_config) + self.log_command_line_config() self.attempt(self.find_ipythondir) self.attempt(self.find_config_file_name) self.attempt(self.find_config_file_paths) self.attempt(self.pre_load_file_config) self.attempt(self.load_file_config) + self.set_file_config_log_level() self.attempt(self.post_load_file_config) + self.log_file_config() self.attempt(self.merge_configs) + self.log_master_config() self.attempt(self.pre_construct) self.attempt(self.construct) self.attempt(self.post_construct) @@ -132,9 +141,18 @@ class Application(object): """ self.default_config = Config() self.default_config.Global.ipythondir = get_ipython_dir() + + def log_default_config(self): self.log.debug('Default config loaded:') self.log.debug(repr(self.default_config)) + def set_default_config_log_level(self): + try: + self.log_level = self.default_config.Global.log_level + except AttributeError: + # Fallback to the default_log_level class attribute + pass + def create_command_line_config(self): """Create and return a command line config loader.""" return IPythonArgParseConfigLoader(description=self.name) @@ -144,26 +162,25 @@ class Application(object): pass def load_command_line_config(self): - """Load the command line config. - - This method also sets ``self.debug``. - """ - + """Load the command line config.""" loader = self.create_command_line_config() self.command_line_config = loader.load_config() self.extra_args = loader.get_extra_args() + def set_command_line_config_log_level(self): try: self.log_level = self.command_line_config.Global.log_level except AttributeError: - pass # Use existing value which is set in Application.init_logger. - self.log.debug("Command line config loaded:") - self.log.debug(repr(self.command_line_config)) + pass def post_load_command_line_config(self): """Do actions just after loading the command line config.""" pass + def log_command_line_config(self): + self.log.debug("Command line config loaded:") + self.log.debug(repr(self.command_line_config)) + def find_ipythondir(self): """Set the IPython directory. @@ -180,16 +197,19 @@ class Application(object): self.ipythondir = self.default_config.Global.ipythondir sys.path.append(os.path.abspath(self.ipythondir)) if not os.path.isdir(self.ipythondir): - os.makedirs(self.ipythondir, mode = 0777) + os.makedirs(self.ipythondir, mode=0777) self.log.debug("IPYTHONDIR set to: %s" % self.ipythondir) def find_config_file_name(self): """Find the config file name for this application. + This must set ``self.config_file_name`` to the filename of the + config file to use (just the filename). The search paths for the + config file are set in :meth:`find_config_file_paths` and then passed + to the config file loader where they are resolved to an absolute path. + If a profile has been set at the command line, this will resolve - it. The search paths for the config file are set in - :meth:`find_config_file_paths` and then passed to the config file - loader where they are resolved to an absolute path. + it. """ try: @@ -206,7 +226,11 @@ class Application(object): pass def find_config_file_paths(self): - """Set the search paths for resolving the config file.""" + """Set the search paths for resolving the config file. + + This must set ``self.config_file_paths`` to a sequence of search + paths to pass to the config file loader. + """ self.config_file_paths = (os.getcwd(), self.ipythondir) def pre_load_file_config(self): @@ -236,12 +260,11 @@ class Application(object): self.log.warn("Error loading config file: <%s>" % \ self.config_file_name, exc_info=True) self.file_config = Config() - else: - self.log.debug("Config file loaded: <%s>" % loader.full_filename) - self.log.debug(repr(self.file_config)) + + def set_file_config_log_level(self): # We need to keeep self.log_level updated. But we only use the value # of the file_config if a value was not specified at the command - # line. + # line, because the command line overrides everything. if not hasattr(self.command_line_config.Global, 'log_level'): try: self.log_level = self.file_config.Global.log_level @@ -252,6 +275,11 @@ class Application(object): """Do actions after the config file is loaded.""" pass + def log_file_config(self): + if hasattr(self.file_config.Global, 'config_file'): + self.log.debug("Config file loaded: <%s>" % self.file_config.Global.config_file) + self.log.debug(repr(self.file_config)) + def merge_configs(self): """Merge the default, command line and file config objects.""" config = Config() @@ -259,6 +287,8 @@ class Application(object): config._merge(self.file_config) config._merge(self.command_line_config) self.master_config = config + + def log_master_config(self): self.log.debug("Master config created:") self.log.debug(repr(self.master_config)) diff --git a/IPython/kernel/client.py b/IPython/kernel/client.py index 905c395..9b2499f 100644 --- a/IPython/kernel/client.py +++ b/IPython/kernel/client.py @@ -41,7 +41,7 @@ from IPython.kernel.twistedutil import blockingCallFromThread # These enable various things from IPython.kernel import codeutil -import IPython.kernel.magic +# import IPython.kernel.magic # Other things that the user will need from IPython.kernel.task import MapTask, StringTask diff --git a/IPython/kernel/fcutil.py b/IPython/kernel/fcutil.py index 4d6b310..e7ec060 100644 --- a/IPython/kernel/fcutil.py +++ b/IPython/kernel/fcutil.py @@ -152,24 +152,42 @@ class FCServiceFactory(AdaptedConfiguredObjectFactory): keys: furl_file and interface_chain. The other attributes are the standard ones for Foolscap. -""" + """ ip = Str('', config=True) port = Int(0, config=True) secure = Bool(True, config=True) cert_file = Str('', config=True) location = Str('', config=True) + reuse_furls = Bool(False, config=True) Interfaces = Instance(klass=Config, kw={}, allow_none=False, config=True) + def __init__(self, config, adaptee): + super(FCServiceFactory, self).__init__(config, adaptee) + self._check_reuse_furls() + def _ip_changed(self, name, old, new): if new == 'localhost' or new == '127.0.0.1': self.location = '127.0.0.1' + def _check_reuse_furls(self): + if not self.reuse_furls: + furl_files = [i.furl_file for i in self.Interfaces.values()] + for ff in furl_files: + fullfile = self._get_security_file(ff) + if os.path.isfile(fullfile): + os.remove(fullfile) + + def _get_security_file(self, filename): + return os.path.join(self.config.Global.security_dir, filename) + def create(self): """Create and return the Foolscap tub with everything running.""" self.tub, self.listener = make_tub( - self.ip, self.port, self.secure, self.cert_file) + self.ip, self.port, self.secure, self._get_security_file(self.cert_file)) + log.msg("Created a tub and listener [%r]: %r, %r" % (self.__class__, self.tub, self.listener)) + log.msg("Interfaces to register [%r]: %r" % (self.__class__, self.Interfaces)) if not self.secure: log.msg("WARNING: running with no security: %s" % self.__class__.__name__) reactor.callWhenRunning(self.set_location_and_register) @@ -189,13 +207,14 @@ class FCServiceFactory(AdaptedConfiguredObjectFactory): """Run through the interfaces, adapt and register.""" for ifname, ifconfig in self.Interfaces.iteritems(): + ff = self._get_security_file(ifconfig.furl_file) log.msg("Adapting %r to interface: %s" % (self.adaptee, ifname)) - log.msg("Saving furl for interface [%s] to file: %s" % (ifname, ifconfig.furl_file)) - check_furl_file_security(ifconfig.furl_file, self.secure) + log.msg("Saving furl for interface [%s] to file: %s" % (ifname, ff)) + check_furl_file_security(ff, self.secure) adaptee = self.adaptee for i in ifconfig.interface_chain: adaptee = import_item(i)(adaptee) - d.addCallback(self.register, adaptee, furl_file=ifconfig.furl_file) + d.addCallback(self.register, adaptee, furl_file=ff) def register(self, empty, ref, furl_file): """Register the reference with the FURL file. diff --git a/IPython/kernel/scripts/ipcontrollerapp.py b/IPython/kernel/scripts/ipcontrollerapp.py index 3d85dd2..5f828ba 100644 --- a/IPython/kernel/scripts/ipcontrollerapp.py +++ b/IPython/kernel/scripts/ipcontrollerapp.py @@ -16,6 +16,7 @@ The IPython controller application #----------------------------------------------------------------------------- import copy +import logging import os import sys @@ -40,7 +41,7 @@ from IPython.kernel.configobjfactory import ( from IPython.kernel.fcutil import FCServiceFactory #----------------------------------------------------------------------------- -# Components for creating services +# Default interfaces #----------------------------------------------------------------------------- @@ -50,11 +51,13 @@ default_client_interfaces.Task.interface_chain = [ 'IPython.kernel.task.ITaskController', 'IPython.kernel.taskfc.IFCTaskController' ] + default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl' default_client_interfaces.MultiEngine.interface_chain = [ 'IPython.kernel.multiengine.IMultiEngine', 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine' ] + default_client_interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl' # Make this a dict we can pass to Config.__init__ for the default @@ -67,12 +70,17 @@ default_engine_interfaces = Config() default_engine_interfaces.Default.interface_chain = [ 'IPython.kernel.enginefc.IFCControllerBase' ] + default_engine_interfaces.Default.furl_file = 'ipcontroller-engine.furl' # Make this a dict we can pass to Config.__init__ for the default default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items())) +#----------------------------------------------------------------------------- +# Service factories +#----------------------------------------------------------------------------- + class FCClientServiceFactory(FCServiceFactory): """A Foolscap implementation of the client services.""" @@ -86,7 +94,7 @@ class FCEngineServiceFactory(FCServiceFactory): """A Foolscap implementation of the engine services.""" cert_file = Str('ipcontroller-engine.pem', config=True) - interfaces = Instance(klass=dict, kw=default_engine_interfaces, + Interfaces = Instance(klass=dict, kw=default_engine_interfaces, allow_none=False, config=True) @@ -116,21 +124,6 @@ cl_args = ( action='store_false', dest='FCClientServiceFactory.secure', default=NoConfigDefault, help='Turn off all client security.') ), - (('--client-cert-file',), dict( - type=str, dest='FCClientServiceFactory.cert_file', default=NoConfigDefault, - help='File to store the client SSL certificate in.', - metavar='FCClientServiceFactory.cert_file') - ), - (('--task-furl-file',), dict( - type=str, dest='FCClientServiceFactory.Interfaces.Task.furl_file', default=NoConfigDefault, - help='File to store the FURL in for task clients to connect with.', - metavar='FCClientServiceFactory.Interfaces.Task.furl_file') - ), - (('--multiengine-furl-file',), dict( - type=str, dest='FCClientServiceFactory.Interfaces.MultiEngine.furl_file', default=NoConfigDefault, - help='File to store the FURL in for multiengine clients to connect with.', - metavar='FCClientServiceFactory.Interfaces.MultiEngine.furl_file') - ), # Engine config (('--engine-ip',), dict( type=str, dest='FCEngineServiceFactory.ip', default=NoConfigDefault, @@ -151,26 +144,20 @@ cl_args = ( action='store_false', dest='FCEngineServiceFactory.secure', default=NoConfigDefault, help='Turn off all engine security.') ), - (('--engine-cert-file',), dict( - type=str, dest='FCEngineServiceFactory.cert_file', default=NoConfigDefault, - help='File to store the client SSL certificate in.', - metavar='FCEngineServiceFactory.cert_file') - ), - (('--engine-furl-file',), dict( - type=str, dest='FCEngineServiceFactory.Interfaces.Default.furl_file', default=NoConfigDefault, - help='File to store the FURL in for engines to connect with.', - metavar='FCEngineServiceFactory.Interfaces.Default.furl_file') - ), # Global config - (('-l','--logfile'), dict( - type=str, dest='Global.logfile', default=NoConfigDefault, - help='Log file name (default is stdout)', - metavar='Global.logfile') + (('--log-to-file',), dict( + action='store_true', dest='Global.log_to_file', default=NoConfigDefault, + help='Log to a file in the log directory (default is stdout)') ), - (('-r',), dict( + (('-r','--reuse-furls'), dict( action='store_true', dest='Global.reuse_furls', default=NoConfigDefault, help='Try to reuse all FURL files.') - ) + ), + (('-cluster_dir', '--cluster-dir',), dict( + type=str, dest='Global.cluster_dir', default=NoConfigDefault, + help='Absolute or relative path to the cluster directory.', + metavar='Global.cluster_dir') + ), ) @@ -185,12 +172,18 @@ class IPControllerApp(Application): name = 'ipcontroller' config_file_name = _default_config_file_name + default_log_level = logging.DEBUG def create_default_config(self): super(IPControllerApp, self).create_default_config() - self.default_config.Global.logfile = '' self.default_config.Global.reuse_furls = False self.default_config.Global.import_statements = [] + self.default_config.Global.profile = 'default' + self.default_config.Global.log_dir_name = 'log' + self.default_config.Global.security_dir_name = 'security' + self.default_config.Global.log_to_file = False + # Resolve the default cluster_dir using the default profile + self.default_config.Global.cluster_dir = '' def create_command_line_config(self): """Create and return a command line config loader.""" @@ -199,6 +192,75 @@ class IPControllerApp(Application): description="Start an IPython controller", version=release.version) + def find_config_file_name(self): + """Find the config file name for this application.""" + self.find_cluster_dir() + self.create_cluster_dir() + + def find_cluster_dir(self): + """This resolves into full paths, the various cluster directories. + + This method must set ``self.cluster_dir`` to the full paths of + the directory. + """ + # Ignore self.command_line_config.Global.config_file + # Instead, first look for an explicit cluster_dir + try: + self.cluster_dir = self.command_line_config.Global.cluster_dir + except AttributeError: + self.cluster_dir = self.default_config.Global.cluster_dir + self.cluster_dir = os.path.expandvars(os.path.expanduser(self.cluster_dir)) + if not self.cluster_dir: + # Then look for a profile + try: + self.profile = self.command_line_config.Global.profile + except AttributeError: + self.profile = self.default_config.Global.profile + cluster_dir_name = 'cluster_' + self.profile + try_this = os.path.join(os.getcwd(), cluster_dir_name) + if os.path.isdir(try_this): + self.cluster_dir = try_this + else: + self.cluster_dir = os.path.join(self.ipythondir, cluster_dir_name) + # These have to be set because they could be different from the one + # that we just computed. Because command line has the highest + # priority, this will always end up in the master_config. + self.default_config.Global.cluster_dir = self.cluster_dir + self.command_line_config.Global.cluster_dir = self.cluster_dir + + def create_cluster_dir(self): + """Make sure that the cluster, security and log dirs exist.""" + if not os.path.isdir(self.cluster_dir): + os.makedirs(self.cluster_dir, mode=0777) + + def find_config_file_paths(self): + """Set the search paths for resolving the config file.""" + self.config_file_paths = (self.cluster_dir,) + + def pre_construct(self): + # Now set the security_dir and log_dir and create them. We use + # the names an construct the absolute paths. + security_dir = os.path.join(self.master_config.Global.cluster_dir, + self.master_config.Global.security_dir_name) + log_dir = os.path.join(self.master_config.Global.cluster_dir, + self.master_config.Global.log_dir_name) + if not os.path.isdir(security_dir): + os.mkdir(security_dir, 0700) + else: + os.chmod(security_dir, 0700) + if not os.path.isdir(log_dir): + os.mkdir(log_dir, 0777) + + self.security_dir = self.master_config.Global.security_dir = security_dir + self.log_dir = self.master_config.Global.log_dir = log_dir + + # Now setup reuse_furls + if hasattr(self.master_config.Global.reuse_furls): + self.master_config.FCClientServiceFactory.reuse_furls = \ + self.master_config.Global.reuse_furls + self.master_config.FCEngineServiceFactory.reuse_furls = \ + self.master_config.Global.reuse_furls + def construct(self): # I am a little hesitant to put these into InteractiveShell itself. # But that might be the place for them @@ -206,7 +268,6 @@ class IPControllerApp(Application): self.start_logging() self.import_statements() - self.reuse_furls() # Create the service hierarchy self.main_service = service.MultiService() @@ -223,16 +284,13 @@ class IPControllerApp(Application): engine_service.setServiceParent(self.main_service) def start_logging(self): - logfile = self.master_config.Global.logfile - if logfile: - logfile = logfile + str(os.getpid()) + '.log' - try: - openLogFile = open(logfile, 'w') - except: - openLogFile = sys.stdout + if self.master_config.Global.log_to_file: + log_filename = self.name + '-' + str(os.getpid()) + '.log' + logfile = os.path.join(self.log_dir, log_filename) + open_log_file = open(logfile, 'w') else: - openLogFile = sys.stdout - log.startLogging(openLogFile) + open_log_file = sys.stdout + log.startLogging(open_log_file) def import_statements(self): statements = self.master_config.Global.import_statements @@ -242,20 +300,6 @@ class IPControllerApp(Application): except: log.msg("Error running import statement: %s" % s) - def reuse_furls(self): - # This logic might need to be moved into the components - # Delete old furl files unless the reuse_furls is set - reuse = self.master_config.Global.reuse_furls - # if not reuse: - # paths = ( - # self.master_config.FCEngineServiceFactory.Interfaces.Default.furl_file, - # self.master_config.FCClientServiceFactory.Interfaces.Task.furl_file, - # self.master_config.FCClientServiceFactory.Interfaces.MultiEngine.furl_file - # ) - # for p in paths: - # if os.path.isfile(p): - # os.remove(p) - def start_app(self): # Start the controller service and set things running self.main_service.startService()