General work on the kernel config.

IPython/config/default/ipcontroller_config.py

@@ -6,10 +6,14 b' c = get_config()'
 # Global configuration
 #-----------------------------------------------------------------------------
 
-c.Global.logfile = ''
-c.Global.import_statement = ''
+c.Global.log_to_file = False
+c.Global.import_statements = []
 c.Global.reuse_furls = False
 
+# You shouldn't have to edit these
+c.Global.log_dir_name = 'log'
+c.Global.security_dir_name = 'security'
+
 
 #-----------------------------------------------------------------------------
 # Configure the client services
@@ -19,18 +23,23 b" c.FCClientServiceFactory.ip = ''"
 c.FCClientServiceFactory.port = 0 
 c.FCClientServiceFactory.location = ''
 c.FCClientServiceFactory.secure =  True
+c.FCClientServiceFactory.reuse_furls = False
 c.FCClientServiceFactory.cert_file = 'ipcontroller-client.pem'
 
 c.FCClientServiceFactory.Interfaces.Task.interface_chain = [
     'IPython.kernel.task.ITaskController',
     'IPython.kernel.taskfc.IFCTaskController'
 ]
+# This is just the filename of the furl file.  The path is always the
+# security dir of the cluster directory.
 c.FCClientServiceFactory.Interfaces.Task.furl_file = 'ipcontroller-tc.furl'
 
 c.FCClientServiceFactory.Interfaces.MultiEngine.interface_chain = [
     'IPython.kernel.multiengine.IMultiEngine',
     'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
 ]
+# This is just the filename of the furl file.  The path is always the
+# security dir of the cluster directory.
 c.FCClientServiceFactory.Interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl'
 
 
@@ -42,19 +51,17 b" c.FCEngineServiceFactory.ip = ''"
 c.FCEngineServiceFactory.port = 0
 c.FCEngineServiceFactory.location = ''
 c.FCEngineServiceFactory.secure = True
+c.FCEngineServiceFactory.reuse_furls = False
 c.FCEngineServiceFactory.cert_file = 'ipcontroller-engine.pem'
 
-engine_config = Config()
-engine_config.furl_file = 
-c.Global.engine_furl_file = 'ipcontroller-engine.furl'
-c.Global.engine_fc_interface = 'IPython.kernel.enginefc.IFCControllerBase'
-
+c.FCEngineServiceFactory.Intefaces.Default.interface_chain = [
+    'IPython.kernel.enginefc.IFCControllerBase'
+]
 
+# This is just the filename of the furl file.  The path is always the
+# security dir of the cluster directory.
+c.FCEngineServiceFactory.Intefaces.Default.furl_file = 'ipcontroller-engine.furl'
 
 
 
-CLIENT_INTERFACES = dict(
-    TASK = dict(FURL_FILE = 'ipcontroller-tc.furl'),
-    MULTIENGINE = dict(FURLFILE='ipcontroller-mec.furl')
-)
 

IPython/core/application.py

@@ -46,7 +46,7 b' class IPythonArgParseConfigLoader(ArgParseConfigLoader):'
     """Default command line options for IPython based applications."""
 
     def _add_other_arguments(self):
-        self.parser.add_argument('-ipythondir', '--ipythondir', 
+        self.parser.add_argument('-ipythondir', '--ipython-dir', 
             dest='Global.ipythondir',type=str,
             help='Set to override default location of Global.ipythondir.',
             default=NoConfigDefault,
@@ -77,6 +77,8 b' class Application(object):'
 
     config_file_name = 'ipython_config.py'
     name = 'ipython'
+    default_log_level = logging.WARN
+    
 
     def __init__(self):
         self.init_logger()
@@ -85,7 +87,7 b' class Application(object):'
     def init_logger(self):
         self.log = logging.getLogger(self.__class__.__name__)
         # This is used as the default until the command line arguments are read.
-        self.log.setLevel(logging.WARN)
+        self.log.setLevel(self.default_log_level)
         self._log_handler = logging.StreamHandler()
         self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
         self._log_handler.setFormatter(self._log_formatter)
@@ -102,16 +104,23 b' class Application(object):'
     def start(self):
         """Start the application."""
         self.attempt(self.create_default_config)
+        self.log_default_config()
+        self.set_default_config_log_level()
         self.attempt(self.pre_load_command_line_config)
         self.attempt(self.load_command_line_config, action='abort')
+        self.set_command_line_config_log_level()
         self.attempt(self.post_load_command_line_config)
+        self.log_command_line_config()
         self.attempt(self.find_ipythondir)
         self.attempt(self.find_config_file_name)
         self.attempt(self.find_config_file_paths)
         self.attempt(self.pre_load_file_config)
         self.attempt(self.load_file_config)
+        self.set_file_config_log_level()
         self.attempt(self.post_load_file_config)
+        self.log_file_config()
         self.attempt(self.merge_configs)
+        self.log_master_config()
         self.attempt(self.pre_construct)
         self.attempt(self.construct)
         self.attempt(self.post_construct)
@@ -132,9 +141,18 b' class Application(object):'
         """
         self.default_config = Config()
         self.default_config.Global.ipythondir = get_ipython_dir()
+
+    def log_default_config(self):
         self.log.debug('Default config loaded:')
         self.log.debug(repr(self.default_config))
 
+    def set_default_config_log_level(self):
+        try:
+            self.log_level = self.default_config.Global.log_level
+        except AttributeError:
+            # Fallback to the default_log_level class attribute
+            pass
+
     def create_command_line_config(self):
         """Create and return a command line config loader."""
         return IPythonArgParseConfigLoader(description=self.name)
@@ -144,26 +162,25 b' class Application(object):'
         pass
 
     def load_command_line_config(self):
-        """Load the command line config.
-
-        This method also sets ``self.debug``.
-        """
-
+        """Load the command line config."""
         loader = self.create_command_line_config()
         self.command_line_config = loader.load_config()
         self.extra_args = loader.get_extra_args()
 
+    def set_command_line_config_log_level(self):
         try:
             self.log_level = self.command_line_config.Global.log_level
         except AttributeError:
-            pass # Use existing value which is set in Application.init_logger.
-        self.log.debug("Command line config loaded:")
-        self.log.debug(repr(self.command_line_config))
+            pass
 
     def post_load_command_line_config(self):
         """Do actions just after loading the command line config."""
         pass
 
+    def log_command_line_config(self):
+        self.log.debug("Command line config loaded:")
+        self.log.debug(repr(self.command_line_config))
+
     def find_ipythondir(self):
         """Set the IPython directory.
 
@@ -180,16 +197,19 b' class Application(object):'
             self.ipythondir = self.default_config.Global.ipythondir
         sys.path.append(os.path.abspath(self.ipythondir))
         if not os.path.isdir(self.ipythondir):
-            os.makedirs(self.ipythondir, mode = 0777)
+            os.makedirs(self.ipythondir, mode=0777)
         self.log.debug("IPYTHONDIR set to: %s" % self.ipythondir)
 
     def find_config_file_name(self):
         """Find the config file name for this application.
 
+        This must set ``self.config_file_name`` to the filename of the
+        config file to use (just the filename). The search paths for the
+        config file are set in :meth:`find_config_file_paths` and then passed
+        to the config file loader where they are resolved to an absolute path.
+
         If a profile has been set at the command line, this will resolve
-        it.  The search paths for the config file are set in
-        :meth:`find_config_file_paths` and then passed to the config file
-        loader where they are resolved to an absolute path.
+        it.
         """
 
         try:
@@ -206,7 +226,11 b' class Application(object):'
             pass
 
     def find_config_file_paths(self):
-        """Set the search paths for resolving the config file."""
+        """Set the search paths for resolving the config file.
+
+        This must set ``self.config_file_paths`` to a sequence of search
+        paths to pass to the config file loader.
+        """
         self.config_file_paths = (os.getcwd(), self.ipythondir)
 
     def pre_load_file_config(self):
@@ -236,12 +260,11 b' class Application(object):'
             self.log.warn("Error loading config file: <%s>" % \
                            self.config_file_name, exc_info=True)
             self.file_config = Config()
-        else:
-            self.log.debug("Config file loaded: <%s>" % loader.full_filename)
-            self.log.debug(repr(self.file_config))
+
+    def set_file_config_log_level(self):
         # We need to keeep self.log_level updated.  But we only use the value
         # of the file_config if a value was not specified at the command
-        # line.
+        # line, because the command line overrides everything.
         if not hasattr(self.command_line_config.Global, 'log_level'):
             try:
                 self.log_level = self.file_config.Global.log_level
@@ -252,6 +275,11 b' class Application(object):'
         """Do actions after the config file is loaded."""
         pass
 
+    def log_file_config(self):
+        if hasattr(self.file_config.Global, 'config_file'):
+            self.log.debug("Config file loaded: <%s>" % self.file_config.Global.config_file)
+            self.log.debug(repr(self.file_config))
+
     def merge_configs(self):
         """Merge the default, command line and file config objects."""
         config = Config()
@@ -259,6 +287,8 b' class Application(object):'
         config._merge(self.file_config)
         config._merge(self.command_line_config)
         self.master_config = config
+
+    def log_master_config(self):
         self.log.debug("Master config created:")
         self.log.debug(repr(self.master_config))
 

IPython/kernel/client.py

@@ -41,7 +41,7 b' from IPython.kernel.twistedutil import blockingCallFromThread'
 
 # These enable various things 
 from IPython.kernel import codeutil
-import IPython.kernel.magic
+# import IPython.kernel.magic
 
 # Other things that the user will need
 from IPython.kernel.task import MapTask, StringTask

IPython/kernel/fcutil.py

@@ -152,24 +152,42 b' class FCServiceFactory(AdaptedConfiguredObjectFactory):'
         keys: furl_file and interface_chain.
 
     The other attributes are the standard ones for Foolscap.
-"""
+    """
 
     ip = Str('', config=True)
     port = Int(0, config=True)
     secure = Bool(True, config=True)
     cert_file = Str('', config=True)
     location = Str('', config=True)
+    reuse_furls = Bool(False, config=True)
     Interfaces = Instance(klass=Config, kw={}, allow_none=False, config=True)
 
+    def __init__(self, config, adaptee):
+        super(FCServiceFactory, self).__init__(config, adaptee)
+        self._check_reuse_furls()
+
     def _ip_changed(self, name, old, new):
         if new == 'localhost' or new == '127.0.0.1':
             self.location = '127.0.0.1'
 
+    def _check_reuse_furls(self):
+        if not self.reuse_furls:
+            furl_files = [i.furl_file for i in self.Interfaces.values()]
+            for ff in furl_files:
+                fullfile = self._get_security_file(ff)
+                if os.path.isfile(fullfile):
+                    os.remove(fullfile)
+
+    def _get_security_file(self, filename):
+        return os.path.join(self.config.Global.security_dir, filename)
+
     def create(self):
         """Create and return the Foolscap tub with everything running."""
 
         self.tub, self.listener = make_tub(
-            self.ip, self.port, self.secure, self.cert_file)
+            self.ip, self.port, self.secure, self._get_security_file(self.cert_file))
+        log.msg("Created a tub and listener [%r]: %r, %r" % (self.__class__, self.tub, self.listener))
+        log.msg("Interfaces to register [%r]: %r" % (self.__class__, self.Interfaces))
         if not self.secure:
             log.msg("WARNING: running with no security: %s" % self.__class__.__name__)
         reactor.callWhenRunning(self.set_location_and_register)
@@ -189,13 +207,14 b' class FCServiceFactory(AdaptedConfiguredObjectFactory):'
         """Run through the interfaces, adapt and register."""
 
         for ifname, ifconfig in self.Interfaces.iteritems():
+            ff = self._get_security_file(ifconfig.furl_file)
             log.msg("Adapting %r to interface: %s" % (self.adaptee, ifname))
-            log.msg("Saving furl for interface [%s] to file: %s" % (ifname, ifconfig.furl_file))
-            check_furl_file_security(ifconfig.furl_file, self.secure)
+            log.msg("Saving furl for interface [%s] to file: %s" % (ifname, ff))
+            check_furl_file_security(ff, self.secure)
             adaptee = self.adaptee
             for i in ifconfig.interface_chain:
                 adaptee = import_item(i)(adaptee)
-            d.addCallback(self.register, adaptee, furl_file=ifconfig.furl_file)
+            d.addCallback(self.register, adaptee, furl_file=ff)
 
     def register(self, empty, ref, furl_file):
         """Register the reference with the FURL file.

IPython/kernel/scripts/ipcontrollerapp.py

@@ -16,6 +16,7 b' The IPython controller application'
 #-----------------------------------------------------------------------------
 
 import copy
+import logging
 import os
 import sys
 
@@ -40,7 +41,7 b' from IPython.kernel.configobjfactory import ('
 from IPython.kernel.fcutil import FCServiceFactory
 
 #-----------------------------------------------------------------------------
-# Components for creating services
+# Default interfaces
 #-----------------------------------------------------------------------------
 
 
@@ -50,11 +51,13 b' default_client_interfaces.Task.interface_chain = ['
     'IPython.kernel.task.ITaskController',
     'IPython.kernel.taskfc.IFCTaskController'
 ]
+
 default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
 default_client_interfaces.MultiEngine.interface_chain = [
     'IPython.kernel.multiengine.IMultiEngine',
     'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
 ]
+
 default_client_interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl'
 
 # Make this a dict we can pass to Config.__init__ for the default
@@ -67,12 +70,17 b' default_engine_interfaces = Config()'
 default_engine_interfaces.Default.interface_chain = [
     'IPython.kernel.enginefc.IFCControllerBase'
 ]
+
 default_engine_interfaces.Default.furl_file = 'ipcontroller-engine.furl'
 
 # Make this a dict we can pass to Config.__init__ for the default
 default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
 
 
+#-----------------------------------------------------------------------------
+# Service factories
+#-----------------------------------------------------------------------------
+
 
 class FCClientServiceFactory(FCServiceFactory):
     """A Foolscap implementation of the client services."""
@@ -86,7 +94,7 b' class FCEngineServiceFactory(FCServiceFactory):'
     """A Foolscap implementation of the engine services."""
 
     cert_file = Str('ipcontroller-engine.pem', config=True)
-    interfaces = Instance(klass=dict, kw=default_engine_interfaces,
+    Interfaces = Instance(klass=dict, kw=default_engine_interfaces,
                           allow_none=False, config=True)
 
 
@@ -116,21 +124,6 b' cl_args = ('
         action='store_false', dest='FCClientServiceFactory.secure', default=NoConfigDefault,
         help='Turn off all client security.')
     ),
-    (('--client-cert-file',), dict(
-        type=str, dest='FCClientServiceFactory.cert_file', default=NoConfigDefault,
-        help='File to store the client SSL certificate in.',
-        metavar='FCClientServiceFactory.cert_file')
-    ),
-    (('--task-furl-file',), dict(
-        type=str, dest='FCClientServiceFactory.Interfaces.Task.furl_file', default=NoConfigDefault,
-        help='File to store the FURL in for task clients to connect with.',
-        metavar='FCClientServiceFactory.Interfaces.Task.furl_file')
-    ),
-    (('--multiengine-furl-file',), dict(
-        type=str, dest='FCClientServiceFactory.Interfaces.MultiEngine.furl_file', default=NoConfigDefault,
-        help='File to store the FURL in for multiengine clients to connect with.',
-        metavar='FCClientServiceFactory.Interfaces.MultiEngine.furl_file')
-    ),
     # Engine config
     (('--engine-ip',), dict(
         type=str, dest='FCEngineServiceFactory.ip', default=NoConfigDefault,
@@ -151,26 +144,20 b' cl_args = ('
         action='store_false', dest='FCEngineServiceFactory.secure', default=NoConfigDefault,
         help='Turn off all engine security.')
     ),
-    (('--engine-cert-file',), dict(
-        type=str, dest='FCEngineServiceFactory.cert_file', default=NoConfigDefault,
-        help='File to store the client SSL certificate in.',
-        metavar='FCEngineServiceFactory.cert_file')
-    ),
-    (('--engine-furl-file',), dict(
-        type=str, dest='FCEngineServiceFactory.Interfaces.Default.furl_file', default=NoConfigDefault,
-        help='File to store the FURL in for engines to connect with.',
-        metavar='FCEngineServiceFactory.Interfaces.Default.furl_file')
-    ),
     # Global config
-    (('-l','--logfile'), dict(
-        type=str, dest='Global.logfile', default=NoConfigDefault,
-        help='Log file name (default is stdout)',
-        metavar='Global.logfile')
+    (('--log-to-file',), dict(
+        action='store_true', dest='Global.log_to_file', default=NoConfigDefault,
+        help='Log to a file in the log directory (default is stdout)')
     ),
-    (('-r',), dict(
+    (('-r','--reuse-furls'), dict(
         action='store_true', dest='Global.reuse_furls', default=NoConfigDefault,
         help='Try to reuse all FURL files.')
-    )
+    ),
+    (('-cluster_dir', '--cluster-dir',), dict(
+        type=str, dest='Global.cluster_dir', default=NoConfigDefault,
+        help='Absolute or relative path to the cluster directory.',
+        metavar='Global.cluster_dir')
+    ),
 )
 
 
@@ -185,12 +172,18 b' class IPControllerApp(Application):'
 
     name = 'ipcontroller'
     config_file_name = _default_config_file_name
+    default_log_level = logging.DEBUG
 
     def create_default_config(self):
         super(IPControllerApp, self).create_default_config()
-        self.default_config.Global.logfile = ''
         self.default_config.Global.reuse_furls = False
         self.default_config.Global.import_statements = []
+        self.default_config.Global.profile = 'default'
+        self.default_config.Global.log_dir_name = 'log'
+        self.default_config.Global.security_dir_name = 'security'
+        self.default_config.Global.log_to_file = False
+        # Resolve the default cluster_dir using the default profile
+        self.default_config.Global.cluster_dir = ''
 
     def create_command_line_config(self):
         """Create and return a command line config loader."""
@@ -199,6 +192,75 b' class IPControllerApp(Application):'
             description="Start an IPython controller",
             version=release.version)
 
+    def find_config_file_name(self):
+        """Find the config file name for this application."""
+        self.find_cluster_dir()
+        self.create_cluster_dir()
+
+    def find_cluster_dir(self):
+        """This resolves into full paths, the various cluster directories.
+
+        This method must set ``self.cluster_dir`` to the full paths of 
+        the directory.
+        """
+        # Ignore self.command_line_config.Global.config_file
+        # Instead, first look for an explicit cluster_dir
+        try:
+            self.cluster_dir = self.command_line_config.Global.cluster_dir
+        except AttributeError:
+            self.cluster_dir = self.default_config.Global.cluster_dir
+        self.cluster_dir = os.path.expandvars(os.path.expanduser(self.cluster_dir))
+        if not self.cluster_dir:
+            # Then look for a profile
+            try:
+                self.profile = self.command_line_config.Global.profile
+            except AttributeError:
+                self.profile = self.default_config.Global.profile
+            cluster_dir_name = 'cluster_' + self.profile
+            try_this = os.path.join(os.getcwd(), cluster_dir_name)
+            if os.path.isdir(try_this):
+                self.cluster_dir = try_this
+            else:
+                self.cluster_dir = os.path.join(self.ipythondir, cluster_dir_name)
+        # These have to be set because they could be different from the one
+        # that we just computed.  Because command line has the highest
+        # priority, this will always end up in the master_config.
+        self.default_config.Global.cluster_dir = self.cluster_dir
+        self.command_line_config.Global.cluster_dir = self.cluster_dir
+
+    def create_cluster_dir(self):
+        """Make sure that the cluster, security and log dirs exist."""
+        if not os.path.isdir(self.cluster_dir):
+            os.makedirs(self.cluster_dir, mode=0777)
+
+    def find_config_file_paths(self):
+        """Set the search paths for resolving the config file."""
+        self.config_file_paths = (self.cluster_dir,)
+
+    def pre_construct(self):
+        # Now set the security_dir and log_dir and create them.  We use
+        # the names an construct the absolute paths.
+        security_dir = os.path.join(self.master_config.Global.cluster_dir,
+                                    self.master_config.Global.security_dir_name)
+        log_dir = os.path.join(self.master_config.Global.cluster_dir, 
+                               self.master_config.Global.log_dir_name)
+        if not os.path.isdir(security_dir):
+            os.mkdir(security_dir, 0700)
+        else:
+            os.chmod(security_dir, 0700)
+        if not os.path.isdir(log_dir):
+            os.mkdir(log_dir, 0777)
+
+        self.security_dir = self.master_config.Global.security_dir = security_dir
+        self.log_dir = self.master_config.Global.log_dir = log_dir
+
+        # Now setup reuse_furls
+        if hasattr(self.master_config.Global.reuse_furls):
+            self.master_config.FCClientServiceFactory.reuse_furls = \
+                self.master_config.Global.reuse_furls
+            self.master_config.FCEngineServiceFactory.reuse_furls = \
+                self.master_config.Global.reuse_furls
+
     def construct(self):
         # I am a little hesitant to put these into InteractiveShell itself.
         # But that might be the place for them
@@ -206,7 +268,6 b' class IPControllerApp(Application):'
 
         self.start_logging()
         self.import_statements()
-        self.reuse_furls()
 
         # Create the service hierarchy
         self.main_service = service.MultiService()
@@ -223,16 +284,13 b' class IPControllerApp(Application):'
         engine_service.setServiceParent(self.main_service)
 
     def start_logging(self):
-        logfile = self.master_config.Global.logfile
-        if logfile:
-            logfile = logfile + str(os.getpid()) + '.log'
-            try:
-                openLogFile = open(logfile, 'w')
-            except:
-                openLogFile = sys.stdout
+        if self.master_config.Global.log_to_file:
+            log_filename = self.name + '-' + str(os.getpid()) + '.log'
+            logfile = os.path.join(self.log_dir, log_filename)
+            open_log_file = open(logfile, 'w')
         else:
-            openLogFile = sys.stdout
-        log.startLogging(openLogFile)
+            open_log_file = sys.stdout
+        log.startLogging(open_log_file)
 
     def import_statements(self):
         statements = self.master_config.Global.import_statements
@@ -242,20 +300,6 b' class IPControllerApp(Application):'
             except:
                 log.msg("Error running import statement: %s" % s)
 
-    def reuse_furls(self):
-        # This logic might need to be moved into the components
-        # Delete old furl files unless the reuse_furls is set
-        reuse = self.master_config.Global.reuse_furls
-        # if not reuse:
-        #     paths = (
-        #         self.master_config.FCEngineServiceFactory.Interfaces.Default.furl_file,
-        #         self.master_config.FCClientServiceFactory.Interfaces.Task.furl_file,
-        #         self.master_config.FCClientServiceFactory.Interfaces.MultiEngine.furl_file
-        #     )
-        #     for p in paths:
-        #         if os.path.isfile(p):
-        #             os.remove(p)
-
     def start_app(self):
         # Start the controller service and set things running
         self.main_service.startService()