Finished refactoring ipcontroller to be a proper application.

* The :command:`ipcontroller` script has been refactored into a proper IPython
  application that uses the new config system.  This also introduces the
  idea of an application directory, which for :command:`ipcontroller` is
  :file:`cluster_<profile>` by default.  This cluster directory has the 
  config file, the log directory and the security directory for the
  controller and engine.

IPython/config/default/ipcontroller_config.py

@@ -6,62 +6,67 b' c = get_config()'
 # Global configuration
 #-----------------------------------------------------------------------------
 
-c.Global.log_to_file = False
-c.Global.import_statements = []
-c.Global.reuse_furls = False
+# Basic Global config attributes
+# c.Global.log_to_file = False
+# c.Global.import_statements = ['import math']
+# c.Global.reuse_furls = True
+# c.Global.secure = True
 
-# You shouldn't have to edit these
-c.Global.log_dir_name = 'log'
-c.Global.security_dir_name = 'security'
+# You shouldn't have to modify these
+# c.Global.log_dir_name = 'log'
+# c.Global.security_dir_name = 'security'
 
 
 #-----------------------------------------------------------------------------
 # Configure the client services
 #-----------------------------------------------------------------------------
 
-c.FCClientServiceFactory.ip = ''
-c.FCClientServiceFactory.port = 0 
-c.FCClientServiceFactory.location = ''
-c.FCClientServiceFactory.secure =  True
-c.FCClientServiceFactory.reuse_furls = False
-c.FCClientServiceFactory.cert_file = 'ipcontroller-client.pem'
-
-c.FCClientServiceFactory.Interfaces.Task.interface_chain = [
-    'IPython.kernel.task.ITaskController',
-    'IPython.kernel.taskfc.IFCTaskController'
-]
-# This is just the filename of the furl file.  The path is always the
-# security dir of the cluster directory.
-c.FCClientServiceFactory.Interfaces.Task.furl_file = 'ipcontroller-tc.furl'
-
-c.FCClientServiceFactory.Interfaces.MultiEngine.interface_chain = [
-    'IPython.kernel.multiengine.IMultiEngine',
-    'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
-]
-# This is just the filename of the furl file.  The path is always the
-# security dir of the cluster directory.
-c.FCClientServiceFactory.Interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl'
-
+# Basic client service config attributes
+# c.FCClientServiceFactory.ip = ''
+# c.FCClientServiceFactory.port = 0
+# c.FCClientServiceFactory.location = ''
+# c.FCClientServiceFactory.secure =  True
+# c.FCClientServiceFactory.reuse_furls = False
+
+# You shouldn't have to modify the rest of this section
+# c.FCClientServiceFactory.cert_file = 'ipcontroller-client.pem'
+
+# default_client_interfaces = Config()
+# default_client_interfaces.Task.interface_chain = [
+#     'IPython.kernel.task.ITaskController',
+#     'IPython.kernel.taskfc.IFCTaskController'
+# ]
+# 
+# default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
+# 
+# default_client_interfaces.MultiEngine.interface_chain = [
+#     'IPython.kernel.multiengine.IMultiEngine',
+#     'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
+# ]
+# 
+# default_client_interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl'
+# 
+# c.FCEngineServiceFactory.interfaces = default_client_interfaces
 
 #-----------------------------------------------------------------------------
 # Configure the engine services
 #-----------------------------------------------------------------------------
 
-c.FCEngineServiceFactory.ip = ''
-c.FCEngineServiceFactory.port = 0
-c.FCEngineServiceFactory.location = ''
-c.FCEngineServiceFactory.secure = True
-c.FCEngineServiceFactory.reuse_furls = False
-c.FCEngineServiceFactory.cert_file = 'ipcontroller-engine.pem'
-
-c.FCEngineServiceFactory.Intefaces.Default.interface_chain = [
-    'IPython.kernel.enginefc.IFCControllerBase'
-]
-
-# This is just the filename of the furl file.  The path is always the
-# security dir of the cluster directory.
-c.FCEngineServiceFactory.Intefaces.Default.furl_file = 'ipcontroller-engine.furl'
-
-
-
-
+# Basic config attributes for the engine services
+# c.FCEngineServiceFactory.ip = ''
+# c.FCEngineServiceFactory.port = 0
+# c.FCEngineServiceFactory.location = ''
+# c.FCEngineServiceFactory.secure = True
+# c.FCEngineServiceFactory.reuse_furls = False
+
+# You shouldn't have to modify the rest of this section
+# c.FCEngineServiceFactory.cert_file = 'ipcontroller-engine.pem'
+
+# default_engine_interfaces = Config()
+# default_engine_interfaces.Default.interface_chain = [
+#     'IPython.kernel.enginefc.IFCControllerBase'
+# ]
+# 
+# default_engine_interfaces.Default.furl_file = 'ipcontroller-engine.furl'
+# 
+# c.FCEngineServiceFactory.interfaces = default_engine_interfaces

IPython/core/application.py

@@ -77,9 +77,9 b' class Application(object):'
     """Load a config, construct an app and run it.
     """
 
-    config_file_name = 'ipython_config.py'
     name = 'ipython'
     description = 'IPython: an enhanced interactive Python shell.'
+    config_file_name = 'ipython_config.py'
     default_log_level = logging.WARN
     
 
@@ -351,7 +351,11 b' class AppWithDirArgParseConfigLoader(ArgParseConfigLoader):'
             metavar='Global.ipythondir')
         self.parser.add_argument('-p','-profile', '--profile',
             dest='Global.profile',type=str,
-            help='The string name of the ipython profile to be used.',
+            help='The string name of the profile to be used. This determines '
+            'the name of the application dir: basename_<profile>.  The basename is '
+            'determined by the particular application.  The default profile '
+            'is named "default".  This convention is used if the -app_dir '
+            'option is not used.',
             default=NoConfigDefault,
             metavar='Global.profile')
         self.parser.add_argument('-log_level', '--log-level',
@@ -360,22 +364,40 b' class AppWithDirArgParseConfigLoader(ArgParseConfigLoader):'
             default=NoConfigDefault)
         self.parser.add_argument('-app_dir', '--app-dir',
             dest='Global.app_dir',type=str,
-            help='Set the application directory where everything for this '
-            'application will be found (including the config file).',
+            help='Set the application dir where everything for this '
+            'application will be found (including the config file). This '
+            'overrides the logic used by the profile option.',
             default=NoConfigDefault,
             metavar='Global.app_dir')
 
 
 class ApplicationWithDir(Application):
+    """An application that puts everything into a application directory.
 
-    name = 'appname'
-    description = 'Application: foo and bar it.'
-    config_file_name = 'appname_config.py' 
-    default_log_level = logging.WARN
+    Instead of looking for things in the ipythondir, this type of application
+    will use its own private directory called the "application directory"
+    for things like config files, log files, etc.
+
+    The application directory is resolved as follows:
+
+    * If the ``--app-dir`` option is given, it is used.
+    * If ``--app-dir`` is not given, the application directory is resolve using
+      ``app_dir_basename`` and ``profile`` as ``<app_dir_basename>_<profile>``.
+      The search path for this directory is then i) cwd if it is found there
+      and ii) in ipythondir otherwise.
+
+    The config file for the application is to be put in the application
+    dir and named the value of the ``config_file_name`` class attribute.
+    """
+
+    # The basename used for the application dir: <app_dir_basename>_<profile>
+    app_dir_basename = 'cluster'
 
     def create_default_config(self):
         super(ApplicationWithDir, self).create_default_config()
         self.default_config.Global.profile = 'default'
+        # The application dir.  This is empty initially so the default is to
+        # try to resolve this using the profile.
         self.default_config.Global.app_dir = ''
 
     def create_command_line_config(self):
@@ -391,10 +413,10 b' class ApplicationWithDir(Application):'
         self.create_app_dir()
 
     def find_app_dir(self):
-        """This resolves into full paths, the app directory.
+        """This resolves the app directory.
 
-        This method must set ``self.app_dir`` to the full path of
-        the directory.
+        This method must set ``self.app_dir`` to the location of the app
+        dir.
         """
         # Instead, first look for an explicit app_dir
         try:
@@ -408,7 +430,7 b' class ApplicationWithDir(Application):'
                 self.profile = self.command_line_config.Global.profile
             except AttributeError:
                 self.profile = self.default_config.Global.profile
-            app_dir_name = 'cluster_' + self.profile
+            app_dir_name = self.app_dir_basename + '_' + self.profile
             try_this = os.path.join(os.getcwd(), app_dir_name)
             if os.path.isdir(try_this):
                 self.app_dir = try_this
@@ -421,7 +443,7 b' class ApplicationWithDir(Application):'
         self.command_line_config.Global.app_dir = self.app_dir
 
     def create_app_dir(self):
-        """Make sure that the cluster, security and log dirs exist."""
+        """Make sure that the app dir exists."""
         if not os.path.isdir(self.app_dir):
             os.makedirs(self.app_dir, mode=0777)
 

IPython/core/component.py

@@ -296,7 +296,19 b' class Component(HasTraitlets):'
             if new._has_section(sname):
                 my_config = new[sname]
                 for k, v in traitlets.items():
+                    # Don't allow traitlets with config=True to start with
+                    # uppercase.  Otherwise, they are confused with Config
+                    # subsections.  But, developers shouldn't have uppercase
+                    # attributes anyways! (PEP 6)
+                    if k[0].upper()==k[0] and not k.startswith('_'):
+                        raise ComponentError('Component traitlets with '
+                        'config=True must start with a lowercase so they are '
+                        'not confused with Config subsections: %s.%s' % \
+                        (self.__class__.__name__, k))
                     try:
+                        # Here we grab the value from the config
+                        # If k has the naming convention of a config
+                        # section, it will be auto created.
                         config_value = my_config[k]
                     except KeyError:
                         pass

IPython/core/ipapp.py

@@ -294,13 +294,13 b' class IPythonAppCLConfigLoader(BaseAppArgParseConfigLoader):'
     arguments = cl_args
 
 
-_default_config_file_name = 'ipython_config.py'
+default_config_file_name = 'ipython_config.py'
 
 
 class IPythonApp(Application):
     name = 'ipython'
     description = 'IPython: an enhanced interactive Python shell.'
-    config_file_name = _default_config_file_name
+    config_file_name = default_config_file_name
 
     def create_default_config(self):
         super(IPythonApp, self).create_default_config()
@@ -533,7 +533,7 b' def load_default_config(ipythondir=None):'
     """
     if ipythondir is None:
         ipythondir = get_ipython_dir()
-    cl = PyFileConfigLoader(_default_config_file_name, ipythondir)
+    cl = PyFileConfigLoader(default_config_file_name, ipythondir)
     config = cl.load_config()
     return config
 

IPython/kernel/configobjfactory.py

@@ -68,7 +68,11 b' class AdaptedConfiguredObjectFactory(Component):'
     # zi.implements(IAdaptedConfiguredObjectFactory)
 
     def __init__(self, config, adaptee):
+        # print
+        # print "config pre:", config
         super(AdaptedConfiguredObjectFactory, self).__init__(None, config=config)
+        # print
+        # print "config post:", config
         self.adaptee = adaptee
 
     def create(self):

IPython/kernel/fcutil.py

@@ -67,11 +67,11 b' def is_secure(furl):'
         elif furl.startswith("pbu://"):
             return False
     else:
-        raise ValueError("invalid furl: %s" % furl)
+        raise ValueError("invalid FURL: %s" % furl)
 
 
 def is_valid(furl):
-    """Is the str a valid furl or not."""
+    """Is the str a valid FURL or not."""
     if isinstance(furl, str):
         if furl.startswith("pb://") or furl.startswith("pbu://"):
             return True
@@ -88,11 +88,11 b' def find_furl(furl_or_file):'
         furl = open(furl_or_file, 'r').read().strip()
         if is_valid(furl):
             return furl
-    raise ValueError("not a furl or a file containing a furl: %s" % furl_or_file)
+    raise ValueError("not a FURL or a file containing a FURL: %s" % furl_or_file)
 
 
 def get_temp_furlfile(filename):
-    """Return a temporary furl file."""
+    """Return a temporary FURL file."""
     return tempfile.mktemp(dir=os.path.dirname(filename),
                            prefix=os.path.basename(filename))
 
@@ -132,6 +132,7 b' def make_tub(ip, port, secure, cert_file):'
         strport = "tcp:%i" % port
     else:
         strport = "tcp:%i:interface=%s" % (port, ip)
+    log.msg("Starting listener with [secure=%r] on: %s" % (secure, strport))
     listener = tub.listenOn(strport)
     
     return tub, listener
@@ -147,7 +148,7 b' class FCServiceFactory(AdaptedConfiguredObjectFactory):'
 
     Attributes
     ----------
-    Interfaces : Config
+    interfaces : Config
         A Config instance whose values are sub-Config objects having two
         keys: furl_file and interface_chain.
 
@@ -160,7 +161,7 b' class FCServiceFactory(AdaptedConfiguredObjectFactory):'
     cert_file = Str('', config=True)
     location = Str('', config=True)
     reuse_furls = Bool(False, config=True)
-    Interfaces = Instance(klass=Config, kw={}, allow_none=False, config=True)
+    interfaces = Instance(klass=Config, kw={}, allow_none=False, config=True)
 
     def __init__(self, config, adaptee):
         super(FCServiceFactory, self).__init__(config, adaptee)
@@ -171,11 +172,14 b' class FCServiceFactory(AdaptedConfiguredObjectFactory):'
             self.location = '127.0.0.1'
 
     def _check_reuse_furls(self):
-        if not self.reuse_furls:
-            furl_files = [i.furl_file for i in self.Interfaces.values()]
-            for ff in furl_files:
-                fullfile = self._get_security_file(ff)
+        furl_files = [i.furl_file for i in self.interfaces.values()]
+        for ff in furl_files:
+            fullfile = self._get_security_file(ff)
+            if self.reuse_furls:
+                log.msg("Reusing FURL file: %s" % fullfile)
+            else:
                 if os.path.isfile(fullfile):
+                    log.msg("Removing old FURL file: %s" % fullfile)
                     os.remove(fullfile)
 
     def _get_security_file(self, filename):
@@ -185,11 +189,14 b' class FCServiceFactory(AdaptedConfiguredObjectFactory):'
         """Create and return the Foolscap tub with everything running."""
 
         self.tub, self.listener = make_tub(
-            self.ip, self.port, self.secure, self._get_security_file(self.cert_file))
-        log.msg("Created a tub and listener [%r]: %r, %r" % (self.__class__, self.tub, self.listener))
-        log.msg("Interfaces to register [%r]: %r" % (self.__class__, self.Interfaces))
+            self.ip, self.port, self.secure, 
+            self._get_security_file(self.cert_file)
+        )
+        # log.msg("Interfaces to register [%r]: %r" % \
+        #     (self.__class__, self.interfaces))
         if not self.secure:
-            log.msg("WARNING: running with no security: %s" % self.__class__.__name__)
+            log.msg("WARNING: running with no security: %s" % \
+                self.__class__.__name__)
         reactor.callWhenRunning(self.set_location_and_register)
         return self.tub
 
@@ -206,10 +213,11 b' class FCServiceFactory(AdaptedConfiguredObjectFactory):'
     def adapt_to_interfaces(self, d):
         """Run through the interfaces, adapt and register."""
 
-        for ifname, ifconfig in self.Interfaces.iteritems():
+        for ifname, ifconfig in self.interfaces.iteritems():
             ff = self._get_security_file(ifconfig.furl_file)
-            log.msg("Adapting %r to interface: %s" % (self.adaptee, ifname))
-            log.msg("Saving furl for interface [%s] to file: %s" % (ifname, ff))
+            log.msg("Adapting [%s] to interface: %s" % \
+                (self.adaptee.__class__.__name__, ifname))
+            log.msg("Saving FURL for interface [%s] to file: %s" % (ifname, ff))
             check_furl_file_security(ff, self.secure)
             adaptee = self.adaptee
             for i in ifconfig.interface_chain:

IPython/kernel/ipcontrollerapp.py

@@ -28,7 +28,7 b' from IPython.config.loader import Config, NoConfigDefault'
 
 from IPython.core.application import (
     ApplicationWithDir, 
-    BaseAppArgParseConfigLoader
+    AppWithDirArgParseConfigLoader
 )
 
 from IPython.core import release
@@ -49,7 +49,7 b' from IPython.kernel.fcutil import FCServiceFactory'
 #-----------------------------------------------------------------------------
 
 
-# The default client interfaces for FCClientServiceFactory.Interfaces
+# The default client interfaces for FCClientServiceFactory.interfaces
 default_client_interfaces = Config()
 default_client_interfaces.Task.interface_chain = [
     'IPython.kernel.task.ITaskController',
@@ -57,6 +57,7 b' default_client_interfaces.Task.interface_chain = ['
 ]
 
 default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
+
 default_client_interfaces.MultiEngine.interface_chain = [
     'IPython.kernel.multiengine.IMultiEngine',
     'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
@@ -69,7 +70,7 b' default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()'
 
 
 
-# The default engine interfaces for FCEngineServiceFactory.Interfaces
+# The default engine interfaces for FCEngineServiceFactory.interfaces
 default_engine_interfaces = Config()
 default_engine_interfaces.Default.interface_chain = [
     'IPython.kernel.enginefc.IFCControllerBase'
@@ -90,7 +91,7 b' class FCClientServiceFactory(FCServiceFactory):'
     """A Foolscap implementation of the client services."""
 
     cert_file = Str('ipcontroller-client.pem', config=True)
-    Interfaces = Instance(klass=Config, kw=default_client_interfaces,
+    interfaces = Instance(klass=Config, kw=default_client_interfaces,
                           allow_none=False, config=True)
 
 
@@ -98,7 +99,7 b' class FCEngineServiceFactory(FCServiceFactory):'
     """A Foolscap implementation of the engine services."""
 
     cert_file = Str('ipcontroller-engine.pem', config=True)
-    Interfaces = Instance(klass=dict, kw=default_engine_interfaces,
+    interfaces = Instance(klass=dict, kw=default_engine_interfaces,
                           allow_none=False, config=True)
 
 
@@ -124,10 +125,6 b' cl_args = ('
         help='The hostname or ip that clients should connect to.',
         metavar='FCClientServiceFactory.location')
     ),
-    (('-x',), dict(
-        action='store_false', dest='FCClientServiceFactory.secure', default=NoConfigDefault,
-        help='Turn off all client security.')
-    ),
     # Engine config
     (('--engine-ip',), dict(
         type=str, dest='FCEngineServiceFactory.ip', default=NoConfigDefault,
@@ -144,10 +141,6 b' cl_args = ('
         help='The hostname or ip that engines should connect to.',
         metavar='FCEngineServiceFactory.location')
     ),
-    (('-y',), dict(
-        action='store_false', dest='FCEngineServiceFactory.secure', default=NoConfigDefault,
-        help='Turn off all engine security.')
-    ),
     # Global config
     (('--log-to-file',), dict(
         action='store_true', dest='Global.log_to_file', default=NoConfigDefault,
@@ -156,33 +149,61 b' cl_args = ('
     (('-r','--reuse-furls'), dict(
         action='store_true', dest='Global.reuse_furls', default=NoConfigDefault,
         help='Try to reuse all FURL files.')
+    ),
+    (('-ns','--no-security'), dict(
+        action='store_false', dest='Global.secure', default=NoConfigDefault,
+        help='Turn off SSL encryption for all connections.')
     )
 )
 
 
-class IPControllerAppCLConfigLoader(BaseAppArgParseConfigLoader):
+class IPControllerAppCLConfigLoader(AppWithDirArgParseConfigLoader):
 
     arguments = cl_args
 
 
-_default_config_file_name = 'ipcontroller_config.py'
+default_config_file_name = 'ipcontroller_config.py'
 
 
 class IPControllerApp(ApplicationWithDir):
 
     name = 'ipcontroller'
+    app_dir_basename = 'cluster'
     description = 'Start the IPython controller for parallel computing.'
-    config_file_name = _default_config_file_name
-    default_log_level = logging.DEBUG
+    config_file_name = default_config_file_name
+    default_log_level = logging.WARN
 
     def create_default_config(self):
         super(IPControllerApp, self).create_default_config()
         self.default_config.Global.reuse_furls = False
+        self.default_config.Global.secure = True
         self.default_config.Global.import_statements = []
         self.default_config.Global.log_dir_name = 'log'
         self.default_config.Global.security_dir_name = 'security'
         self.default_config.Global.log_to_file = False
 
+    def create_command_line_config(self):
+        """Create and return a command line config loader."""
+        return IPControllerAppCLConfigLoader(
+            description=self.description, 
+            version=release.version
+        )
+
+    def post_load_command_line_config(self):
+        # Now setup reuse_furls
+        if hasattr(self.command_line_config.Global, 'reuse_furls'):
+            self.command_line_config.FCClientServiceFactory.reuse_furls = \
+                self.command_line_config.Global.reuse_furls
+            self.command_line_config.FCEngineServiceFactory.reuse_furls = \
+                self.command_line_config.Global.reuse_furls
+            del self.command_line_config.Global.reuse_furls
+        if hasattr(self.command_line_config.Global, 'secure'):
+            self.command_line_config.FCClientServiceFactory.secure = \
+                self.command_line_config.Global.secure
+            self.command_line_config.FCEngineServiceFactory.secure = \
+                self.command_line_config.Global.secure
+            del self.command_line_config.Global.secure
+
     def pre_construct(self):
         # Now set the security_dir and log_dir and create them.  We use
         # the names an construct the absolute paths.
@@ -200,13 +221,6 b' class IPControllerApp(ApplicationWithDir):'
         self.security_dir = self.master_config.Global.security_dir = security_dir
         self.log_dir = self.master_config.Global.log_dir = log_dir
 
-        # Now setup reuse_furls
-        if hasattr(self.master_config.Global, 'reuse_furls'):
-            self.master_config.FCClientServiceFactory.reuse_furls = \
-                self.master_config.Global.reuse_furls
-            self.master_config.FCEngineServiceFactory.reuse_furls = \
-                self.master_config.Global.reuse_furls
-
     def construct(self):
         # I am a little hesitant to put these into InteractiveShell itself.
         # But that might be the place for them
@@ -214,7 +228,7 b' class IPControllerApp(ApplicationWithDir):'
 
         self.start_logging()
         self.import_statements()
-
+        
         # Create the service hierarchy
         self.main_service = service.MultiService()
         # The controller service
@@ -242,9 +256,10 b' class IPControllerApp(ApplicationWithDir):'
         statements = self.master_config.Global.import_statements
         for s in statements:
             try:
+                log.msg("Executing statement: '%s'" % s)
                 exec s in globals(), locals()
             except:
-                log.msg("Error running import statement: %s" % s)
+                log.msg("Error running statement: %s" % s)
 
     def start_app(self):
         # Start the controller service and set things running