General work on the kernel config.

IPython/config/default/ipcontroller_config.py

@@ -1,60 +1,67 b''
 from IPython.config.loader import Config
 
 c = get_config()
 
 #-----------------------------------------------------------------------------
 # Global configuration
 #-----------------------------------------------------------------------------
 
-c.Global.logfile = ''
-c.Global.import_statement = ''
+c.Global.log_to_file = False
+c.Global.import_statements = []
 c.Global.reuse_furls = False
 
+# You shouldn't have to edit these
+c.Global.log_dir_name = 'log'
+c.Global.security_dir_name = 'security'
+
 
 #-----------------------------------------------------------------------------
 # Configure the client services
 #-----------------------------------------------------------------------------
 
 c.FCClientServiceFactory.ip = ''
 c.FCClientServiceFactory.port = 0 
 c.FCClientServiceFactory.location = ''
 c.FCClientServiceFactory.secure =  True
+c.FCClientServiceFactory.reuse_furls = False
 c.FCClientServiceFactory.cert_file = 'ipcontroller-client.pem'
 
 c.FCClientServiceFactory.Interfaces.Task.interface_chain = [
     'IPython.kernel.task.ITaskController',
     'IPython.kernel.taskfc.IFCTaskController'
 ]
+# This is just the filename of the furl file.  The path is always the
+# security dir of the cluster directory.
 c.FCClientServiceFactory.Interfaces.Task.furl_file = 'ipcontroller-tc.furl'
 
 c.FCClientServiceFactory.Interfaces.MultiEngine.interface_chain = [
     'IPython.kernel.multiengine.IMultiEngine',
     'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
 ]
+# This is just the filename of the furl file.  The path is always the
+# security dir of the cluster directory.
 c.FCClientServiceFactory.Interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl'
 
 
 #-----------------------------------------------------------------------------
 # Configure the engine services
 #-----------------------------------------------------------------------------
 
 c.FCEngineServiceFactory.ip = ''
 c.FCEngineServiceFactory.port = 0
 c.FCEngineServiceFactory.location = ''
 c.FCEngineServiceFactory.secure = True
+c.FCEngineServiceFactory.reuse_furls = False
 c.FCEngineServiceFactory.cert_file = 'ipcontroller-engine.pem'
 
-engine_config = Config()
-engine_config.furl_file = 
-c.Global.engine_furl_file = 'ipcontroller-engine.furl'
-c.Global.engine_fc_interface = 'IPython.kernel.enginefc.IFCControllerBase'
-
+c.FCEngineServiceFactory.Intefaces.Default.interface_chain = [
+    'IPython.kernel.enginefc.IFCControllerBase'
+]
 
+# This is just the filename of the furl file.  The path is always the
+# security dir of the cluster directory.
+c.FCEngineServiceFactory.Intefaces.Default.furl_file = 'ipcontroller-engine.furl'
 
 
 
-CLIENT_INTERFACES = dict(
-    TASK = dict(FURL_FILE = 'ipcontroller-tc.furl'),
-    MULTIENGINE = dict(FURLFILE='ipcontroller-mec.furl')
-)
 

IPython/core/application.py

@@ -1,304 +1,334 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 An application for IPython
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 
 Notes
 -----
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import logging
 import os
 import sys
 import traceback
 from copy import deepcopy
 
 from IPython.utils.genutils import get_ipython_dir, filefind
 from IPython.config.loader import (
     PyFileConfigLoader,
     ArgParseConfigLoader,
     Config,
     NoConfigDefault
 )
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 
 class IPythonArgParseConfigLoader(ArgParseConfigLoader):
     """Default command line options for IPython based applications."""
 
     def _add_other_arguments(self):
-        self.parser.add_argument('-ipythondir', '--ipythondir', 
+        self.parser.add_argument('-ipythondir', '--ipython-dir', 
             dest='Global.ipythondir',type=str,
             help='Set to override default location of Global.ipythondir.',
             default=NoConfigDefault,
             metavar='Global.ipythondir')
         self.parser.add_argument('-p','-profile', '--profile',
             dest='Global.profile',type=str,
             help='The string name of the ipython profile to be used.',
             default=NoConfigDefault,
             metavar='Global.profile')
         self.parser.add_argument('-log_level', '--log-level',
             dest="Global.log_level",type=int,
             help='Set the log level (0,10,20,30,40,50).  Default is 30.',
             default=NoConfigDefault)
         self.parser.add_argument('-config_file', '--config-file',
             dest='Global.config_file',type=str,
             help='Set the config file name to override default.',
             default=NoConfigDefault,
             metavar='Global.config_file')
 
 
 class ApplicationError(Exception):
     pass
 
 
 class Application(object):
     """Load a config, construct an app and run it.
     """
 
     config_file_name = 'ipython_config.py'
     name = 'ipython'
+    default_log_level = logging.WARN
+    
 
     def __init__(self):
         self.init_logger()
         self.default_config_file_name = self.config_file_name
 
     def init_logger(self):
         self.log = logging.getLogger(self.__class__.__name__)
         # This is used as the default until the command line arguments are read.
-        self.log.setLevel(logging.WARN)
+        self.log.setLevel(self.default_log_level)
         self._log_handler = logging.StreamHandler()
         self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
         self._log_handler.setFormatter(self._log_formatter)
         self.log.addHandler(self._log_handler)
 
     def _set_log_level(self, level):
         self.log.setLevel(level)
 
     def _get_log_level(self):
         return self.log.level
 
     log_level = property(_get_log_level, _set_log_level)
 
     def start(self):
         """Start the application."""
         self.attempt(self.create_default_config)
+        self.log_default_config()
+        self.set_default_config_log_level()
         self.attempt(self.pre_load_command_line_config)
         self.attempt(self.load_command_line_config, action='abort')
+        self.set_command_line_config_log_level()
         self.attempt(self.post_load_command_line_config)
+        self.log_command_line_config()
         self.attempt(self.find_ipythondir)
         self.attempt(self.find_config_file_name)
         self.attempt(self.find_config_file_paths)
         self.attempt(self.pre_load_file_config)
         self.attempt(self.load_file_config)
+        self.set_file_config_log_level()
         self.attempt(self.post_load_file_config)
+        self.log_file_config()
         self.attempt(self.merge_configs)
+        self.log_master_config()
         self.attempt(self.pre_construct)
         self.attempt(self.construct)
         self.attempt(self.post_construct)
         self.attempt(self.start_app)
 
     #-------------------------------------------------------------------------
     # Various stages of Application creation
     #-------------------------------------------------------------------------
 
     def create_default_config(self):
         """Create defaults that can't be set elsewhere.
 
         For the most part, we try to set default in the class attributes
         of Components.  But, defaults the top-level Application (which is
         not a HasTraitlets or Component) are not set in this way.  Instead
         we set them here.  The Global section is for variables like this that
         don't belong to a particular component.
         """
         self.default_config = Config()
         self.default_config.Global.ipythondir = get_ipython_dir()
+
+    def log_default_config(self):
         self.log.debug('Default config loaded:')
         self.log.debug(repr(self.default_config))
 
+    def set_default_config_log_level(self):
+        try:
+            self.log_level = self.default_config.Global.log_level
+        except AttributeError:
+            # Fallback to the default_log_level class attribute
+            pass
+
     def create_command_line_config(self):
         """Create and return a command line config loader."""
         return IPythonArgParseConfigLoader(description=self.name)
 
     def pre_load_command_line_config(self):
         """Do actions just before loading the command line config."""
         pass
 
     def load_command_line_config(self):
-        """Load the command line config.
-
-        This method also sets ``self.debug``.
-        """
-
+        """Load the command line config."""
         loader = self.create_command_line_config()
         self.command_line_config = loader.load_config()
         self.extra_args = loader.get_extra_args()
 
+    def set_command_line_config_log_level(self):
         try:
             self.log_level = self.command_line_config.Global.log_level
         except AttributeError:
-            pass # Use existing value which is set in Application.init_logger.
-        self.log.debug("Command line config loaded:")
-        self.log.debug(repr(self.command_line_config))
+            pass
 
     def post_load_command_line_config(self):
         """Do actions just after loading the command line config."""
         pass
 
+    def log_command_line_config(self):
+        self.log.debug("Command line config loaded:")
+        self.log.debug(repr(self.command_line_config))
+
     def find_ipythondir(self):
         """Set the IPython directory.
 
         This sets ``self.ipythondir``, but the actual value that is passed
         to the application is kept in either ``self.default_config`` or
         ``self.command_line_config``.  This also added ``self.ipythondir`` to
         ``sys.path`` so config files there can be references by other config
         files.
         """
 
         try:
             self.ipythondir = self.command_line_config.Global.ipythondir
         except AttributeError:
             self.ipythondir = self.default_config.Global.ipythondir
         sys.path.append(os.path.abspath(self.ipythondir))
         if not os.path.isdir(self.ipythondir):
             os.makedirs(self.ipythondir, mode = 0777)
         self.log.debug("IPYTHONDIR set to: %s" % self.ipythondir)
 
     def find_config_file_name(self):
         """Find the config file name for this application.
 
+        This must set ``self.config_file_name`` to the filename of the
+        config file to use (just the filename). The search paths for the
+        config file are set in :meth:`find_config_file_paths` and then passed
+        to the config file loader where they are resolved to an absolute path.
+
         If a profile has been set at the command line, this will resolve
-        it.  The search paths for the config file are set in
-        :meth:`find_config_file_paths` and then passed to the config file
-        loader where they are resolved to an absolute path.
+        it.
         """
 
         try:
             self.config_file_name = self.command_line_config.Global.config_file
         except AttributeError:
             pass
 
         try:
             self.profile_name = self.command_line_config.Global.profile
             name_parts = self.config_file_name.split('.')
             name_parts.insert(1, '_' + self.profile_name + '.')
             self.config_file_name = ''.join(name_parts)
         except AttributeError:
             pass
 
     def find_config_file_paths(self):
-        """Set the search paths for resolving the config file."""
+        """Set the search paths for resolving the config file.
+
+        This must set ``self.config_file_paths`` to a sequence of search
+        paths to pass to the config file loader.
+        """
         self.config_file_paths = (os.getcwd(), self.ipythondir)
 
     def pre_load_file_config(self):
         """Do actions before the config file is loaded."""
         pass
 
     def load_file_config(self):
         """Load the config file.
         
         This tries to load the config file from disk.  If successful, the
         ``CONFIG_FILE`` config variable is set to the resolved config file
         location.  If not successful, an empty config is used.
         """
         self.log.debug("Attempting to load config file: <%s>" % self.config_file_name)
         loader = PyFileConfigLoader(self.config_file_name,
                                     path=self.config_file_paths)
         try:
             self.file_config = loader.load_config()
             self.file_config.Global.config_file = loader.full_filename
         except IOError:
             # Only warn if the default config file was NOT being used.
             if not self.config_file_name==self.default_config_file_name:
                 self.log.warn("Config file not found, skipping: <%s>" % \
                                self.config_file_name, exc_info=True)
             self.file_config = Config()
         except:
             self.log.warn("Error loading config file: <%s>" % \
                            self.config_file_name, exc_info=True)
             self.file_config = Config()
-        else:
-            self.log.debug("Config file loaded: <%s>" % loader.full_filename)
-            self.log.debug(repr(self.file_config))
+
+    def set_file_config_log_level(self):
         # We need to keeep self.log_level updated.  But we only use the value
         # of the file_config if a value was not specified at the command
-        # line.
+        # line, because the command line overrides everything.
         if not hasattr(self.command_line_config.Global, 'log_level'):
             try:
                 self.log_level = self.file_config.Global.log_level
             except AttributeError:
                 pass # Use existing value
 
     def post_load_file_config(self):
         """Do actions after the config file is loaded."""
         pass
 
+    def log_file_config(self):
+        if hasattr(self.file_config.Global, 'config_file'):
+            self.log.debug("Config file loaded: <%s>" % self.file_config.Global.config_file)
+            self.log.debug(repr(self.file_config))
+
     def merge_configs(self):
         """Merge the default, command line and file config objects."""
         config = Config()
         config._merge(self.default_config)
         config._merge(self.file_config)
         config._merge(self.command_line_config)
         self.master_config = config
+
+    def log_master_config(self):
         self.log.debug("Master config created:")
         self.log.debug(repr(self.master_config))
 
     def pre_construct(self):
         """Do actions after the config has been built, but before construct."""
         pass
 
     def construct(self):
         """Construct the main components that make up this app."""
         self.log.debug("Constructing components for application")
 
     def post_construct(self):
         """Do actions after construct, but before starting the app."""
         pass
 
     def start_app(self):
         """Actually start the app."""
         self.log.debug("Starting application")
 
     #-------------------------------------------------------------------------
     # Utility methods
     #-------------------------------------------------------------------------
 
     def abort(self):
         """Abort the starting of the application."""
         self.log.critical("Aborting application: %s" % self.name, exc_info=True)
         sys.exit(1)
 
     def exit(self):
         self.log.critical("Aborting application: %s" % self.name)
         sys.exit(1)
 
     def attempt(self, func, action='abort'):
         try:
             func()
         except SystemExit:
             self.exit()
         except:
             if action == 'abort':
                 self.abort()
             elif action == 'exit':
                 self.exit()
       
  No newline at end of file

IPython/kernel/client.py

@@ -1,96 +1,96 b''
 # encoding: utf-8
 
 """This module contains blocking clients for the controller interfaces.
 
 Unlike the clients in `asyncclient.py`, the clients in this module are fully
 blocking.  This means that methods on the clients return the actual results
 rather than a deferred to the result.  Also, we manage the Twisted reactor
 for you.  This is done by running the reactor in a thread.
 
 The main classes in this module are:
 
     * MultiEngineClient
     * TaskClient
     * Task
     * CompositeError
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import sys
 
 # from IPython.utils import growl
 # growl.start("IPython1 Client")
 
 
 from twisted.internet import reactor
 from IPython.kernel.clientconnector import ClientConnector
 from IPython.kernel.twistedutil import ReactorInThread
 from IPython.kernel.twistedutil import blockingCallFromThread
 
 # These enable various things 
 from IPython.kernel import codeutil
-import IPython.kernel.magic
+# import IPython.kernel.magic
 
 # Other things that the user will need
 from IPython.kernel.task import MapTask, StringTask
 from IPython.kernel.error import CompositeError
 
 #-------------------------------------------------------------------------------
 # Code
 #-------------------------------------------------------------------------------
 
 _client_tub = ClientConnector()
 
 
 def get_multiengine_client(furl_or_file=''):
     """Get the blocking MultiEngine client.
     
     :Parameters:
         furl_or_file : str
             A furl or a filename containing a furl.  If empty, the
             default furl_file will be used
             
     :Returns:
         The connected MultiEngineClient instance
     """
     client = blockingCallFromThread(_client_tub.get_multiengine_client, 
         furl_or_file)
     return client.adapt_to_blocking_client()
 
 def get_task_client(furl_or_file=''):
     """Get the blocking Task client.
     
     :Parameters:
         furl_or_file : str
             A furl or a filename containing a furl.  If empty, the
             default furl_file will be used
             
     :Returns:
         The connected TaskClient instance
     """
     client = blockingCallFromThread(_client_tub.get_task_client, 
         furl_or_file)
     return client.adapt_to_blocking_client()
 
 
 MultiEngineClient = get_multiengine_client
 TaskClient = get_task_client
 
 
 
 # Now we start the reactor in a thread
 rit = ReactorInThread()
 rit.setDaemon(True)
 rit.start()
  No newline at end of file

IPython/kernel/fcutil.py

@@ -1,209 +1,228 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 Foolscap related utilities.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import tempfile
 
 from twisted.internet import reactor, defer
 from twisted.python import log
 
 from foolscap import Tub, UnauthenticatedTub
 
 from IPython.config.loader import Config
 
 from IPython.kernel.configobjfactory import AdaptedConfiguredObjectFactory
 
 from IPython.kernel.error import SecurityError
 
 from IPython.utils.traitlets import Int, Str, Bool, Instance
 from IPython.utils.importstring import import_item
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 
 # We do this so if a user doesn't have OpenSSL installed, it will try to use
 # an UnauthenticatedTub.  But, they will still run into problems if they
 # try to use encrypted furls.
 try:
     import OpenSSL
 except:
     Tub = UnauthenticatedTub
     have_crypto = False
 else:
     have_crypto = True
 
 
 def check_furl_file_security(furl_file, secure):
     """Remove the old furl_file if changing security modes."""
     if os.path.isfile(furl_file):
         f = open(furl_file, 'r')
         oldfurl = f.read().strip()
         f.close()
         if (oldfurl.startswith('pb://') and not secure) or (oldfurl.startswith('pbu://') and secure):
             os.remove(furl_file)
 
 
 def is_secure(furl):
     """Is the given FURL secure or not."""
     if is_valid(furl):
         if furl.startswith("pb://"):
             return True
         elif furl.startswith("pbu://"):
             return False
     else:
         raise ValueError("invalid furl: %s" % furl)
 
 
 def is_valid(furl):
     """Is the str a valid furl or not."""
     if isinstance(furl, str):
         if furl.startswith("pb://") or furl.startswith("pbu://"):
             return True
     else:
         return False
 
 
 def find_furl(furl_or_file):
     """Find, validate and return a FURL in a string or file."""
     if isinstance(furl_or_file, str):
         if is_valid(furl_or_file):
             return furl_or_file
     if os.path.isfile(furl_or_file):
         furl = open(furl_or_file, 'r').read().strip()
         if is_valid(furl):
             return furl
     raise ValueError("not a furl or a file containing a furl: %s" % furl_or_file)
 
 
 def get_temp_furlfile(filename):
     """Return a temporary furl file."""
     return tempfile.mktemp(dir=os.path.dirname(filename),
                            prefix=os.path.basename(filename))
 
 
 def make_tub(ip, port, secure, cert_file):
     """Create a listening tub given an ip, port, and cert_file location.
     
     Parameters
     ----------
     ip : str
         The ip address or hostname that the tub should listen on.  
         Empty means all interfaces.
     port : int
         The port that the tub should listen on.  A value of 0 means
         pick a random port
     secure: bool
         Will the connection be secure (in the Foolscap sense).
     cert_file: str
         A filename of a file to be used for theSSL certificate.
 
     Returns
     -------
     A tub, listener tuple.
     """
     if secure:
         if have_crypto:
             tub = Tub(certFile=cert_file)
         else:
             raise SecurityError("OpenSSL/pyOpenSSL is not available, so we "
                                 "can't run in secure mode.  Try running without "
                                 "security using 'ipcontroller -xy'.")
     else:
         tub = UnauthenticatedTub()
     
     # Set the strport based on the ip and port and start listening
     if ip == '':
         strport = "tcp:%i" % port
     else:
         strport = "tcp:%i:interface=%s" % (port, ip)
     listener = tub.listenOn(strport)
     
     return tub, listener
 
 
 class FCServiceFactory(AdaptedConfiguredObjectFactory):
     """This class creates a tub with various services running in it.
 
     The basic idea is that :meth:`create` returns a running :class:`Tub`
     instance that has a number of Foolscap references registered in it.
     This class is a subclass of :class:`IPython.core.component.Component`
     so the IPython configuration and component system are used.
 
     Attributes
     ----------
     Interfaces : Config
         A Config instance whose values are sub-Config objects having two
         keys: furl_file and interface_chain.
 
     The other attributes are the standard ones for Foolscap.
 """
 
     ip = Str('', config=True)
     port = Int(0, config=True)
     secure = Bool(True, config=True)
     cert_file = Str('', config=True)
     location = Str('', config=True)
+    reuse_furls = Bool(False, config=True)
     Interfaces = Instance(klass=Config, kw={}, allow_none=False, config=True)
 
+    def __init__(self, config, adaptee):
+        super(FCServiceFactory, self).__init__(config, adaptee)
+        self._check_reuse_furls()
+
     def _ip_changed(self, name, old, new):
         if new == 'localhost' or new == '127.0.0.1':
             self.location = '127.0.0.1'
 
+    def _check_reuse_furls(self):
+        if not self.reuse_furls:
+            furl_files = [i.furl_file for i in self.Interfaces.values()]
+            for ff in furl_files:
+                fullfile = self._get_security_file(ff)
+                if os.path.isfile(fullfile):
+                    os.remove(fullfile)
+
+    def _get_security_file(self, filename):
+        return os.path.join(self.config.Global.security_dir, filename)
+
     def create(self):
         """Create and return the Foolscap tub with everything running."""
 
         self.tub, self.listener = make_tub(
-            self.ip, self.port, self.secure, self.cert_file)
+            self.ip, self.port, self.secure, self._get_security_file(self.cert_file))
+        log.msg("Created a tub and listener [%r]: %r, %r" % (self.__class__, self.tub, self.listener))
+        log.msg("Interfaces to register [%r]: %r" % (self.__class__, self.Interfaces))
         if not self.secure:
             log.msg("WARNING: running with no security: %s" % self.__class__.__name__)
         reactor.callWhenRunning(self.set_location_and_register)
         return self.tub
 
     def set_location_and_register(self):
         """Set the location for the tub and return a deferred."""
 
         if self.location == '':
             d = self.tub.setLocationAutomatically()
         else:
             d = defer.maybeDeferred(self.tub.setLocation,
                 "%s:%i" % (self.location, self.listener.getPortnum()))
         self.adapt_to_interfaces(d)
 
     def adapt_to_interfaces(self, d):
         """Run through the interfaces, adapt and register."""
 
         for ifname, ifconfig in self.Interfaces.iteritems():
+            ff = self._get_security_file(ifconfig.furl_file)
             log.msg("Adapting %r to interface: %s" % (self.adaptee, ifname))
-            log.msg("Saving furl for interface [%s] to file: %s" % (ifname, ifconfig.furl_file))
-            check_furl_file_security(ifconfig.furl_file, self.secure)
+            log.msg("Saving furl for interface [%s] to file: %s" % (ifname, ff))
+            check_furl_file_security(ff, self.secure)
             adaptee = self.adaptee
             for i in ifconfig.interface_chain:
                 adaptee = import_item(i)(adaptee)
-            d.addCallback(self.register, adaptee, furl_file=ifconfig.furl_file)
+            d.addCallback(self.register, adaptee, furl_file=ff)
 
     def register(self, empty, ref, furl_file):
         """Register the reference with the FURL file.
 
         The FURL file is created and then moved to make sure that when the
         file appears, the buffer has been flushed and the file closed.
         """
         temp_furl_file = get_temp_furlfile(furl_file)
         self.tub.registerReference(ref, furlFile=temp_furl_file)
         os.rename(temp_furl_file, furl_file)
 

IPython/kernel/scripts/ipcontrollerapp.py

@@ -1,266 +1,310 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The IPython controller application
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import copy
+import logging
 import os
 import sys
 
 from twisted.application import service
 from twisted.internet import reactor, defer
 from twisted.python import log
 
 from IPython.config.loader import Config, NoConfigDefault
 
 from IPython.core.application import Application, IPythonArgParseConfigLoader
 from IPython.core import release
 
 from IPython.utils.traitlets import Int, Str, Bool, Instance
 from IPython.utils.importstring import import_item
 
 from IPython.kernel import controllerservice
 from IPython.kernel.configobjfactory import (
     ConfiguredObjectFactory,
     AdaptedConfiguredObjectFactory
 )
 
 from IPython.kernel.fcutil import FCServiceFactory
 
 #-----------------------------------------------------------------------------
-# Components for creating services
+# Default interfaces
 #-----------------------------------------------------------------------------
 
 
 # The default client interfaces for FCClientServiceFactory.Interfaces
 default_client_interfaces = Config()
 default_client_interfaces.Task.interface_chain = [
     'IPython.kernel.task.ITaskController',
     'IPython.kernel.taskfc.IFCTaskController'
 ]
+
 default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
 default_client_interfaces.MultiEngine.interface_chain = [
     'IPython.kernel.multiengine.IMultiEngine',
     'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
 ]
+
 default_client_interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl'
 
 # Make this a dict we can pass to Config.__init__ for the default
 default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))
 
 
 
 # The default engine interfaces for FCEngineServiceFactory.Interfaces
 default_engine_interfaces = Config()
 default_engine_interfaces.Default.interface_chain = [
     'IPython.kernel.enginefc.IFCControllerBase'
 ]
+
 default_engine_interfaces.Default.furl_file = 'ipcontroller-engine.furl'
 
 # Make this a dict we can pass to Config.__init__ for the default
 default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
 
 
+#-----------------------------------------------------------------------------
+# Service factories
+#-----------------------------------------------------------------------------
+
 
 class FCClientServiceFactory(FCServiceFactory):
     """A Foolscap implementation of the client services."""
 
     cert_file = Str('ipcontroller-client.pem', config=True)
     Interfaces = Instance(klass=Config, kw=default_client_interfaces,
                           allow_none=False, config=True)
 
 
 class FCEngineServiceFactory(FCServiceFactory):
     """A Foolscap implementation of the engine services."""
 
     cert_file = Str('ipcontroller-engine.pem', config=True)
-    interfaces = Instance(klass=dict, kw=default_engine_interfaces,
+    Interfaces = Instance(klass=dict, kw=default_engine_interfaces,
                           allow_none=False, config=True)
 
 
 #-----------------------------------------------------------------------------
 # The main application
 #-----------------------------------------------------------------------------
 
 
 cl_args = (
     # Client config
     (('--client-ip',), dict(
         type=str, dest='FCClientServiceFactory.ip', default=NoConfigDefault,
         help='The IP address or hostname the controller will listen on for client connections.',
         metavar='FCClientServiceFactory.ip')
     ),
     (('--client-port',), dict(
         type=int, dest='FCClientServiceFactory.port', default=NoConfigDefault,
         help='The port the controller will listen on for client connections.',
         metavar='FCClientServiceFactory.port')
     ),
     (('--client-location',), dict(
         type=str, dest='FCClientServiceFactory.location', default=NoConfigDefault,
         help='The hostname or ip that clients should connect to.',
         metavar='FCClientServiceFactory.location')
     ),
     (('-x',), dict(
         action='store_false', dest='FCClientServiceFactory.secure', default=NoConfigDefault,
         help='Turn off all client security.')
     ),
-    (('--client-cert-file',), dict(
-        type=str, dest='FCClientServiceFactory.cert_file', default=NoConfigDefault,
-        help='File to store the client SSL certificate in.',
-        metavar='FCClientServiceFactory.cert_file')
-    ),
-    (('--task-furl-file',), dict(
-        type=str, dest='FCClientServiceFactory.Interfaces.Task.furl_file', default=NoConfigDefault,
-        help='File to store the FURL in for task clients to connect with.',
-        metavar='FCClientServiceFactory.Interfaces.Task.furl_file')
-    ),
-    (('--multiengine-furl-file',), dict(
-        type=str, dest='FCClientServiceFactory.Interfaces.MultiEngine.furl_file', default=NoConfigDefault,
-        help='File to store the FURL in for multiengine clients to connect with.',
-        metavar='FCClientServiceFactory.Interfaces.MultiEngine.furl_file')
-    ),
     # Engine config
     (('--engine-ip',), dict(
         type=str, dest='FCEngineServiceFactory.ip', default=NoConfigDefault,
         help='The IP address or hostname the controller will listen on for engine connections.',
         metavar='FCEngineServiceFactory.ip')
     ),
     (('--engine-port',), dict(
         type=int, dest='FCEngineServiceFactory.port', default=NoConfigDefault,
         help='The port the controller will listen on for engine connections.',
         metavar='FCEngineServiceFactory.port')
     ),
     (('--engine-location',), dict(
         type=str, dest='FCEngineServiceFactory.location', default=NoConfigDefault,
         help='The hostname or ip that engines should connect to.',
         metavar='FCEngineServiceFactory.location')
     ),
     (('-y',), dict(
         action='store_false', dest='FCEngineServiceFactory.secure', default=NoConfigDefault,
         help='Turn off all engine security.')
     ),
-    (('--engine-cert-file',), dict(
-        type=str, dest='FCEngineServiceFactory.cert_file', default=NoConfigDefault,
-        help='File to store the client SSL certificate in.',
-        metavar='FCEngineServiceFactory.cert_file')
-    ),
-    (('--engine-furl-file',), dict(
-        type=str, dest='FCEngineServiceFactory.Interfaces.Default.furl_file', default=NoConfigDefault,
-        help='File to store the FURL in for engines to connect with.',
-        metavar='FCEngineServiceFactory.Interfaces.Default.furl_file')
-    ),
     # Global config
-    (('-l','--logfile'), dict(
-        type=str, dest='Global.logfile', default=NoConfigDefault,
-        help='Log file name (default is stdout)',
-        metavar='Global.logfile')
+    (('--log-to-file',), dict(
+        action='store_true', dest='Global.log_to_file', default=NoConfigDefault,
+        help='Log to a file in the log directory (default is stdout)')
     ),
-    (('-r',), dict(
+    (('-r','--reuse-furls'), dict(
         action='store_true', dest='Global.reuse_furls', default=NoConfigDefault,
         help='Try to reuse all FURL files.')
-    )
+    ),
+    (('-cluster_dir', '--cluster-dir',), dict(
+        type=str, dest='Global.cluster_dir', default=NoConfigDefault,
+        help='Absolute or relative path to the cluster directory.',
+        metavar='Global.cluster_dir')
+    ),
 )
 
 
 class IPControllerAppCLConfigLoader(IPythonArgParseConfigLoader):
 
     arguments = cl_args
 
 
 _default_config_file_name = 'ipcontroller_config.py'
 
 class IPControllerApp(Application):
 
     name = 'ipcontroller'
     config_file_name = _default_config_file_name
+    default_log_level = logging.DEBUG
 
     def create_default_config(self):
         super(IPControllerApp, self).create_default_config()
-        self.default_config.Global.logfile = ''
         self.default_config.Global.reuse_furls = False
         self.default_config.Global.import_statements = []
+        self.default_config.Global.profile = 'default'
+        self.default_config.Global.log_dir_name = 'log'
+        self.default_config.Global.security_dir_name = 'security'
+        self.default_config.Global.log_to_file = False
+        # Resolve the default cluster_dir using the default profile
+        self.default_config.Global.cluster_dir = ''
 
     def create_command_line_config(self):
         """Create and return a command line config loader."""
 
         return IPControllerAppCLConfigLoader(
             description="Start an IPython controller",
             version=release.version)
 
+    def find_config_file_name(self):
+        """Find the config file name for this application."""
+        self.find_cluster_dir()
+        self.create_cluster_dir()
+
+    def find_cluster_dir(self):
+        """This resolves into full paths, the various cluster directories.
+
+        This method must set ``self.cluster_dir`` to the full paths of 
+        the directory.
+        """
+        # Ignore self.command_line_config.Global.config_file
+        # Instead, first look for an explicit cluster_dir
+        try:
+            self.cluster_dir = self.command_line_config.Global.cluster_dir
+        except AttributeError:
+            self.cluster_dir = self.default_config.Global.cluster_dir
+        self.cluster_dir = os.path.expandvars(os.path.expanduser(self.cluster_dir))
+        if not self.cluster_dir:
+            # Then look for a profile
+            try:
+                self.profile = self.command_line_config.Global.profile
+            except AttributeError:
+                self.profile = self.default_config.Global.profile
+            cluster_dir_name = 'cluster_' + self.profile
+            try_this = os.path.join(os.getcwd(), cluster_dir_name)
+            if os.path.isdir(try_this):
+                self.cluster_dir = try_this
+            else:
+                self.cluster_dir = os.path.join(self.ipythondir, cluster_dir_name)
+        # These have to be set because they could be different from the one
+        # that we just computed.  Because command line has the highest
+        # priority, this will always end up in the master_config.
+        self.default_config.Global.cluster_dir = self.cluster_dir
+        self.command_line_config.Global.cluster_dir = self.cluster_dir
+
+    def create_cluster_dir(self):
+        """Make sure that the cluster, security and log dirs exist."""
+        if not os.path.isdir(self.cluster_dir):
+            os.makedirs(self.cluster_dir, mode=0777)
+
+    def find_config_file_paths(self):
+        """Set the search paths for resolving the config file."""
+        self.config_file_paths = (self.cluster_dir,)
+
+    def pre_construct(self):
+        # Now set the security_dir and log_dir and create them.  We use
+        # the names an construct the absolute paths.
+        security_dir = os.path.join(self.master_config.Global.cluster_dir,
+                                    self.master_config.Global.security_dir_name)
+        log_dir = os.path.join(self.master_config.Global.cluster_dir, 
+                               self.master_config.Global.log_dir_name)
+        if not os.path.isdir(security_dir):
+            os.mkdir(security_dir, 0700)
+        else:
+            os.chmod(security_dir, 0700)
+        if not os.path.isdir(log_dir):
+            os.mkdir(log_dir, 0777)
+
+        self.security_dir = self.master_config.Global.security_dir = security_dir
+        self.log_dir = self.master_config.Global.log_dir = log_dir
+
+        # Now setup reuse_furls
+        if hasattr(self.master_config.Global.reuse_furls):
+            self.master_config.FCClientServiceFactory.reuse_furls = \
+                self.master_config.Global.reuse_furls
+            self.master_config.FCEngineServiceFactory.reuse_furls = \
+                self.master_config.Global.reuse_furls
+
     def construct(self):
         # I am a little hesitant to put these into InteractiveShell itself.
         # But that might be the place for them
         sys.path.insert(0, '')
 
         self.start_logging()
         self.import_statements()
-        self.reuse_furls()
 
         # Create the service hierarchy
         self.main_service = service.MultiService()
         # The controller service
         controller_service = controllerservice.ControllerService()
         controller_service.setServiceParent(self.main_service)
         # The client tub and all its refereceables
         csfactory = FCClientServiceFactory(self.master_config, controller_service)
         client_service = csfactory.create()
         client_service.setServiceParent(self.main_service)
         # The engine tub
         esfactory = FCEngineServiceFactory(self.master_config, controller_service)
         engine_service = esfactory.create()
         engine_service.setServiceParent(self.main_service)
 
     def start_logging(self):
-        logfile = self.master_config.Global.logfile
-        if logfile:
-            logfile = logfile + str(os.getpid()) + '.log'
-            try:
-                openLogFile = open(logfile, 'w')
-            except:
-                openLogFile = sys.stdout
+        if self.master_config.Global.log_to_file:
+            log_filename = self.name + '-' + str(os.getpid()) + '.log'
+            logfile = os.path.join(self.log_dir, log_filename)
+            open_log_file = open(logfile, 'w')
         else:
-            openLogFile = sys.stdout
-        log.startLogging(openLogFile)
+            open_log_file = sys.stdout
+        log.startLogging(open_log_file)
 
     def import_statements(self):
         statements = self.master_config.Global.import_statements
         for s in statements:
             try:
                 exec s in globals(), locals()
             except:
                 log.msg("Error running import statement: %s" % s)
 
-    def reuse_furls(self):
-        # This logic might need to be moved into the components
-        # Delete old furl files unless the reuse_furls is set
-        reuse = self.master_config.Global.reuse_furls
-        # if not reuse:
-        #     paths = (
-        #         self.master_config.FCEngineServiceFactory.Interfaces.Default.furl_file,
-        #         self.master_config.FCClientServiceFactory.Interfaces.Task.furl_file,
-        #         self.master_config.FCClientServiceFactory.Interfaces.MultiEngine.furl_file
-        #     )
-        #     for p in paths:
-        #         if os.path.isfile(p):
-        #             os.remove(p)
-
     def start_app(self):
         # Start the controller service and set things running
         self.main_service.startService()
         reactor.run()
 
 if __name__ == '__main__':
     app = IPControllerApp()
     app.start()