diff --git a/IPython/config/default/ipcontroller_config.py b/IPython/config/default/ipcontroller_config.py index 216d855..0ee2548 100644 --- a/IPython/config/default/ipcontroller_config.py +++ b/IPython/config/default/ipcontroller_config.py @@ -6,62 +6,67 @@ c = get_config() # Global configuration #----------------------------------------------------------------------------- -c.Global.log_to_file = False -c.Global.import_statements = [] -c.Global.reuse_furls = False +# Basic Global config attributes +# c.Global.log_to_file = False +# c.Global.import_statements = ['import math'] +# c.Global.reuse_furls = True +# c.Global.secure = True -# You shouldn't have to edit these -c.Global.log_dir_name = 'log' -c.Global.security_dir_name = 'security' +# You shouldn't have to modify these +# c.Global.log_dir_name = 'log' +# c.Global.security_dir_name = 'security' #----------------------------------------------------------------------------- # Configure the client services #----------------------------------------------------------------------------- -c.FCClientServiceFactory.ip = '' -c.FCClientServiceFactory.port = 0 -c.FCClientServiceFactory.location = '' -c.FCClientServiceFactory.secure = True -c.FCClientServiceFactory.reuse_furls = False -c.FCClientServiceFactory.cert_file = 'ipcontroller-client.pem' - -c.FCClientServiceFactory.Interfaces.Task.interface_chain = [ - 'IPython.kernel.task.ITaskController', - 'IPython.kernel.taskfc.IFCTaskController' -] -# This is just the filename of the furl file. The path is always the -# security dir of the cluster directory. -c.FCClientServiceFactory.Interfaces.Task.furl_file = 'ipcontroller-tc.furl' - -c.FCClientServiceFactory.Interfaces.MultiEngine.interface_chain = [ - 'IPython.kernel.multiengine.IMultiEngine', - 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine' -] -# This is just the filename of the furl file. The path is always the -# security dir of the cluster directory. -c.FCClientServiceFactory.Interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl' - +# Basic client service config attributes +# c.FCClientServiceFactory.ip = '' +# c.FCClientServiceFactory.port = 0 +# c.FCClientServiceFactory.location = '' +# c.FCClientServiceFactory.secure = True +# c.FCClientServiceFactory.reuse_furls = False + +# You shouldn't have to modify the rest of this section +# c.FCClientServiceFactory.cert_file = 'ipcontroller-client.pem' + +# default_client_interfaces = Config() +# default_client_interfaces.Task.interface_chain = [ +# 'IPython.kernel.task.ITaskController', +# 'IPython.kernel.taskfc.IFCTaskController' +# ] +# +# default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl' +# +# default_client_interfaces.MultiEngine.interface_chain = [ +# 'IPython.kernel.multiengine.IMultiEngine', +# 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine' +# ] +# +# default_client_interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl' +# +# c.FCEngineServiceFactory.interfaces = default_client_interfaces #----------------------------------------------------------------------------- # Configure the engine services #----------------------------------------------------------------------------- -c.FCEngineServiceFactory.ip = '' -c.FCEngineServiceFactory.port = 0 -c.FCEngineServiceFactory.location = '' -c.FCEngineServiceFactory.secure = True -c.FCEngineServiceFactory.reuse_furls = False -c.FCEngineServiceFactory.cert_file = 'ipcontroller-engine.pem' - -c.FCEngineServiceFactory.Intefaces.Default.interface_chain = [ - 'IPython.kernel.enginefc.IFCControllerBase' -] - -# This is just the filename of the furl file. The path is always the -# security dir of the cluster directory. -c.FCEngineServiceFactory.Intefaces.Default.furl_file = 'ipcontroller-engine.furl' - - - - +# Basic config attributes for the engine services +# c.FCEngineServiceFactory.ip = '' +# c.FCEngineServiceFactory.port = 0 +# c.FCEngineServiceFactory.location = '' +# c.FCEngineServiceFactory.secure = True +# c.FCEngineServiceFactory.reuse_furls = False + +# You shouldn't have to modify the rest of this section +# c.FCEngineServiceFactory.cert_file = 'ipcontroller-engine.pem' + +# default_engine_interfaces = Config() +# default_engine_interfaces.Default.interface_chain = [ +# 'IPython.kernel.enginefc.IFCControllerBase' +# ] +# +# default_engine_interfaces.Default.furl_file = 'ipcontroller-engine.furl' +# +# c.FCEngineServiceFactory.interfaces = default_engine_interfaces diff --git a/IPython/core/application.py b/IPython/core/application.py index 42b2f46..49e22e5 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -77,9 +77,9 @@ class Application(object): """Load a config, construct an app and run it. """ - config_file_name = 'ipython_config.py' name = 'ipython' description = 'IPython: an enhanced interactive Python shell.' + config_file_name = 'ipython_config.py' default_log_level = logging.WARN @@ -351,7 +351,11 @@ class AppWithDirArgParseConfigLoader(ArgParseConfigLoader): metavar='Global.ipythondir') self.parser.add_argument('-p','-profile', '--profile', dest='Global.profile',type=str, - help='The string name of the ipython profile to be used.', + help='The string name of the profile to be used. This determines ' + 'the name of the application dir: basename_. The basename is ' + 'determined by the particular application. The default profile ' + 'is named "default". This convention is used if the -app_dir ' + 'option is not used.', default=NoConfigDefault, metavar='Global.profile') self.parser.add_argument('-log_level', '--log-level', @@ -360,22 +364,40 @@ class AppWithDirArgParseConfigLoader(ArgParseConfigLoader): default=NoConfigDefault) self.parser.add_argument('-app_dir', '--app-dir', dest='Global.app_dir',type=str, - help='Set the application directory where everything for this ' - 'application will be found (including the config file).', + help='Set the application dir where everything for this ' + 'application will be found (including the config file). This ' + 'overrides the logic used by the profile option.', default=NoConfigDefault, metavar='Global.app_dir') class ApplicationWithDir(Application): + """An application that puts everything into a application directory. - name = 'appname' - description = 'Application: foo and bar it.' - config_file_name = 'appname_config.py' - default_log_level = logging.WARN + Instead of looking for things in the ipythondir, this type of application + will use its own private directory called the "application directory" + for things like config files, log files, etc. + + The application directory is resolved as follows: + + * If the ``--app-dir`` option is given, it is used. + * If ``--app-dir`` is not given, the application directory is resolve using + ``app_dir_basename`` and ``profile`` as ``_``. + The search path for this directory is then i) cwd if it is found there + and ii) in ipythondir otherwise. + + The config file for the application is to be put in the application + dir and named the value of the ``config_file_name`` class attribute. + """ + + # The basename used for the application dir: _ + app_dir_basename = 'cluster' def create_default_config(self): super(ApplicationWithDir, self).create_default_config() self.default_config.Global.profile = 'default' + # The application dir. This is empty initially so the default is to + # try to resolve this using the profile. self.default_config.Global.app_dir = '' def create_command_line_config(self): @@ -391,10 +413,10 @@ class ApplicationWithDir(Application): self.create_app_dir() def find_app_dir(self): - """This resolves into full paths, the app directory. + """This resolves the app directory. - This method must set ``self.app_dir`` to the full path of - the directory. + This method must set ``self.app_dir`` to the location of the app + dir. """ # Instead, first look for an explicit app_dir try: @@ -408,7 +430,7 @@ class ApplicationWithDir(Application): self.profile = self.command_line_config.Global.profile except AttributeError: self.profile = self.default_config.Global.profile - app_dir_name = 'cluster_' + self.profile + app_dir_name = self.app_dir_basename + '_' + self.profile try_this = os.path.join(os.getcwd(), app_dir_name) if os.path.isdir(try_this): self.app_dir = try_this @@ -421,7 +443,7 @@ class ApplicationWithDir(Application): self.command_line_config.Global.app_dir = self.app_dir def create_app_dir(self): - """Make sure that the cluster, security and log dirs exist.""" + """Make sure that the app dir exists.""" if not os.path.isdir(self.app_dir): os.makedirs(self.app_dir, mode=0777) diff --git a/IPython/core/component.py b/IPython/core/component.py index 9e72790..b2bbeda 100644 --- a/IPython/core/component.py +++ b/IPython/core/component.py @@ -296,7 +296,19 @@ class Component(HasTraitlets): if new._has_section(sname): my_config = new[sname] for k, v in traitlets.items(): + # Don't allow traitlets with config=True to start with + # uppercase. Otherwise, they are confused with Config + # subsections. But, developers shouldn't have uppercase + # attributes anyways! (PEP 6) + if k[0].upper()==k[0] and not k.startswith('_'): + raise ComponentError('Component traitlets with ' + 'config=True must start with a lowercase so they are ' + 'not confused with Config subsections: %s.%s' % \ + (self.__class__.__name__, k)) try: + # Here we grab the value from the config + # If k has the naming convention of a config + # section, it will be auto created. config_value = my_config[k] except KeyError: pass diff --git a/IPython/core/ipapp.py b/IPython/core/ipapp.py index d64cf6f..1dad392 100644 --- a/IPython/core/ipapp.py +++ b/IPython/core/ipapp.py @@ -294,13 +294,13 @@ class IPythonAppCLConfigLoader(BaseAppArgParseConfigLoader): arguments = cl_args -_default_config_file_name = 'ipython_config.py' +default_config_file_name = 'ipython_config.py' class IPythonApp(Application): name = 'ipython' description = 'IPython: an enhanced interactive Python shell.' - config_file_name = _default_config_file_name + config_file_name = default_config_file_name def create_default_config(self): super(IPythonApp, self).create_default_config() @@ -533,7 +533,7 @@ def load_default_config(ipythondir=None): """ if ipythondir is None: ipythondir = get_ipython_dir() - cl = PyFileConfigLoader(_default_config_file_name, ipythondir) + cl = PyFileConfigLoader(default_config_file_name, ipythondir) config = cl.load_config() return config diff --git a/IPython/kernel/configobjfactory.py b/IPython/kernel/configobjfactory.py index 3d8cd24..64a9f47 100644 --- a/IPython/kernel/configobjfactory.py +++ b/IPython/kernel/configobjfactory.py @@ -68,7 +68,11 @@ class AdaptedConfiguredObjectFactory(Component): # zi.implements(IAdaptedConfiguredObjectFactory) def __init__(self, config, adaptee): + # print + # print "config pre:", config super(AdaptedConfiguredObjectFactory, self).__init__(None, config=config) + # print + # print "config post:", config self.adaptee = adaptee def create(self): diff --git a/IPython/kernel/fcutil.py b/IPython/kernel/fcutil.py index e7ec060..08ceda4 100644 --- a/IPython/kernel/fcutil.py +++ b/IPython/kernel/fcutil.py @@ -67,11 +67,11 @@ def is_secure(furl): elif furl.startswith("pbu://"): return False else: - raise ValueError("invalid furl: %s" % furl) + raise ValueError("invalid FURL: %s" % furl) def is_valid(furl): - """Is the str a valid furl or not.""" + """Is the str a valid FURL or not.""" if isinstance(furl, str): if furl.startswith("pb://") or furl.startswith("pbu://"): return True @@ -88,11 +88,11 @@ def find_furl(furl_or_file): furl = open(furl_or_file, 'r').read().strip() if is_valid(furl): return furl - raise ValueError("not a furl or a file containing a furl: %s" % furl_or_file) + raise ValueError("not a FURL or a file containing a FURL: %s" % furl_or_file) def get_temp_furlfile(filename): - """Return a temporary furl file.""" + """Return a temporary FURL file.""" return tempfile.mktemp(dir=os.path.dirname(filename), prefix=os.path.basename(filename)) @@ -132,6 +132,7 @@ def make_tub(ip, port, secure, cert_file): strport = "tcp:%i" % port else: strport = "tcp:%i:interface=%s" % (port, ip) + log.msg("Starting listener with [secure=%r] on: %s" % (secure, strport)) listener = tub.listenOn(strport) return tub, listener @@ -147,7 +148,7 @@ class FCServiceFactory(AdaptedConfiguredObjectFactory): Attributes ---------- - Interfaces : Config + interfaces : Config A Config instance whose values are sub-Config objects having two keys: furl_file and interface_chain. @@ -160,7 +161,7 @@ class FCServiceFactory(AdaptedConfiguredObjectFactory): cert_file = Str('', config=True) location = Str('', config=True) reuse_furls = Bool(False, config=True) - Interfaces = Instance(klass=Config, kw={}, allow_none=False, config=True) + interfaces = Instance(klass=Config, kw={}, allow_none=False, config=True) def __init__(self, config, adaptee): super(FCServiceFactory, self).__init__(config, adaptee) @@ -171,11 +172,14 @@ class FCServiceFactory(AdaptedConfiguredObjectFactory): self.location = '127.0.0.1' def _check_reuse_furls(self): - if not self.reuse_furls: - furl_files = [i.furl_file for i in self.Interfaces.values()] - for ff in furl_files: - fullfile = self._get_security_file(ff) + furl_files = [i.furl_file for i in self.interfaces.values()] + for ff in furl_files: + fullfile = self._get_security_file(ff) + if self.reuse_furls: + log.msg("Reusing FURL file: %s" % fullfile) + else: if os.path.isfile(fullfile): + log.msg("Removing old FURL file: %s" % fullfile) os.remove(fullfile) def _get_security_file(self, filename): @@ -185,11 +189,14 @@ class FCServiceFactory(AdaptedConfiguredObjectFactory): """Create and return the Foolscap tub with everything running.""" self.tub, self.listener = make_tub( - self.ip, self.port, self.secure, self._get_security_file(self.cert_file)) - log.msg("Created a tub and listener [%r]: %r, %r" % (self.__class__, self.tub, self.listener)) - log.msg("Interfaces to register [%r]: %r" % (self.__class__, self.Interfaces)) + self.ip, self.port, self.secure, + self._get_security_file(self.cert_file) + ) + # log.msg("Interfaces to register [%r]: %r" % \ + # (self.__class__, self.interfaces)) if not self.secure: - log.msg("WARNING: running with no security: %s" % self.__class__.__name__) + log.msg("WARNING: running with no security: %s" % \ + self.__class__.__name__) reactor.callWhenRunning(self.set_location_and_register) return self.tub @@ -206,10 +213,11 @@ class FCServiceFactory(AdaptedConfiguredObjectFactory): def adapt_to_interfaces(self, d): """Run through the interfaces, adapt and register.""" - for ifname, ifconfig in self.Interfaces.iteritems(): + for ifname, ifconfig in self.interfaces.iteritems(): ff = self._get_security_file(ifconfig.furl_file) - log.msg("Adapting %r to interface: %s" % (self.adaptee, ifname)) - log.msg("Saving furl for interface [%s] to file: %s" % (ifname, ff)) + log.msg("Adapting [%s] to interface: %s" % \ + (self.adaptee.__class__.__name__, ifname)) + log.msg("Saving FURL for interface [%s] to file: %s" % (ifname, ff)) check_furl_file_security(ff, self.secure) adaptee = self.adaptee for i in ifconfig.interface_chain: diff --git a/IPython/kernel/ipcontrollerapp.py b/IPython/kernel/ipcontrollerapp.py index 65abba8..dcb85d1 100644 --- a/IPython/kernel/ipcontrollerapp.py +++ b/IPython/kernel/ipcontrollerapp.py @@ -28,7 +28,7 @@ from IPython.config.loader import Config, NoConfigDefault from IPython.core.application import ( ApplicationWithDir, - BaseAppArgParseConfigLoader + AppWithDirArgParseConfigLoader ) from IPython.core import release @@ -49,7 +49,7 @@ from IPython.kernel.fcutil import FCServiceFactory #----------------------------------------------------------------------------- -# The default client interfaces for FCClientServiceFactory.Interfaces +# The default client interfaces for FCClientServiceFactory.interfaces default_client_interfaces = Config() default_client_interfaces.Task.interface_chain = [ 'IPython.kernel.task.ITaskController', @@ -57,6 +57,7 @@ default_client_interfaces.Task.interface_chain = [ ] default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl' + default_client_interfaces.MultiEngine.interface_chain = [ 'IPython.kernel.multiengine.IMultiEngine', 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine' @@ -69,7 +70,7 @@ default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items() -# The default engine interfaces for FCEngineServiceFactory.Interfaces +# The default engine interfaces for FCEngineServiceFactory.interfaces default_engine_interfaces = Config() default_engine_interfaces.Default.interface_chain = [ 'IPython.kernel.enginefc.IFCControllerBase' @@ -90,7 +91,7 @@ class FCClientServiceFactory(FCServiceFactory): """A Foolscap implementation of the client services.""" cert_file = Str('ipcontroller-client.pem', config=True) - Interfaces = Instance(klass=Config, kw=default_client_interfaces, + interfaces = Instance(klass=Config, kw=default_client_interfaces, allow_none=False, config=True) @@ -98,7 +99,7 @@ class FCEngineServiceFactory(FCServiceFactory): """A Foolscap implementation of the engine services.""" cert_file = Str('ipcontroller-engine.pem', config=True) - Interfaces = Instance(klass=dict, kw=default_engine_interfaces, + interfaces = Instance(klass=dict, kw=default_engine_interfaces, allow_none=False, config=True) @@ -124,10 +125,6 @@ cl_args = ( help='The hostname or ip that clients should connect to.', metavar='FCClientServiceFactory.location') ), - (('-x',), dict( - action='store_false', dest='FCClientServiceFactory.secure', default=NoConfigDefault, - help='Turn off all client security.') - ), # Engine config (('--engine-ip',), dict( type=str, dest='FCEngineServiceFactory.ip', default=NoConfigDefault, @@ -144,10 +141,6 @@ cl_args = ( help='The hostname or ip that engines should connect to.', metavar='FCEngineServiceFactory.location') ), - (('-y',), dict( - action='store_false', dest='FCEngineServiceFactory.secure', default=NoConfigDefault, - help='Turn off all engine security.') - ), # Global config (('--log-to-file',), dict( action='store_true', dest='Global.log_to_file', default=NoConfigDefault, @@ -156,33 +149,61 @@ cl_args = ( (('-r','--reuse-furls'), dict( action='store_true', dest='Global.reuse_furls', default=NoConfigDefault, help='Try to reuse all FURL files.') + ), + (('-ns','--no-security'), dict( + action='store_false', dest='Global.secure', default=NoConfigDefault, + help='Turn off SSL encryption for all connections.') ) ) -class IPControllerAppCLConfigLoader(BaseAppArgParseConfigLoader): +class IPControllerAppCLConfigLoader(AppWithDirArgParseConfigLoader): arguments = cl_args -_default_config_file_name = 'ipcontroller_config.py' +default_config_file_name = 'ipcontroller_config.py' class IPControllerApp(ApplicationWithDir): name = 'ipcontroller' + app_dir_basename = 'cluster' description = 'Start the IPython controller for parallel computing.' - config_file_name = _default_config_file_name - default_log_level = logging.DEBUG + config_file_name = default_config_file_name + default_log_level = logging.WARN def create_default_config(self): super(IPControllerApp, self).create_default_config() self.default_config.Global.reuse_furls = False + self.default_config.Global.secure = True self.default_config.Global.import_statements = [] self.default_config.Global.log_dir_name = 'log' self.default_config.Global.security_dir_name = 'security' self.default_config.Global.log_to_file = False + def create_command_line_config(self): + """Create and return a command line config loader.""" + return IPControllerAppCLConfigLoader( + description=self.description, + version=release.version + ) + + def post_load_command_line_config(self): + # Now setup reuse_furls + if hasattr(self.command_line_config.Global, 'reuse_furls'): + self.command_line_config.FCClientServiceFactory.reuse_furls = \ + self.command_line_config.Global.reuse_furls + self.command_line_config.FCEngineServiceFactory.reuse_furls = \ + self.command_line_config.Global.reuse_furls + del self.command_line_config.Global.reuse_furls + if hasattr(self.command_line_config.Global, 'secure'): + self.command_line_config.FCClientServiceFactory.secure = \ + self.command_line_config.Global.secure + self.command_line_config.FCEngineServiceFactory.secure = \ + self.command_line_config.Global.secure + del self.command_line_config.Global.secure + def pre_construct(self): # Now set the security_dir and log_dir and create them. We use # the names an construct the absolute paths. @@ -200,13 +221,6 @@ class IPControllerApp(ApplicationWithDir): self.security_dir = self.master_config.Global.security_dir = security_dir self.log_dir = self.master_config.Global.log_dir = log_dir - # Now setup reuse_furls - if hasattr(self.master_config.Global, 'reuse_furls'): - self.master_config.FCClientServiceFactory.reuse_furls = \ - self.master_config.Global.reuse_furls - self.master_config.FCEngineServiceFactory.reuse_furls = \ - self.master_config.Global.reuse_furls - def construct(self): # I am a little hesitant to put these into InteractiveShell itself. # But that might be the place for them @@ -214,7 +228,7 @@ class IPControllerApp(ApplicationWithDir): self.start_logging() self.import_statements() - + # Create the service hierarchy self.main_service = service.MultiService() # The controller service @@ -242,9 +256,10 @@ class IPControllerApp(ApplicationWithDir): statements = self.master_config.Global.import_statements for s in statements: try: + log.msg("Executing statement: '%s'" % s) exec s in globals(), locals() except: - log.msg("Error running import statement: %s" % s) + log.msg("Error running statement: %s" % s) def start_app(self): # Start the controller service and set things running