add cluster_id to parallel apps

This allows multiple controller instances per profile.  Default behavior remains unchanged.

IPython/parallel/apps/baseapp.py

@@ -1,244 +1,257 b''
 # encoding: utf-8
 """
 The Base Application class for IPython.parallel apps
 
 Authors:
 
 * Brian Granger
 * Min RK
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
 
 import os
 import logging
 import re
 import sys
 
 from subprocess import Popen, PIPE
 
 from IPython.core import release
 from IPython.core.crashhandler import CrashHandler
 from IPython.core.application import (
     BaseIPythonApplication,
     base_aliases as base_ip_aliases,
     base_flags as base_ip_flags
 )
 from IPython.utils.path import expand_path
 
 from IPython.utils.traitlets import Unicode, Bool, Instance, Dict, List
 
 #-----------------------------------------------------------------------------
 # Module errors
 #-----------------------------------------------------------------------------
 
 class PIDFileError(Exception):
     pass
 
 
 #-----------------------------------------------------------------------------
 # Crash handler for this application
 #-----------------------------------------------------------------------------
 
 class ParallelCrashHandler(CrashHandler):
     """sys.excepthook for IPython itself, leaves a detailed report on disk."""
 
     def __init__(self, app):
         contact_name = release.authors['Min'][0]
         contact_email = release.authors['Min'][1]
         bug_tracker = 'http://github.com/ipython/ipython/issues'
         super(ParallelCrashHandler,self).__init__(
             app, contact_name, contact_email, bug_tracker
         )
 
 
 #-----------------------------------------------------------------------------
 # Main application
 #-----------------------------------------------------------------------------
 base_aliases = {}
 base_aliases.update(base_ip_aliases)
 base_aliases.update({
     'profile-dir' : 'ProfileDir.location',
     'work-dir' : 'BaseParallelApplication.work_dir',
     'log-to-file' : 'BaseParallelApplication.log_to_file',
     'clean-logs' : 'BaseParallelApplication.clean_logs',
     'log-url' : 'BaseParallelApplication.log_url',
+    'cluster-id' : 'BaseParallelApplication.cluster_id',
 })
 
 base_flags = {
     'log-to-file' : (
         {'BaseParallelApplication' : {'log_to_file' : True}},
         "send log output to a file"
     )
 }
 base_flags.update(base_ip_flags)
 
 class BaseParallelApplication(BaseIPythonApplication):
     """The base Application for IPython.parallel apps
     
     Principle extensions to BaseIPyythonApplication:
     
     * work_dir
     * remote logging via pyzmq
     * IOLoop instance
     """
 
     crash_handler_class = ParallelCrashHandler
     
     def _log_level_default(self):
         # temporarily override default_log_level to INFO
         return logging.INFO
 
     work_dir = Unicode(os.getcwdu(), config=True,
         help='Set the working dir for the process.'
     )
     def _work_dir_changed(self, name, old, new):
         self.work_dir = unicode(expand_path(new))
 
     log_to_file = Bool(config=True,
         help="whether to log to a file")
 
     clean_logs = Bool(False, config=True,
         help="whether to cleanup old logfiles before starting")
 
     log_url = Unicode('', config=True,
         help="The ZMQ URL of the iplogger to aggregate logging.")
 
+    cluster_id = Unicode('', config=True,
+        help="""String id to add to runtime files, to prevent name collisions when
+        using multiple clusters with a single profile.
+        
+        When set, files will be named like: 'ipcontroller-<cluster_id>-engine.json'
+        """
+    )
+    def _cluster_id_changed(self, name, old, new):
+        self.name = self.__class__.name
+        if new:
+            self.name += '-%s'%new
+    
     def _config_files_default(self):
         return ['ipcontroller_config.py', 'ipengine_config.py', 'ipcluster_config.py']
     
     loop = Instance('zmq.eventloop.ioloop.IOLoop')
     def _loop_default(self):
         from zmq.eventloop.ioloop import IOLoop
         return IOLoop.instance()
 
     aliases = Dict(base_aliases)
     flags = Dict(base_flags)
     
     def initialize(self, argv=None):
         """initialize the app"""
         super(BaseParallelApplication, self).initialize(argv)
         self.to_work_dir()
         self.reinit_logging()
         
     def to_work_dir(self):
         wd = self.work_dir
         if unicode(wd) != os.getcwdu():
             os.chdir(wd)
             self.log.info("Changing to working dir: %s" % wd)
         # This is the working dir by now.
         sys.path.insert(0, '')
 
     def reinit_logging(self):
         # Remove old log files
         log_dir = self.profile_dir.log_dir
         if self.clean_logs:
             for f in os.listdir(log_dir):
                 if re.match(r'%s-\d+\.(log|err|out)'%self.name,f):
                     os.remove(os.path.join(log_dir, f))
         if self.log_to_file:
             # Start logging to the new log file
             log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
             logfile = os.path.join(log_dir, log_filename)
             open_log_file = open(logfile, 'w')
         else:
             open_log_file = None
         if open_log_file is not None:
             self.log.removeHandler(self._log_handler)
             self._log_handler = logging.StreamHandler(open_log_file)
             self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
             self._log_handler.setFormatter(self._log_formatter)
             self.log.addHandler(self._log_handler)
         # do not propagate log messages to root logger
         # ipcluster app will sometimes print duplicate messages during shutdown
         # if this is 1 (default):
         self.log.propagate = False
 
     def write_pid_file(self, overwrite=False):
         """Create a .pid file in the pid_dir with my pid.
 
         This must be called after pre_construct, which sets `self.pid_dir`.
         This raises :exc:`PIDFileError` if the pid file exists already.
         """
         pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
         if os.path.isfile(pid_file):
             pid = self.get_pid_from_file()
             if not overwrite:
                 raise PIDFileError(
                     'The pid file [%s] already exists. \nThis could mean that this '
                     'server is already running with [pid=%s].' % (pid_file, pid)
                 )
         with open(pid_file, 'w') as f:
             self.log.info("Creating pid file: %s" % pid_file)
             f.write(repr(os.getpid())+'\n')
 
     def remove_pid_file(self):
         """Remove the pid file.
 
         This should be called at shutdown by registering a callback with
         :func:`reactor.addSystemEventTrigger`. This needs to return
         ``None``.
         """
         pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
         if os.path.isfile(pid_file):
             try:
                 self.log.info("Removing pid file: %s" % pid_file)
                 os.remove(pid_file)
             except:
                 self.log.warn("Error removing the pid file: %s" % pid_file)
 
     def get_pid_from_file(self):
         """Get the pid from the pid file.
 
         If the  pid file doesn't exist a :exc:`PIDFileError` is raised.
         """
         pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
         if os.path.isfile(pid_file):
             with open(pid_file, 'r') as f:
                 s = f.read().strip()
                 try:
                     pid = int(s)
                 except:
                     raise PIDFileError("invalid pid file: %s (contents: %r)"%(pid_file, s))
                 return pid
         else:
             raise PIDFileError('pid file not found: %s' % pid_file)
     
     def check_pid(self, pid):
         if os.name == 'nt':
             try:
                 import ctypes
                 # returns 0 if no such process (of ours) exists
                 # positive int otherwise
                 p = ctypes.windll.kernel32.OpenProcess(1,0,pid)
             except Exception:
                 self.log.warn(
                     "Could not determine whether pid %i is running via `OpenProcess`. "
                     " Making the likely assumption that it is."%pid
                 )
                 return True
             return bool(p)
         else:
             try:
                 p = Popen(['ps','x'], stdout=PIPE, stderr=PIPE)
                 output,_ = p.communicate()
             except OSError:
                 self.log.warn(
                     "Could not determine whether pid %i is running via `ps x`. "
                     " Making the likely assumption that it is."%pid
                 )
                 return True
             pids = map(int, re.findall(r'^\W*\d+', output, re.MULTILINE))
             return pid in pids

IPython/parallel/apps/ipcontrollerapp.py

@@ -1,441 +1,452 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The IPython controller application.
 
 Authors:
 
 * Brian Granger
 * MinRK
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
 
 import os
 import socket
 import stat
 import sys
 import uuid
 
 from multiprocessing import Process
 
 import zmq
 from zmq.devices import ProcessMonitoredQueue
 from zmq.log.handlers import PUBHandler
 from zmq.utils import jsonapi as json
 
 from IPython.config.application import boolean_flag
 from IPython.core.profiledir import ProfileDir
 
 from IPython.parallel.apps.baseapp import (
     BaseParallelApplication,
     base_aliases,
     base_flags,
 )
 from IPython.utils.importstring import import_item
 from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict
 
 # from IPython.parallel.controller.controller import ControllerFactory
 from IPython.zmq.session import Session
 from IPython.parallel.controller.heartmonitor import HeartMonitor
 from IPython.parallel.controller.hub import HubFactory
 from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
 from IPython.parallel.controller.sqlitedb import SQLiteDB
 
 from IPython.parallel.util import signal_children, split_url, asbytes
 
 # conditional import of MongoDB backend class
 
 try:
     from IPython.parallel.controller.mongodb import MongoDB
 except ImportError:
     maybe_mongo = []
 else:
     maybe_mongo = [MongoDB]
 
 
 #-----------------------------------------------------------------------------
 # Module level variables
 #-----------------------------------------------------------------------------
 
 
 #: The default config file name for this application
 default_config_file_name = u'ipcontroller_config.py'
 
 
 _description = """Start the IPython controller for parallel computing.
 
 The IPython controller provides a gateway between the IPython engines and
 clients. The controller needs to be started before the engines and can be
 configured using command line options or using a cluster directory. Cluster
 directories contain config, log and security files and are usually located in
 your ipython directory and named as "profile_name". See the `profile`
 and `profile-dir` options for details.
 """
 
 _examples = """
 ipcontroller --ip=192.168.0.1 --port=1000  # listen on ip, port for engines
 ipcontroller --scheme=pure  # use the pure zeromq scheduler
 """
 
 
 #-----------------------------------------------------------------------------
 # The main application
 #-----------------------------------------------------------------------------
 flags = {}
 flags.update(base_flags)
 flags.update({
     'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
                     'Use threads instead of processes for the schedulers'),
     'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
                     'use the SQLiteDB backend'),
     'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
                     'use the MongoDB backend'),
     'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
                     'use the in-memory DictDB backend'),
     'reuse' : ({'IPControllerApp' : {'reuse_files' : True}},
                     'reuse existing json connection files')
 })
 
 flags.update(boolean_flag('secure', 'IPControllerApp.secure',
     "Use HMAC digests for authentication of messages.",
     "Don't authenticate messages."
 ))
 aliases = dict(
     secure = 'IPControllerApp.secure',
     ssh = 'IPControllerApp.ssh_server',
     enginessh = 'IPControllerApp.engine_ssh_server',
     location = 'IPControllerApp.location',
 
     ident = 'Session.session',
     user = 'Session.username',
     keyfile = 'Session.keyfile',
 
     url = 'HubFactory.url',
     ip = 'HubFactory.ip',
     transport = 'HubFactory.transport',
     port = 'HubFactory.regport',
 
     ping = 'HeartMonitor.period',
 
     scheme = 'TaskScheduler.scheme_name',
     hwm = 'TaskScheduler.hwm',
 )
 aliases.update(base_aliases)
 
 
 class IPControllerApp(BaseParallelApplication):
 
     name = u'ipcontroller'
     description = _description
     examples = _examples
     config_file_name = Unicode(default_config_file_name)
     classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo
     
     # change default to True
     auto_create = Bool(True, config=True,
         help="""Whether to create profile dir if it doesn't exist.""")
     
     reuse_files = Bool(False, config=True,
         help='Whether to reuse existing json connection files.'
     )
     secure = Bool(True, config=True,
         help='Whether to use HMAC digests for extra message authentication.'
     )
     ssh_server = Unicode(u'', config=True,
         help="""ssh url for clients to use when connecting to the Controller
         processes. It should be of the form: [user@]server[:port]. The
         Controller's listening addresses must be accessible from the ssh server""",
     )
     engine_ssh_server = Unicode(u'', config=True,
         help="""ssh url for engines to use when connecting to the Controller
         processes. It should be of the form: [user@]server[:port]. The
         Controller's listening addresses must be accessible from the ssh server""",
     )
     location = Unicode(u'', config=True,
         help="""The external IP or domain name of the Controller, used for disambiguating
         engine and client connections.""",
     )
     import_statements = List([], config=True,
         help="import statements to be run at startup.  Necessary in some environments"
     )
 
     use_threads = Bool(False, config=True,
         help='Use threads instead of processes for the schedulers',
-        )
+    )
+
+    engine_json_file = Unicode('ipcontroller-engine.json', config=True,
+        help="JSON filename where engine connection info will be stored.")
+    client_json_file = Unicode('ipcontroller-client.json', config=True,
+        help="JSON filename where client connection info will be stored.")
+    
+    def _cluster_id_changed(self, name, old, new):
+        super(IPControllerApp, self)._cluster_id_changed(name, old, new)
+        self.engine_json_file = "%s-engine.json"%self.name
+        self.client_json_file = "%s-client.json"%self.name
+
 
     # internal
     children = List()
     mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')
 
     def _use_threads_changed(self, name, old, new):
         self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
 
     aliases = Dict(aliases)
     flags = Dict(flags)
     
 
     def save_connection_dict(self, fname, cdict):
         """save a connection dict to json file."""
         c = self.config
         url = cdict['url']
         location = cdict['location']
         if not location:
             try:
                 proto,ip,port = split_url(url)
             except AssertionError:
                 pass
             else:
                 try:
                     location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
                 except (socket.gaierror, IndexError):
                     self.log.warn("Could not identify this machine's IP, assuming 127.0.0.1."
                     " You may need to specify '--location=<external_ip_address>' to help"
                     " IPython decide when to connect via loopback.")
                     location = '127.0.0.1'
             cdict['location'] = location
         fname = os.path.join(self.profile_dir.security_dir, fname)
         with open(fname, 'wb') as f:
             f.write(json.dumps(cdict, indent=2))
         os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
     
     def load_config_from_json(self):
         """load config from existing json connector files."""
         c = self.config
         # load from engine config
-        with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-engine.json')) as f:
+        with open(os.path.join(self.profile_dir.security_dir, self.engine_json_file)) as f:
             cfg = json.loads(f.read())
         key = c.Session.key = asbytes(cfg['exec_key'])
         xport,addr = cfg['url'].split('://')
         c.HubFactory.engine_transport = xport
         ip,ports = addr.split(':')
         c.HubFactory.engine_ip = ip
         c.HubFactory.regport = int(ports)
         self.location = cfg['location']
         if not self.engine_ssh_server:
             self.engine_ssh_server = cfg['ssh']
         # load client config
-        with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-client.json')) as f:
+        with open(os.path.join(self.profile_dir.security_dir, self.client_json_file)) as f:
             cfg = json.loads(f.read())
         assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
         xport,addr = cfg['url'].split('://')
         c.HubFactory.client_transport = xport
         ip,ports = addr.split(':')
         c.HubFactory.client_ip = ip
         if not self.ssh_server:
             self.ssh_server = cfg['ssh']
         assert int(ports) == c.HubFactory.regport, "regport mismatch"
     
     def init_hub(self):
         c = self.config
         
         self.do_import_statements()
         reusing = self.reuse_files
         if reusing:
             try:
                 self.load_config_from_json()
             except (AssertionError,IOError):
                 reusing=False
         # check again, because reusing may have failed:
         if reusing:
             pass
         elif self.secure:
             key = str(uuid.uuid4())
             # keyfile = os.path.join(self.profile_dir.security_dir, self.exec_key)
             # with open(keyfile, 'w') as f:
             #     f.write(key)
             # os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
             c.Session.key = asbytes(key)
         else:
             key = c.Session.key = b''
         
         try:
             self.factory = HubFactory(config=c, log=self.log)
             # self.start_logging()
             self.factory.init_hub()
         except:
             self.log.error("Couldn't construct the Controller", exc_info=True)
             self.exit(1)
         
         if not reusing:
             # save to new json config files
             f = self.factory
             cdict = {'exec_key' : key,
                     'ssh' : self.ssh_server,
                     'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
                     'location' : self.location
                     }
-            self.save_connection_dict('ipcontroller-client.json', cdict)
+            self.save_connection_dict(self.client_json_file, cdict)
             edict = cdict
             edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
             edict['ssh'] = self.engine_ssh_server
-            self.save_connection_dict('ipcontroller-engine.json', edict)
+            self.save_connection_dict(self.engine_json_file, edict)
 
     #
     def init_schedulers(self):
         children = self.children
         mq = import_item(str(self.mq_class))
         
         hub = self.factory
         # maybe_inproc = 'inproc://monitor' if self.use_threads else self.monitor_url
         # IOPub relay (in a Process)
         q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub')
         q.bind_in(hub.client_info['iopub'])
         q.bind_out(hub.engine_info['iopub'])
         q.setsockopt_out(zmq.SUBSCRIBE, b'')
         q.connect_mon(hub.monitor_url)
         q.daemon=True
         children.append(q)
 
         # Multiplexer Queue (in a Process)
         q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out')
         q.bind_in(hub.client_info['mux'])
         q.setsockopt_in(zmq.IDENTITY, b'mux')
         q.bind_out(hub.engine_info['mux'])
         q.connect_mon(hub.monitor_url)
         q.daemon=True
         children.append(q)
 
         # Control Queue (in a Process)
         q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol')
         q.bind_in(hub.client_info['control'])
         q.setsockopt_in(zmq.IDENTITY, b'control')
         q.bind_out(hub.engine_info['control'])
         q.connect_mon(hub.monitor_url)
         q.daemon=True
         children.append(q)
         try:
             scheme = self.config.TaskScheduler.scheme_name
         except AttributeError:
             scheme = TaskScheduler.scheme_name.get_default_value()
         # Task Queue (in a Process)
         if scheme == 'pure':
             self.log.warn("task::using pure XREQ Task scheduler")
             q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask')
             # q.setsockopt_out(zmq.HWM, hub.hwm)
             q.bind_in(hub.client_info['task'][1])
             q.setsockopt_in(zmq.IDENTITY, b'task')
             q.bind_out(hub.engine_info['task'])
             q.connect_mon(hub.monitor_url)
             q.daemon=True
             children.append(q)
         elif scheme == 'none':
             self.log.warn("task::using no Task scheduler")
 
         else:
             self.log.info("task::using Python %s Task scheduler"%scheme)
             sargs = (hub.client_info['task'][1], hub.engine_info['task'],
                                 hub.monitor_url, hub.client_info['notification'])
             kwargs = dict(logname='scheduler', loglevel=self.log_level,
                             log_url = self.log_url, config=dict(self.config))
             if 'Process' in self.mq_class:
                 # run the Python scheduler in a Process
                 q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
                 q.daemon=True
                 children.append(q)
             else:
                 # single-threaded Controller
                 kwargs['in_thread'] = True
                 launch_scheduler(*sargs, **kwargs)
 
     
     def save_urls(self):
         """save the registration urls to files."""
         c = self.config
         
         sec_dir = self.profile_dir.security_dir
         cf = self.factory
         
         with open(os.path.join(sec_dir, 'ipcontroller-engine.url'), 'w') as f:
             f.write("%s://%s:%s"%(cf.engine_transport, cf.engine_ip, cf.regport))
         
         with open(os.path.join(sec_dir, 'ipcontroller-client.url'), 'w') as f:
             f.write("%s://%s:%s"%(cf.client_transport, cf.client_ip, cf.regport))
         
     
     def do_import_statements(self):
         statements = self.import_statements
         for s in statements:
             try:
                 self.log.msg("Executing statement: '%s'" % s)
                 exec s in globals(), locals()
             except:
                 self.log.msg("Error running statement: %s" % s)
 
     def forward_logging(self):
         if self.log_url:
             self.log.info("Forwarding logging to %s"%self.log_url)
             context = zmq.Context.instance()
             lsock = context.socket(zmq.PUB)
             lsock.connect(self.log_url)
             handler = PUBHandler(lsock)
             self.log.removeHandler(self._log_handler)
             handler.root_topic = 'controller'
             handler.setLevel(self.log_level)
             self.log.addHandler(handler)
             self._log_handler = handler
     # #
     
     def initialize(self, argv=None):
         super(IPControllerApp, self).initialize(argv)
         self.forward_logging()
         self.init_hub()
         self.init_schedulers()
     
     def start(self):
         # Start the subprocesses:
         self.factory.start()
         child_procs = []
         for child in self.children:
             child.start()
             if isinstance(child, ProcessMonitoredQueue):
                 child_procs.append(child.launcher)
             elif isinstance(child, Process):
                 child_procs.append(child)
         if child_procs:
             signal_children(child_procs)
 
         self.write_pid_file(overwrite=True)
 
         try:
             self.factory.loop.start()
         except KeyboardInterrupt:
             self.log.critical("Interrupted, Exiting...\n")
             
 
 
 def launch_new_instance():
     """Create and run the IPython controller"""
     if sys.platform == 'win32':
         # make sure we don't get called from a multiprocessing subprocess
         # this can result in infinite Controllers being started on Windows
         # which doesn't have a proper fork, so multiprocessing is wonky
         
         # this only comes up when IPython has been installed using vanilla
         # setuptools, and *not* distribute.
         import multiprocessing
         p = multiprocessing.current_process()
         # the main process has name 'MainProcess'
         # subprocesses will have names like 'Process-1'
         if p.name != 'MainProcess':
             # we are a subprocess, don't start another Controller!
             return
     app = IPControllerApp.instance()
     app.initialize()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()

IPython/parallel/apps/ipengineapp.py

@@ -1,336 +1,344 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The IPython engine application
 
 Authors:
 
 * Brian Granger
 * MinRK
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import json
 import os
 import sys
 import time
 
 import zmq
 from zmq.eventloop import ioloop
 
 from IPython.core.profiledir import ProfileDir
 from IPython.parallel.apps.baseapp import (
     BaseParallelApplication,
     base_aliases,
     base_flags,
 )
 from IPython.zmq.log import EnginePUBHandler
 
 from IPython.config.configurable import Configurable
 from IPython.zmq.session import Session
 from IPython.parallel.engine.engine import EngineFactory
 from IPython.parallel.engine.streamkernel import Kernel
 from IPython.parallel.util import disambiguate_url, asbytes
 
 from IPython.utils.importstring import import_item
 from IPython.utils.traitlets import Bool, Unicode, Dict, List, Float
 
 
 #-----------------------------------------------------------------------------
 # Module level variables
 #-----------------------------------------------------------------------------
 
 #: The default config file name for this application
 default_config_file_name = u'ipengine_config.py'
 
 _description = """Start an IPython engine for parallel computing.
 
 IPython engines run in parallel and perform computations on behalf of a client
 and controller. A controller needs to be started before the engines. The
 engine can be configured using command line options or using a cluster
 directory. Cluster directories contain config, log and security files and are
 usually located in your ipython directory and named as "profile_name".
 See the `profile` and `profile-dir` options for details.
 """
 
 _examples = """
 ipengine --ip=192.168.0.1 --port=1000     # connect to hub at ip and port
 ipengine --log-to-file --log-level=DEBUG  # log to a file with DEBUG verbosity
 """
 
 #-----------------------------------------------------------------------------
 # MPI configuration
 #-----------------------------------------------------------------------------
 
 mpi4py_init = """from mpi4py import MPI as mpi
 mpi.size = mpi.COMM_WORLD.Get_size()
 mpi.rank = mpi.COMM_WORLD.Get_rank()
 """
 
 
 pytrilinos_init = """from PyTrilinos import Epetra
 class SimpleStruct:
 pass
 mpi = SimpleStruct()
 mpi.rank = 0
 mpi.size = 0
 """
 
 class MPI(Configurable):
     """Configurable for MPI initialization"""
     use = Unicode('', config=True,
         help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).'
         )
 
     def _on_use_changed(self, old, new):
         # load default init script if it's not set
         if not self.init_script:
             self.init_script = self.default_inits.get(new, '')
 
     init_script = Unicode('', config=True,
         help="Initialization code for MPI")
 
     default_inits = Dict({'mpi4py' : mpi4py_init, 'pytrilinos':pytrilinos_init},
         config=True)
 
 
 #-----------------------------------------------------------------------------
 # Main application
 #-----------------------------------------------------------------------------
 aliases = dict(
     file = 'IPEngineApp.url_file',
     c = 'IPEngineApp.startup_command',
     s = 'IPEngineApp.startup_script',
 
     ident = 'Session.session',
     user = 'Session.username',
     keyfile = 'Session.keyfile',
 
     url = 'EngineFactory.url',
     ssh = 'EngineFactory.sshserver',
     sshkey = 'EngineFactory.sshkey',
     ip = 'EngineFactory.ip',
     transport = 'EngineFactory.transport',
     port = 'EngineFactory.regport',
     location = 'EngineFactory.location',
 
     timeout = 'EngineFactory.timeout',
 
     mpi = 'MPI.use',
 
 )
 aliases.update(base_aliases)
 
 
 class IPEngineApp(BaseParallelApplication):
 
-    name = Unicode(u'ipengine')
-    description = Unicode(_description)
+    name = 'ipengine'
+    description = _description
     examples = _examples
     config_file_name = Unicode(default_config_file_name)
     classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI])
 
     startup_script = Unicode(u'', config=True,
         help='specify a script to be run at startup')
     startup_command = Unicode('', config=True,
             help='specify a command to be run at startup')
 
     url_file = Unicode(u'', config=True,
         help="""The full location of the file containing the connection information for
         the controller. If this is not given, the file must be in the
         security directory of the cluster directory.  This location is
         resolved using the `profile` or `profile_dir` options.""",
         )
     wait_for_url_file = Float(5, config=True,
         help="""The maximum number of seconds to wait for url_file to exist.
         This is useful for batch-systems and shared-filesystems where the
         controller and engine are started at the same time and it
         may take a moment for the controller to write the connector files.""")
 
-    url_file_name = Unicode(u'ipcontroller-engine.json')
+    url_file_name = Unicode(u'ipcontroller-engine.json', config=True)
+
+    def _cluster_id_changed(self, name, old, new):
+        if new:
+            base = 'ipcontroller-%s'%new
+        else:
+            base = 'ipcontroller'
+        self.url_file_name = "%s-engine.json"%base
+
     log_url = Unicode('', config=True,
         help="""The URL for the iploggerapp instance, for forwarding
         logging to a central location.""")
 
     aliases = Dict(aliases)
 
     # def find_key_file(self):
     #     """Set the key file.
     # 
     #     Here we don't try to actually see if it exists for is valid as that
     #     is hadled by the connection logic.
     #     """
     #     config = self.master_config
     #     # Find the actual controller key file
     #     if not config.Global.key_file:
     #         try_this = os.path.join(
     #             config.Global.profile_dir, 
     #             config.Global.security_dir,
     #             config.Global.key_file_name
     #         )
     #         config.Global.key_file = try_this
         
     def find_url_file(self):
         """Set the url file.
 
         Here we don't try to actually see if it exists for is valid as that
         is hadled by the connection logic.
         """
         config = self.config
         # Find the actual controller key file
         if not self.url_file:
             self.url_file = os.path.join(
                 self.profile_dir.security_dir,
                 self.url_file_name
             )
     
     def load_connector_file(self):
         """load config from a JSON connector file,
         at a *lower* priority than command-line/config files.
         """
         
         self.log.info("Loading url_file %r"%self.url_file)
         config = self.config
         
         with open(self.url_file) as f:
             d = json.loads(f.read())
         
         try:
             config.Session.key
         except AttributeError:
             if d['exec_key']:
                 config.Session.key = asbytes(d['exec_key'])
                 
         try:
             config.EngineFactory.location
         except AttributeError:
             config.EngineFactory.location = d['location']
         
         d['url'] = disambiguate_url(d['url'], config.EngineFactory.location)
         try:
             config.EngineFactory.url
         except AttributeError:
             config.EngineFactory.url = d['url']
         
         try:
             config.EngineFactory.sshserver
         except AttributeError:
             config.EngineFactory.sshserver = d['ssh']
         
     def init_engine(self):
         # This is the working dir by now.
         sys.path.insert(0, '')
         config = self.config
         # print config
         self.find_url_file()
         
         # was the url manually specified?
         keys = set(self.config.EngineFactory.keys())
         keys = keys.union(set(self.config.RegistrationFactory.keys()))
         
         if keys.intersection(set(['ip', 'url', 'port'])):
             # Connection info was specified, don't wait for the file
             url_specified = True
             self.wait_for_url_file = 0
         else:
             url_specified = False
 
         if self.wait_for_url_file and not os.path.exists(self.url_file):
             self.log.warn("url_file %r not found"%self.url_file)
             self.log.warn("Waiting up to %.1f seconds for it to arrive."%self.wait_for_url_file)
             tic = time.time()
             while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file):
                 # wait for url_file to exist, for up to 10 seconds
                 time.sleep(0.1)
             
         if os.path.exists(self.url_file):
             self.load_connector_file()
         elif not url_specified:
             self.log.critical("Fatal: url file never arrived: %s"%self.url_file)
             self.exit(1)
         
         
         try:
             exec_lines = config.Kernel.exec_lines
         except AttributeError:
             config.Kernel.exec_lines = []
             exec_lines = config.Kernel.exec_lines
         
         if self.startup_script:
             enc = sys.getfilesystemencoding() or 'utf8'
             cmd="execfile(%r)"%self.startup_script.encode(enc)
             exec_lines.append(cmd)
         if self.startup_command:
             exec_lines.append(self.startup_command)
 
         # Create the underlying shell class and Engine
         # shell_class = import_item(self.master_config.Global.shell_class)
         # print self.config
         try:
             self.engine = EngineFactory(config=config, log=self.log)
         except:
             self.log.error("Couldn't start the Engine", exc_info=True)
             self.exit(1)
     
     def forward_logging(self):
         if self.log_url:
             self.log.info("Forwarding logging to %s"%self.log_url)
             context = self.engine.context
             lsock = context.socket(zmq.PUB)
             lsock.connect(self.log_url)
             self.log.removeHandler(self._log_handler)
             handler = EnginePUBHandler(self.engine, lsock)
             handler.setLevel(self.log_level)
             self.log.addHandler(handler)
             self._log_handler = handler
     
     def init_mpi(self):
         global mpi
         self.mpi = MPI(config=self.config)
 
         mpi_import_statement = self.mpi.init_script
         if mpi_import_statement:
             try:
                 self.log.info("Initializing MPI:")
                 self.log.info(mpi_import_statement)
                 exec mpi_import_statement in globals()
             except:
                 mpi = None
         else:
             mpi = None
 
     def initialize(self, argv=None):
         super(IPEngineApp, self).initialize(argv)
         self.init_mpi()
         self.init_engine()
         self.forward_logging()
     
     def start(self):
         self.engine.start()
         try:
             self.engine.loop.start()
         except KeyboardInterrupt:
             self.log.critical("Engine Interrupted, shutting down...\n")
 
 
 def launch_new_instance():
     """Create and run the IPython engine"""
     app = IPEngineApp.instance()
     app.initialize()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()