add NoDB for non-recording Hub

When used, this disables database-based actions in the Hub.

Useful for minimizing Hub memory consumption.

IPython/parallel/apps/ipcontrollerapp.py

@@ -1,452 +1,459 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The IPython controller application.
 
 Authors:
 
 * Brian Granger
 * MinRK
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
 
 import json
 import os
 import socket
 import stat
 import sys
 
 from multiprocessing import Process
 
 import zmq
 from zmq.devices import ProcessMonitoredQueue
 from zmq.log.handlers import PUBHandler
 
 from IPython.core.profiledir import ProfileDir
 
 from IPython.parallel.apps.baseapp import (
     BaseParallelApplication,
     base_aliases,
     base_flags,
     catch_config_error,
 )
 from IPython.utils.importstring import import_item
 from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict, TraitError
 
 from IPython.zmq.session import (
     Session, session_aliases, session_flags, default_secure
 )
 
 from IPython.parallel.controller.heartmonitor import HeartMonitor
 from IPython.parallel.controller.hub import HubFactory
 from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
 from IPython.parallel.controller.sqlitedb import SQLiteDB
 
 from IPython.parallel.util import signal_children, split_url, disambiguate_url
 
 # conditional import of MongoDB backend class
 
 try:
     from IPython.parallel.controller.mongodb import MongoDB
 except ImportError:
     maybe_mongo = []
 else:
     maybe_mongo = [MongoDB]
 
 
 #-----------------------------------------------------------------------------
 # Module level variables
 #-----------------------------------------------------------------------------
 
 
 #: The default config file name for this application
 default_config_file_name = u'ipcontroller_config.py'
 
 
 _description = """Start the IPython controller for parallel computing.
 
 The IPython controller provides a gateway between the IPython engines and
 clients. The controller needs to be started before the engines and can be
 configured using command line options or using a cluster directory. Cluster
 directories contain config, log and security files and are usually located in
 your ipython directory and named as "profile_name". See the `profile`
 and `profile-dir` options for details.
 """
 
 _examples = """
 ipcontroller --ip=192.168.0.1 --port=1000  # listen on ip, port for engines
 ipcontroller --scheme=pure  # use the pure zeromq scheduler
 """
 
 
 #-----------------------------------------------------------------------------
 # The main application
 #-----------------------------------------------------------------------------
 flags = {}
 flags.update(base_flags)
 flags.update({
     'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
                     'Use threads instead of processes for the schedulers'),
     'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
                     'use the SQLiteDB backend'),
     'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
                     'use the MongoDB backend'),
     'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
                     'use the in-memory DictDB backend'),
+    'nodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.NoDB'}},
+                    """use dummy DB backend, which doesn't store any information.
+                    
+                    This can be used to prevent growth of the memory footprint of the Hub
+                    in cases where its record-keeping is not required.  Requesting results
+                    of tasks submitted by other clients, db_queries, and task resubmission
+                    will not be available."""),
     'reuse' : ({'IPControllerApp' : {'reuse_files' : True}},
                     'reuse existing json connection files')
 })
 
 flags.update(session_flags)
 
 aliases = dict(
     ssh = 'IPControllerApp.ssh_server',
     enginessh = 'IPControllerApp.engine_ssh_server',
     location = 'IPControllerApp.location',
 
     url = 'HubFactory.url',
     ip = 'HubFactory.ip',
     transport = 'HubFactory.transport',
     port = 'HubFactory.regport',
 
     ping = 'HeartMonitor.period',
 
     scheme = 'TaskScheduler.scheme_name',
     hwm = 'TaskScheduler.hwm',
 )
 aliases.update(base_aliases)
 aliases.update(session_aliases)
 
 
 class IPControllerApp(BaseParallelApplication):
 
     name = u'ipcontroller'
     description = _description
     examples = _examples
     config_file_name = Unicode(default_config_file_name)
     classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo
     
     # change default to True
     auto_create = Bool(True, config=True,
         help="""Whether to create profile dir if it doesn't exist.""")
     
     reuse_files = Bool(False, config=True,
         help='Whether to reuse existing json connection files.'
     )
     ssh_server = Unicode(u'', config=True,
         help="""ssh url for clients to use when connecting to the Controller
         processes. It should be of the form: [user@]server[:port]. The
         Controller's listening addresses must be accessible from the ssh server""",
     )
     engine_ssh_server = Unicode(u'', config=True,
         help="""ssh url for engines to use when connecting to the Controller
         processes. It should be of the form: [user@]server[:port]. The
         Controller's listening addresses must be accessible from the ssh server""",
     )
     location = Unicode(u'', config=True,
         help="""The external IP or domain name of the Controller, used for disambiguating
         engine and client connections.""",
     )
     import_statements = List([], config=True,
         help="import statements to be run at startup.  Necessary in some environments"
     )
 
     use_threads = Bool(False, config=True,
         help='Use threads instead of processes for the schedulers',
     )
 
     engine_json_file = Unicode('ipcontroller-engine.json', config=True,
         help="JSON filename where engine connection info will be stored.")
     client_json_file = Unicode('ipcontroller-client.json', config=True,
         help="JSON filename where client connection info will be stored.")
     
     def _cluster_id_changed(self, name, old, new):
         super(IPControllerApp, self)._cluster_id_changed(name, old, new)
         self.engine_json_file = "%s-engine.json" % self.name
         self.client_json_file = "%s-client.json" % self.name
 
 
     # internal
     children = List()
     mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')
 
     def _use_threads_changed(self, name, old, new):
         self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
 
     aliases = Dict(aliases)
     flags = Dict(flags)
     
 
     def save_connection_dict(self, fname, cdict):
         """save a connection dict to json file."""
         c = self.config
         url = cdict['url']
         location = cdict['location']
         if not location:
             try:
                 proto,ip,port = split_url(url)
             except AssertionError:
                 pass
             else:
                 try:
                     location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
                 except (socket.gaierror, IndexError):
                     self.log.warn("Could not identify this machine's IP, assuming 127.0.0.1."
                     " You may need to specify '--location=<external_ip_address>' to help"
                     " IPython decide when to connect via loopback.")
                     location = '127.0.0.1'
             cdict['location'] = location
         fname = os.path.join(self.profile_dir.security_dir, fname)
         self.log.info("writing connection info to %s", fname)
         with open(fname, 'w') as f:
             f.write(json.dumps(cdict, indent=2))
         os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
     
     def load_config_from_json(self):
         """load config from existing json connector files."""
         c = self.config
         self.log.debug("loading config from JSON")
         # load from engine config
         fname = os.path.join(self.profile_dir.security_dir, self.engine_json_file)
         self.log.info("loading connection info from %s", fname)
         with open(fname) as f:
             cfg = json.loads(f.read())
         key = cfg['exec_key']
         # json gives unicode, Session.key wants bytes
         c.Session.key = key.encode('ascii')
         xport,addr = cfg['url'].split('://')
         c.HubFactory.engine_transport = xport
         ip,ports = addr.split(':')
         c.HubFactory.engine_ip = ip
         c.HubFactory.regport = int(ports)
         self.location = cfg['location']
         if not self.engine_ssh_server:
             self.engine_ssh_server = cfg['ssh']
         # load client config
         fname = os.path.join(self.profile_dir.security_dir, self.client_json_file)
         self.log.info("loading connection info from %s", fname)
         with open(fname) as f:
             cfg = json.loads(f.read())
         assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
         xport,addr = cfg['url'].split('://')
         c.HubFactory.client_transport = xport
         ip,ports = addr.split(':')
         c.HubFactory.client_ip = ip
         if not self.ssh_server:
             self.ssh_server = cfg['ssh']
         assert int(ports) == c.HubFactory.regport, "regport mismatch"
     
     def load_secondary_config(self):
         """secondary config, loading from JSON and setting defaults"""
         if self.reuse_files:
             try:
                 self.load_config_from_json()
             except (AssertionError,IOError) as e:
                 self.log.error("Could not load config from JSON: %s" % e)
                 self.reuse_files=False
         # switch Session.key default to secure
         default_secure(self.config)
         self.log.debug("Config changed")
         self.log.debug(repr(self.config))
         
     def init_hub(self):
         c = self.config
         
         self.do_import_statements()
         
         try:
             self.factory = HubFactory(config=c, log=self.log)
             # self.start_logging()
             self.factory.init_hub()
         except TraitError:
             raise
         except Exception:
             self.log.error("Couldn't construct the Controller", exc_info=True)
             self.exit(1)
         
         if not self.reuse_files:
             # save to new json config files
             f = self.factory
             cdict = {'exec_key' : f.session.key.decode('ascii'),
                     'ssh' : self.ssh_server,
                     'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
                     'location' : self.location
                     }
             self.save_connection_dict(self.client_json_file, cdict)
             edict = cdict
             edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
             edict['ssh'] = self.engine_ssh_server
             self.save_connection_dict(self.engine_json_file, edict)
 
     #
     def init_schedulers(self):
         children = self.children
         mq = import_item(str(self.mq_class))
         
         hub = self.factory
         # disambiguate url, in case of *
         monitor_url = disambiguate_url(hub.monitor_url)
         # maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url
         # IOPub relay (in a Process)
         q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub')
         q.bind_in(hub.client_info['iopub'])
         q.bind_out(hub.engine_info['iopub'])
         q.setsockopt_out(zmq.SUBSCRIBE, b'')
         q.connect_mon(monitor_url)
         q.daemon=True
         children.append(q)
 
         # Multiplexer Queue (in a Process)
         q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out')
         q.bind_in(hub.client_info['mux'])
         q.setsockopt_in(zmq.IDENTITY, b'mux')
         q.bind_out(hub.engine_info['mux'])
         q.connect_mon(monitor_url)
         q.daemon=True
         children.append(q)
 
         # Control Queue (in a Process)
         q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol')
         q.bind_in(hub.client_info['control'])
         q.setsockopt_in(zmq.IDENTITY, b'control')
         q.bind_out(hub.engine_info['control'])
         q.connect_mon(monitor_url)
         q.daemon=True
         children.append(q)
         try:
             scheme = self.config.TaskScheduler.scheme_name
         except AttributeError:
             scheme = TaskScheduler.scheme_name.get_default_value()
         # Task Queue (in a Process)
         if scheme == 'pure':
             self.log.warn("task::using pure XREQ Task scheduler")
             q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask')
             # q.setsockopt_out(zmq.HWM, hub.hwm)
             q.bind_in(hub.client_info['task'][1])
             q.setsockopt_in(zmq.IDENTITY, b'task')
             q.bind_out(hub.engine_info['task'])
             q.connect_mon(monitor_url)
             q.daemon=True
             children.append(q)
         elif scheme == 'none':
             self.log.warn("task::using no Task scheduler")
 
         else:
             self.log.info("task::using Python %s Task scheduler"%scheme)
             sargs = (hub.client_info['task'][1], hub.engine_info['task'],
                                 monitor_url, disambiguate_url(hub.client_info['notification']))
             kwargs = dict(logname='scheduler', loglevel=self.log_level,
                             log_url = self.log_url, config=dict(self.config))
             if 'Process' in self.mq_class:
                 # run the Python scheduler in a Process
                 q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
                 q.daemon=True
                 children.append(q)
             else:
                 # single-threaded Controller
                 kwargs['in_thread'] = True
                 launch_scheduler(*sargs, **kwargs)
 
     
     def save_urls(self):
         """save the registration urls to files."""
         c = self.config
         
         sec_dir = self.profile_dir.security_dir
         cf = self.factory
         
         with open(os.path.join(sec_dir, 'ipcontroller-engine.url'), 'w') as f:
             f.write("%s://%s:%s"%(cf.engine_transport, cf.engine_ip, cf.regport))
         
         with open(os.path.join(sec_dir, 'ipcontroller-client.url'), 'w') as f:
             f.write("%s://%s:%s"%(cf.client_transport, cf.client_ip, cf.regport))
         
     
     def do_import_statements(self):
         statements = self.import_statements
         for s in statements:
             try:
                 self.log.msg("Executing statement: '%s'" % s)
                 exec s in globals(), locals()
             except:
                 self.log.msg("Error running statement: %s" % s)
 
     def forward_logging(self):
         if self.log_url:
             self.log.info("Forwarding logging to %s"%self.log_url)
             context = zmq.Context.instance()
             lsock = context.socket(zmq.PUB)
             lsock.connect(self.log_url)
             handler = PUBHandler(lsock)
             self.log.removeHandler(self._log_handler)
             handler.root_topic = 'controller'
             handler.setLevel(self.log_level)
             self.log.addHandler(handler)
             self._log_handler = handler
     
     @catch_config_error
     def initialize(self, argv=None):
         super(IPControllerApp, self).initialize(argv)
         self.forward_logging()
         self.load_secondary_config()
         self.init_hub()
         self.init_schedulers()
     
     def start(self):
         # Start the subprocesses:
         self.factory.start()
         child_procs = []
         for child in self.children:
             child.start()
             if isinstance(child, ProcessMonitoredQueue):
                 child_procs.append(child.launcher)
             elif isinstance(child, Process):
                 child_procs.append(child)
         if child_procs:
             signal_children(child_procs)
 
         self.write_pid_file(overwrite=True)
 
         try:
             self.factory.loop.start()
         except KeyboardInterrupt:
             self.log.critical("Interrupted, Exiting...\n")
             
 
 
 def launch_new_instance():
     """Create and run the IPython controller"""
     if sys.platform == 'win32':
         # make sure we don't get called from a multiprocessing subprocess
         # this can result in infinite Controllers being started on Windows
         # which doesn't have a proper fork, so multiprocessing is wonky
         
         # this only comes up when IPython has been installed using vanilla
         # setuptools, and *not* distribute.
         import multiprocessing
         p = multiprocessing.current_process()
         # the main process has name 'MainProcess'
         # subprocesses will have names like 'Process-1'
         if p.name != 'MainProcess':
             # we are a subprocess, don't start another Controller!
             return
     app = IPControllerApp.instance()
     app.initialize()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()

IPython/parallel/controller/dictdb.py

@@ -1,185 +1,216 b''
 """A Task logger that presents our DB interface,
 but exists entirely in memory and implemented with dicts.
 
 Authors:
 
 * Min RK
 
 
 TaskRecords are dicts of the form:
 {
     'msg_id' : str(uuid),
     'client_uuid' : str(uuid),
     'engine_uuid' : str(uuid) or None,
     'header' : dict(header),
     'content': dict(content),
     'buffers': list(buffers),
     'submitted': datetime,
     'started': datetime or None,
     'completed': datetime or None,
     'resubmitted': datetime or None,
     'result_header' : dict(header) or None,
     'result_content' : dict(content) or None,
     'result_buffers' : list(buffers) or None,
 }
 With this info, many of the special categories of tasks can be defined by query:
 
 pending:  completed is None
 client's outstanding: client_uuid = uuid && completed is None
 MIA: arrived is None (and completed is None)
 etc.
 
 EngineRecords are dicts of the form:
 {
     'eid' : int(id),
     'uuid': str(uuid)
 }
 This may be extended, but is currently.
 
 We support a subset of mongodb operators:
     $lt,$gt,$lte,$gte,$ne,$in,$nin,$all,$mod,$exists
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 
 from datetime import datetime
 
 from IPython.config.configurable import LoggingConfigurable
 
 from IPython.utils.traitlets import Dict, Unicode, Instance
 
 filters = {
  '$lt' : lambda a,b: a < b,
  '$gt' : lambda a,b: b > a,
  '$eq' : lambda a,b: a == b,
  '$ne' : lambda a,b: a != b,
  '$lte': lambda a,b: a <= b,
  '$gte': lambda a,b: a >= b,
  '$in' : lambda a,b: a in b,
  '$nin': lambda a,b: a not in b,
  '$all': lambda a,b: all([ a in bb for bb in b ]),
  '$mod': lambda a,b: a%b[0] == b[1],
  '$exists' : lambda a,b: (b and a is not None) or (a is None and not b)
 }
 
 
 class CompositeFilter(object):
     """Composite filter for matching multiple properties."""
 
     def __init__(self, dikt):
         self.tests = []
         self.values = []
         for key, value in dikt.iteritems():
             self.tests.append(filters[key])
             self.values.append(value)
 
     def __call__(self, value):
         for test,check in zip(self.tests, self.values):
             if not test(value, check):
                 return False
         return True
 
 class BaseDB(LoggingConfigurable):
     """Empty Parent class so traitlets work on DB."""
     # base configurable traits:
     session = Unicode("")
 
 class DictDB(BaseDB):
     """Basic in-memory dict-based object for saving Task Records.
 
     This is the first object to present the DB interface
     for logging tasks out of memory.
 
     The interface is based on MongoDB, so adding a MongoDB
     backend should be straightforward.
     """
 
     _records = Dict()
 
     def _match_one(self, rec, tests):
         """Check if a specific record matches tests."""
         for key,test in tests.iteritems():
             if not test(rec.get(key, None)):
                 return False
         return True
 
     def _match(self, check):
         """Find all the matches for a check dict."""
         matches = []
         tests = {}
         for k,v in check.iteritems():
             if isinstance(v, dict):
                 tests[k] = CompositeFilter(v)
             else:
                 tests[k] = lambda o: o==v
 
         for rec in self._records.itervalues():
             if self._match_one(rec, tests):
                 matches.append(rec)
         return matches
 
     def _extract_subdict(self, rec, keys):
         """extract subdict of keys"""
         d = {}
         d['msg_id'] = rec['msg_id']
         for key in keys:
             d[key] = rec[key]
         return d
 
     def add_record(self, msg_id, rec):
         """Add a new Task Record, by msg_id."""
         if self._records.has_key(msg_id):
             raise KeyError("Already have msg_id %r"%(msg_id))
         self._records[msg_id] = rec
 
     def get_record(self, msg_id):
         """Get a specific Task Record, by msg_id."""
         if not self._records.has_key(msg_id):
             raise KeyError("No such msg_id %r"%(msg_id))
         return self._records[msg_id]
 
     def update_record(self, msg_id, rec):
         """Update the data in an existing record."""
         self._records[msg_id].update(rec)
 
     def drop_matching_records(self, check):
         """Remove a record from the DB."""
         matches = self._match(check)
         for m in matches:
             del self._records[m['msg_id']]
 
     def drop_record(self, msg_id):
         """Remove a record from the DB."""
         del self._records[msg_id]
 
 
     def find_records(self, check, keys=None):
         """Find records matching a query dict, optionally extracting subset of keys.
 
         Returns dict keyed by msg_id of matching records.
 
         Parameters
         ----------
 
         check: dict
             mongodb-style query argument
         keys: list of strs [optional]
             if specified, the subset of keys to extract.  msg_id will *always* be
             included.
         """
         matches = self._match(check)
         if keys:
             return [ self._extract_subdict(rec, keys) for rec in matches ]
         else:
             return matches
 
 
     def get_history(self):
         """get all msg_ids, ordered by time submitted."""
         msg_ids = self._records.keys()
         return sorted(msg_ids, key=lambda m: self._records[m]['submitted'])
+
+class NoDB(DictDB):
+    """A blackhole db backend that actually stores no information.
+    
+    Provides the full DB interface, but raises KeyErrors on any
+    method that tries to access the records.  This can be used to
+    minimize the memory footprint of the Hub when its record-keeping
+    functionality is not required.
+    """
+    
+    def add_record(self, msg_id, record):
+        pass
+    
+    def get_record(self, msg_id):
+        raise KeyError("NoDB does not support record access")
+    
+    def update_record(self, msg_id, record):
+        pass
+    
+    def drop_matching_records(self, check):
+        pass
+    
+    def drop_record(self, msg_id):
+        pass
+    
+    def find_records(self, check, keys=None):
+        raise KeyError("NoDB does not store information")
+    
+    def get_history(self):
+        raise KeyError("NoDB does not store information")
+

docs/source/parallel/parallel_db.txt

@@ -1,114 +1,137 b''
 .. _parallel_db:
 
 =======================
 IPython's Task Database
 =======================
 
 The IPython Hub stores all task requests and results in a database. Currently supported backends
 are: MongoDB, SQLite (the default), and an in-memory DictDB.  The most common use case for
 this is clients requesting results for tasks they did not submit, via:
 
 .. sourcecode:: ipython
 
     In [1]: rc.get_result(task_id)
 
 However, since we have this DB backend, we provide a direct query method in the :class:`client`
 for users who want deeper introspection into their task history. The :meth:`db_query` method of
 the Client is modeled after MongoDB queries, so if you have used MongoDB it should look
 familiar.  In fact, when the MongoDB backend is in use, the query is relayed directly.  However,
 when using other backends, the interface is emulated and only a subset of queries is possible.
 
 .. seealso::
 
     MongoDB query docs: http://www.mongodb.org/display/DOCS/Querying
 
 :meth:`Client.db_query` takes a dictionary query object, with keys from the TaskRecord key list,
 and values of either exact values to test, or MongoDB queries, which are dicts of The form:
 ``{'operator' : 'argument(s)'}``. There is also an optional `keys` argument, that specifies
 which subset of keys should be retrieved. The default is to retrieve all keys excluding the
 request and result buffers. :meth:`db_query` returns a list of TaskRecord dicts. Also like
 MongoDB, the `msg_id` key will always be included, whether requested or not.
 
 TaskRecord keys:
 
 =============== =============== =============
 Key             Type            Description
 =============== =============== =============
 msg_id          uuid(bytes)     The msg ID
 header          dict            The request header
 content         dict            The request content (likely empty)
 buffers         list(bytes)     buffers containing serialized request objects
 submitted       datetime        timestamp for time of submission (set by client)
 client_uuid     uuid(bytes)     IDENT of client's socket
 engine_uuid     uuid(bytes)     IDENT of engine's socket
 started         datetime        time task began execution on engine
 completed       datetime        time task finished execution (success or failure) on engine
 resubmitted     datetime        time of resubmission (if applicable)
 result_header   dict            header for result
 result_content  dict            content for result
 result_buffers  list(bytes)     buffers containing serialized request objects
 queue           bytes           The name of the queue for the task ('mux' or 'task')
 pyin            <unused>        Python input (unused)
 pyout           <unused>        Python output (unused)
 pyerr           <unused>        Python traceback (unused)
 stdout          str             Stream of stdout data
 stderr          str             Stream of stderr data
 
 =============== =============== =============
 
 MongoDB operators we emulate on all backends:
 
 ==========  =================
 Operator    Python equivalent
 ==========  =================
   '$in'       in
   '$nin'      not in
   '$eq'       ==
   '$ne'       !=
   '$ge'       >
   '$gte'      >=
   '$le'       <
   '$lte'      <=
 ==========  =================
 
 
 The DB Query is useful for two primary cases:
 
 1. deep polling of task status or metadata
 2. selecting a subset of tasks, on which to perform a later operation (e.g. wait on result, purge records, resubmit,...)
 
 Example Queries
 ===============
 
 
 To get all msg_ids that are not completed, only retrieving their ID and start time:
 
 .. sourcecode:: ipython
 
     In [1]: incomplete = rc.db_query({'complete' : None}, keys=['msg_id', 'started'])
 
 All jobs started in the last hour by me:
 
 .. sourcecode:: ipython
 
     In [1]: from datetime import datetime, timedelta
 
     In [2]: hourago = datetime.now() - timedelta(1./24)
 
     In [3]: recent = rc.db_query({'started' : {'$gte' : hourago },
                                     'client_uuid' : rc.session.session})
 
 All jobs started more than an hour ago, by clients *other than me*:
 
 .. sourcecode:: ipython
 
     In [3]: recent = rc.db_query({'started' : {'$le' : hourago },
                                     'client_uuid' : {'$ne' : rc.session.session}})
 
 Result headers for all jobs on engine 3 or 4:
 
 .. sourcecode:: ipython
 
     In [1]: uuids = map(rc._engines.get, (3,4))
 
     In [2]: hist34 = rc.db_query({'engine_uuid' : {'$in' : uuids }, keys='result_header')
+
+
+Cost
+====
+
+The advantage of the database backends is, of course, that large amounts of
+data can be stored that won't fit in memory.  The default 'backend' is actually
+to just store all of this information in a Python dictionary.  This is very fast,
+but will run out of memory quickly if you move a lot of data around, or your
+cluster is to run for a long time.
+
+Unfortunately, the DB backends (SQLite and MongoDB) right now are rather slow,
+and can still consume large amounts of resources, particularly if large tasks
+or results are being created at a high frequency.
+
+For this reason, we have added :class:`~.NoDB`,a dummy backend that doesn't
+actually store any information. When you use this database, nothing is stored,
+and any request for results will result in a KeyError.  This obviously prevents
+later requests for results and task resubmission from functioning, but
+sometimes those nice features are not as useful as keeping Hub memory under
+control.
+
+

docs/source/parallel/parallel_process.txt

@@ -1,838 +1,858 b''
 .. _parallel_process:
 
 ===========================================
 Starting the IPython controller and engines
 ===========================================
 
 To use IPython for parallel computing, you need to start one instance of
 the controller and one or more instances of the engine. The controller
 and each engine can run on different machines or on the same machine.
 Because of this, there are many different possibilities.
 
 Broadly speaking, there are two ways of going about starting a controller and engines:
 
 * In an automated manner using the :command:`ipcluster` command.
 * In a more manual way using the :command:`ipcontroller` and 
   :command:`ipengine` commands.
 
 This document describes both of these methods. We recommend that new users
 start with the :command:`ipcluster` command as it simplifies many common usage
 cases.
 
 General considerations
 ======================
 
 Before delving into the details about how you can start a controller and
 engines using the various methods, we outline some of the general issues that
 come up when starting the controller and engines. These things come up no
 matter which method you use to start your IPython cluster.
 
 If you are running engines on multiple machines, you will likely need to instruct the
 controller to listen for connections on an external interface. This can be done by specifying
 the ``ip`` argument on the command-line, or the ``HubFactory.ip`` configurable in
 :file:`ipcontroller_config.py`.
 
 If your machines are on a trusted network, you can safely instruct the controller to listen
 on all public interfaces with::
 
     $> ipcontroller --ip=*
 
 Or you can set the same behavior as the default by adding the following line to your :file:`ipcontroller_config.py`:
 
 .. sourcecode:: python
 
     c.HubFactory.ip = '*'
 
 .. note::
 
     Due to the lack of security in ZeroMQ, the controller will only listen for connections on
     localhost by default. If you see Timeout errors on engines or clients, then the first
     thing you should check is the ip address the controller is listening on, and make sure
     that it is visible from the timing out machine.
     
 .. seealso::
 
     Our `notes <parallel_security>`_ on security in the new parallel computing code.
 
 Let's say that you want to start the controller on ``host0`` and engines on
 hosts ``host1``-``hostn``. The following steps are then required:
 
 1. Start the controller on ``host0`` by running :command:`ipcontroller` on
    ``host0``.  The controller must be instructed to listen on an interface visible
    to the engine machines, via the ``ip`` command-line argument or ``HubFactory.ip``
    in :file:`ipcontroller_config.py`.
 2. Move the JSON file (:file:`ipcontroller-engine.json`) created by the 
    controller from ``host0`` to hosts ``host1``-``hostn``.
 3. Start the engines on hosts ``host1``-``hostn`` by running 
    :command:`ipengine`.  This command has to be told where the JSON file 
    (:file:`ipcontroller-engine.json`) is located.
 
 At this point, the controller and engines will be connected. By default, the JSON files
 created by the controller are put into the :file:`~/.ipython/profile_default/security`
 directory. If the engines share a filesystem with the controller, step 2 can be skipped as
 the engines will automatically look at that location.
 
 The final step required to actually use the running controller from a client is to move
 the JSON file :file:`ipcontroller-client.json` from ``host0`` to any host where clients
 will be run. If these file are put into the :file:`~/.ipython/profile_default/security`
 directory of the client's host, they will be found automatically. Otherwise, the full path
 to them has to be passed to the client's constructor.
 
 Using :command:`ipcluster`
 ===========================
 
 The :command:`ipcluster` command provides a simple way of starting a
 controller and engines in the following situations:
 
 1. When the controller and engines are all run on localhost. This is useful
    for testing or running on a multicore computer.
 2. When engines are started using the :command:`mpiexec` command that comes 
    with most MPI [MPI]_ implementations
 3. When engines are started using the PBS [PBS]_ batch system 
    (or other `qsub` systems, such as SGE).
 4. When the controller is started on localhost and the engines are started on 
    remote nodes using :command:`ssh`.
 5. When engines are started using the Windows HPC Server batch system.
 
 .. note::
 
     Currently :command:`ipcluster` requires that the
     :file:`~/.ipython/profile_<name>/security` directory live on a shared filesystem that is
     seen by both the controller and engines. If you don't have a shared file
     system you will need to use :command:`ipcontroller` and
     :command:`ipengine` directly.
 
 Under the hood, :command:`ipcluster` just uses :command:`ipcontroller`
 and :command:`ipengine` to perform the steps described above.
 
 The simplest way to use ipcluster requires no configuration, and will
 launch a controller and a number of engines on the local machine. For instance, 
 to start one controller and 4 engines on localhost, just do::
 
     $ ipcluster start -n 4
 
 To see other command line options, do::
 
     $ ipcluster -h
 
 
 Configuring an IPython cluster
 ==============================
 
 Cluster configurations are stored as `profiles`.  You can create a new profile with::
 
     $ ipython profile create --parallel --profile=myprofile
 
 This will create the directory :file:`IPYTHONDIR/profile_myprofile`, and populate it
 with the default configuration files for the three IPython cluster commands. Once
 you edit those files, you can continue to call ipcluster/ipcontroller/ipengine
 with no arguments beyond ``profile=myprofile``, and any configuration will be maintained.
 
 There is no limit to the number of profiles you can have, so you can maintain a profile for each
 of your common use cases. The default profile will be used whenever the
 profile argument is not specified, so edit :file:`IPYTHONDIR/profile_default/*_config.py` to
 represent your most common use case.
 
 The configuration files are loaded with commented-out settings and explanations,
 which should cover most of the available possibilities.
 
 Using various batch systems with :command:`ipcluster`
 -----------------------------------------------------
 
 :command:`ipcluster` has a notion of Launchers that can start controllers
 and engines with various remote execution schemes.  Currently supported
 models include :command:`ssh`, :command:`mpiexec`, PBS-style (Torque, SGE, LSF),
 and Windows HPC Server.
 
 In general, these are configured by the :attr:`IPClusterEngines.engine_set_launcher_class`,
 and :attr:`IPClusterStart.controller_launcher_class` configurables, which can be the
 fully specified object name (e.g. ``'IPython.parallel.apps.launcher.LocalControllerLauncher'``),
 but if you are using IPython's builtin launchers, you can specify just the class name,
 or even just the prefix e.g:
 
 .. sourcecode:: python
 
     c.IPClusterEngines.engine_launcher_class = 'SSH'
     # equivalent to
     c.IPClusterEngines.engine_launcher_class = 'SSHEngineSetLauncher'
     # both of which expand to
     c.IPClusterEngines.engine_launcher_class = 'IPython.parallel.apps.launcher.SSHEngineSetLauncher'
 
 The shortest form being of particular use on the command line, where all you need to do to
 get an IPython cluster running with engines started with MPI is:
 
 .. sourcecode:: bash
 
     $> ipcluster start --engines=MPI
 
 Assuming that the default MPI config is sufficient.
 
 .. note::
 
     shortcuts for builtin launcher names were added in 0.12, as was the ``_class`` suffix
     on the configurable names.  If you use the old 0.11 names (e.g. ``engine_set_launcher``),
     they will still work, but you will get a deprecation warning that the name has changed.
 
 
 .. note::
 
     The Launchers and configuration are designed in such a way that advanced
     users can subclass and configure them to fit their own system that we
     have not yet supported (such as Condor)
 
 Using :command:`ipcluster` in mpiexec/mpirun mode
 -------------------------------------------------
 
 
 The mpiexec/mpirun mode is useful if you:
 
 1. Have MPI installed.
 2. Your systems are configured to use the :command:`mpiexec` or
    :command:`mpirun` commands to start MPI processes.
 
 If these are satisfied, you can create a new profile::
 
     $ ipython profile create --parallel --profile=mpi
 
 and edit the file :file:`IPYTHONDIR/profile_mpi/ipcluster_config.py`.
 
 There, instruct ipcluster to use the MPI launchers by adding the lines:
 
 .. sourcecode:: python
 
     c.IPClusterEngines.engine_launcher_class = 'MPIEngineSetLauncher'
 
 If the default MPI configuration is correct, then you can now start your cluster, with::
 
     $ ipcluster start -n 4 --profile=mpi
 
 This does the following:
 
 1. Starts the IPython controller on current host.
 2. Uses :command:`mpiexec` to start 4 engines.
 
 If you have a reason to also start the Controller with mpi, you can specify:
 
 .. sourcecode:: python
 
     c.IPClusterStart.controller_launcher_class = 'MPIControllerLauncher'
 
 .. note::
 
     The Controller *will not* be in the same MPI universe as the engines, so there is not
     much reason to do this unless sysadmins demand it.
 
 On newer MPI implementations (such as OpenMPI), this will work even if you
 don't make any calls to MPI or call :func:`MPI_Init`. However, older MPI
 implementations actually require each process to call :func:`MPI_Init` upon
 starting. The easiest way of having this done is to install the mpi4py
 [mpi4py]_ package and then specify the ``c.MPI.use`` option in :file:`ipengine_config.py`:
 
 .. sourcecode:: python
 
     c.MPI.use = 'mpi4py'
 
 Unfortunately, even this won't work for some MPI implementations. If you are
 having problems with this, you will likely have to use a custom Python
 executable that itself calls :func:`MPI_Init` at the appropriate time.
 Fortunately, mpi4py comes with such a custom Python executable that is easy to
 install and use. However, this custom Python executable approach will not work
 with :command:`ipcluster` currently.
 
 More details on using MPI with IPython can be found :ref:`here <parallelmpi>`.
 
 
 Using :command:`ipcluster` in PBS mode
 --------------------------------------
 
 The PBS mode uses the Portable Batch System (PBS) to start the engines. 
 
 As usual, we will start by creating a fresh profile::
 
     $ ipython profile create --parallel --profile=pbs
 
 And in :file:`ipcluster_config.py`, we will select the PBS launchers for the controller
 and engines:
 
 .. sourcecode:: python
 
     c.IPClusterStart.controller_launcher_class = 'PBSControllerLauncher'
     c.IPClusterEngines.engine_launcher_class = 'PBSEngineSetLauncher'
 
 .. note::
 
     Note that the configurable is IPClusterEngines for the engine launcher, and
     IPClusterStart for the controller launcher. This is because the start command is a
     subclass of the engine command, adding a controller launcher. Since it is a subclass,
     any configuration made in IPClusterEngines is inherited by IPClusterStart unless it is
     overridden.
 
 IPython does provide simple default batch templates for PBS and SGE, but you may need
 to specify your own. Here is a sample PBS script template:
 
 .. sourcecode:: bash
 
     #PBS -N ipython
     #PBS -j oe
     #PBS -l walltime=00:10:00
     #PBS -l nodes={n/4}:ppn=4
     #PBS -q {queue}
 
     cd $PBS_O_WORKDIR
     export PATH=$HOME/usr/local/bin
     export PYTHONPATH=$HOME/usr/local/lib/python2.7/site-packages
     /usr/local/bin/mpiexec -n {n} ipengine --profile-dir={profile_dir}
 
 There are a few important points about this template:
 
 1. This template will be rendered at runtime using IPython's :class:`EvalFormatter`.
    This is simply a subclass of :class:`string.Formatter` that allows simple expressions
    on keys.
 
 2. Instead of putting in the actual number of engines, use the notation
    ``{n}`` to indicate the number of engines to be started. You can also use
    expressions like ``{n/4}`` in the template to indicate the number of nodes.
    There will always be ``{n}`` and ``{profile_dir}`` variables passed to the formatter.
    These allow the batch system to know how many engines, and where the configuration
    files reside. The same is true for the batch queue, with the template variable
    ``{queue}``.
 
 3. Any options to :command:`ipengine` can be given in the batch script
    template, or in :file:`ipengine_config.py`.
 
 4. Depending on the configuration of you system, you may have to set
    environment variables in the script template.
 
 The controller template should be similar, but simpler:
 
 .. sourcecode:: bash
 
     #PBS -N ipython
     #PBS -j oe
     #PBS -l walltime=00:10:00
     #PBS -l nodes=1:ppn=4
     #PBS -q {queue}
 
     cd $PBS_O_WORKDIR
     export PATH=$HOME/usr/local/bin
     export PYTHONPATH=$HOME/usr/local/lib/python2.7/site-packages
     ipcontroller --profile-dir={profile_dir}
 
 
 Once you have created these scripts, save them with names like
 :file:`pbs.engine.template`. Now you can load them into the :file:`ipcluster_config` with:
 
 .. sourcecode:: python
 
     c.PBSEngineSetLauncher.batch_template_file = "pbs.engine.template"
     
     c.PBSControllerLauncher.batch_template_file = "pbs.controller.template"
         
 
 Alternately, you can just define the templates as strings inside :file:`ipcluster_config`.
 
 Whether you are using your own templates or our defaults, the extra configurables available are
 the number of engines to launch (``{n}``, and the batch system queue to which the jobs are to be
 submitted (``{queue}``)). These are configurables, and can be specified in
 :file:`ipcluster_config`:
 
 .. sourcecode:: python
 
     c.PBSLauncher.queue = 'veryshort.q'
     c.IPClusterEngines.n = 64
 
 Note that assuming you are running PBS on a multi-node cluster, the Controller's default behavior
 of listening only on localhost is likely too restrictive.  In this case, also assuming the
 nodes are safely behind a firewall, you can simply instruct the Controller to listen for
 connections on all its interfaces, by adding in :file:`ipcontroller_config`:
 
 .. sourcecode:: python
 
     c.HubFactory.ip = '*'
 
 You can now run the cluster with::
 
     $ ipcluster start --profile=pbs -n 128
 
 Additional configuration options can be found in the PBS section of :file:`ipcluster_config`.
 
 .. note::
 
     Due to the flexibility of configuration, the PBS launchers work with simple changes
     to the template for other :command:`qsub`-using systems, such as Sun Grid Engine,
     and with further configuration in similar batch systems like Condor.
 
 
 Using :command:`ipcluster` in SSH mode
 --------------------------------------
 
 
 The SSH mode uses :command:`ssh` to execute :command:`ipengine` on remote
 nodes and :command:`ipcontroller` can be run remotely as well, or on localhost.
 
 .. note::
 
     When using this mode it highly recommended that you have set up SSH keys
     and are using ssh-agent [SSH]_ for password-less logins.
 
 As usual, we start by creating a clean profile::
 
     $ ipython profile create --parallel --profile=ssh
 
 To use this mode, select the SSH launchers in :file:`ipcluster_config.py`:
 
 .. sourcecode:: python
 
     c.IPClusterEngines.engine_launcher_class = 'SSHEngineSetLauncher'
     # and if the Controller is also to be remote:
     c.IPClusterStart.controller_launcher_class = 'SSHControllerLauncher'
 
 
 
 The controller's remote location and configuration can be specified:
 
 .. sourcecode:: python
 
     # Set the user and hostname for the controller
     # c.SSHControllerLauncher.hostname = 'controller.example.com'
     # c.SSHControllerLauncher.user = os.environ.get('USER','username')
 
     # Set the arguments to be passed to ipcontroller
     # note that remotely launched ipcontroller will not get the contents of 
     # the local ipcontroller_config.py unless it resides on the *remote host*
     # in the location specified by the `profile-dir` argument.
     # c.SSHControllerLauncher.controller_args = ['--reuse', '--ip=*', '--profile-dir=/path/to/cd']
 
 .. note::
 
     SSH mode does not do any file movement, so you will need to distribute configuration
     files manually.  To aid in this, the `reuse_files` flag defaults to True for ssh-launched
     Controllers, so you will only need to do this once, unless you override this flag back
     to False.
 
 Engines are specified in a dictionary, by hostname and the number of engines to be run
 on that host.
 
 .. sourcecode:: python
 
     c.SSHEngineSetLauncher.engines = { 'host1.example.com' : 2,
                 'host2.example.com' : 5,
                 'host3.example.com' : (1, ['--profile-dir=/home/different/location']),
                 'host4.example.com' : 8 }
 
 * The `engines` dict, where the keys are the host we want to run engines on and
   the value is the number of engines to run on that host.
 * on host3, the value is a tuple, where the number of engines is first, and the arguments
   to be passed to :command:`ipengine` are the second element.
 
 For engines without explicitly specified arguments, the default arguments are set in
 a single location:
 
 .. sourcecode:: python
 
     c.SSHEngineSetLauncher.engine_args = ['--profile-dir=/path/to/profile_ssh']
 
 Current limitations of the SSH mode of :command:`ipcluster` are:
 
 * Untested on Windows. Would require a working :command:`ssh` on Windows.
   Also, we are using shell scripts to setup and execute commands on remote
   hosts.
 * No file movement - This is a regression from 0.10, which moved connection files
   around with scp. This will be improved, Pull Requests are welcome.
 
 
 IPython on EC2 with StarCluster
 ===============================
 
 The excellent StarCluster_ toolkit for managing `Amazon EC2`_ clusters has a plugin
 which makes deploying IPython on EC2 quite simple.  The starcluster plugin uses
 :command:`ipcluster` with the SGE launchers to distribute engines across the
 EC2 cluster.  See their `ipcluster plugin documentation`_ for more information.
 
 .. _StarCluster: http://web.mit.edu/starcluster
 .. _Amazon EC2: http://aws.amazon.com/ec2/
 .. _ipcluster plugin documentation: http://web.mit.edu/starcluster/docs/latest/plugins/ipython.html
 
 
 Using the :command:`ipcontroller` and :command:`ipengine` commands
 ==================================================================
 
 It is also possible to use the :command:`ipcontroller` and :command:`ipengine`
 commands to start your controller and engines. This approach gives you full
 control over all aspects of the startup process.
 
 Starting the controller and engine on your local machine
 --------------------------------------------------------
 
 To use :command:`ipcontroller` and :command:`ipengine` to start things on your
 local machine, do the following.
 
 First start the controller::
 
     $ ipcontroller
     
 Next, start however many instances of the engine you want using (repeatedly)
 the command::
 
     $ ipengine
 
 The engines should start and automatically connect to the controller using the
 JSON files in :file:`~/.ipython/profile_default/security`. You are now ready to use the
 controller and engines from IPython.
 
 .. warning:: 
     
     The order of the above operations may be important.  You *must*
     start the controller before the engines, unless you are reusing connection
     information (via ``--reuse``), in which case ordering is not important.
 
 .. note::
 
     On some platforms (OS X), to put the controller and engine into the
     background you may need to give these commands in the form ``(ipcontroller
     &)`` and ``(ipengine &)`` (with the parentheses) for them to work
     properly.
 
 Starting the controller and engines on different hosts
 ------------------------------------------------------
 
 When the controller and engines are running on different hosts, things are
 slightly more complicated, but the underlying ideas are the same:
 
 1. Start the controller on a host using :command:`ipcontroller`. The controller must be
    instructed to listen on an interface visible to the engine machines, via the ``ip``
    command-line argument or ``HubFactory.ip`` in :file:`ipcontroller_config.py`::
    
         $ ipcontroller --ip=192.168.1.16
    
    .. sourcecode:: python
    
         # in ipcontroller_config.py
         HubFactory.ip = '192.168.1.16'
 
 2. Copy :file:`ipcontroller-engine.json` from :file:`~/.ipython/profile_<name>/security` on
    the controller's host to the host where the engines will run.
 3. Use :command:`ipengine` on the engine's hosts to start the engines.
 
 The only thing you have to be careful of is to tell :command:`ipengine` where
 the :file:`ipcontroller-engine.json` file is located. There are two ways you
 can do this:
 
 * Put :file:`ipcontroller-engine.json` in the :file:`~/.ipython/profile_<name>/security`
   directory on the engine's host, where it will be found automatically.
 * Call :command:`ipengine` with the ``--file=full_path_to_the_file``
   flag.
 
 The ``file`` flag works like this::
 
     $ ipengine --file=/path/to/my/ipcontroller-engine.json
 
 .. note::
     
     If the controller's and engine's hosts all have a shared file system  
     (:file:`~/.ipython/profile_<name>/security` is the same on all of them), then things
     will just work!
 
 SSH Tunnels
 ***********
 
 If your engines are not on the same LAN as the controller, or you are on a highly
 restricted network where your nodes cannot see each others ports, then you can
 use SSH tunnels to connect engines to the controller.
 
 .. note::
 
     This does not work in all cases.  Manual tunnels may be an option, but are
     highly inconvenient. Support for manual tunnels will be improved.
 
 You can instruct all engines to use ssh, by specifying the ssh server in 
 :file:`ipcontroller-engine.json`:
 
 .. I know this is really JSON, but the example is a subset of Python:
 .. sourcecode:: python
 
     {
       "url":"tcp://192.168.1.123:56951",
       "exec_key":"26f4c040-587d-4a4e-b58b-030b96399584",
       "ssh":"[e-mail]",
       "location":"192.168.1.123"
     }
 
 This will be specified if you give the ``--enginessh=[e-mail]`` argument when
 starting :command:`ipcontroller`.
 
 Or you can specify an ssh server on the command-line when starting an engine::
 
     $> ipengine --profile=foo --ssh=my.login.node
 
 For example, if your system is totally restricted, then all connections will actually be
 loopback, and ssh tunnels will be used to connect engines to the controller::
 
     [node1] $> ipcontroller --enginessh=node1
     [node2] $> ipengine
     [node3] $> ipcluster engines --n=4
 
 Or if you want to start many engines on each node, the command `ipcluster engines --n=4`
 without any configuration is equivalent to running ipengine 4 times.
 
 An example using ipcontroller/engine with ssh
 ---------------------------------------------
 
 No configuration files are necessary to use ipcontroller/engine in an SSH environment
 without a shared filesystem. You simply need to make sure that the controller is listening
 on an interface visible to the engines, and move the connection file from the controller to
 the engines.
 
 1. start the controller, listening on an ip-address visible to the engine machines::
 
     [controller.host] $ ipcontroller --ip=192.168.1.16
     
     [IPControllerApp] Using existing profile dir: u'/Users/me/.ipython/profile_default'
     [IPControllerApp] Hub listening on tcp://192.168.1.16:63320 for registration.
     [IPControllerApp] Hub using DB backend: 'IPython.parallel.controller.dictdb.DictDB'
     [IPControllerApp] hub::created hub
     [IPControllerApp] writing connection info to /Users/me/.ipython/profile_default/security/ipcontroller-client.json
     [IPControllerApp] writing connection info to /Users/me/.ipython/profile_default/security/ipcontroller-engine.json
     [IPControllerApp] task::using Python leastload Task scheduler
     [IPControllerApp] Heartmonitor started
     [IPControllerApp] Creating pid file: /Users/me/.ipython/profile_default/pid/ipcontroller.pid
     Scheduler started [leastload]
 
 2. on each engine, fetch the connection file with scp::
 
     [engine.host.n] $ scp controller.host:.ipython/profile_default/security/ipcontroller-engine.json ./
 
    .. note::
 
         The log output of ipcontroller above shows you where the json files were written.
         They will be in :file:`~/.ipython` (or :file:`~/.config/ipython`) under
         :file:`profile_default/security/ipcontroller-engine.json`
 
 3. start the engines, using the connection file::
 
     [engine.host.n] $ ipengine --file=./ipcontroller-engine.json
 
 A couple of notes:
 
 * You can avoid having to fetch the connection file every time by adding ``--reuse`` flag
   to ipcontroller, which instructs the controller to read the previous connection file for
   connection info, rather than generate a new one with randomized ports.
 
 * In step 2, if you fetch the connection file directly into the security dir of a profile,
   then you need not specify its path directly, only the profile (assumes the path exists,
   otherwise you must create it first)::
 
     [engine.host.n] $ scp controller.host:.ipython/profile_default/security/ipcontroller-engine.json ~/.ipython/profile_ssh/security/
     [engine.host.n] $ ipengine --profile=ssh
 
   Of course, if you fetch the file into the default profile, no arguments must be passed to
   ipengine at all.
 
 * Note that ipengine *did not* specify the ip argument. In general, it is unlikely for any
   connection information to be specified at the command-line to ipengine, as all of this
   information should be contained in the connection file written by ipcontroller.
 
 Make JSON files persistent
 --------------------------
 
 At fist glance it may seem that that managing the JSON files is a bit
 annoying. Going back to the house and key analogy, copying the JSON around
 each time you start the controller is like having to make a new key every time
 you want to unlock the door and enter your house. As with your house, you want
 to be able to create the key (or JSON file) once, and then simply use it at
 any point in the future.
 
 To do this, the only thing you have to do is specify the `--reuse` flag, so that
 the connection information in the JSON files remains accurate::
 
     $ ipcontroller --reuse
 
 Then, just copy the JSON files over the first time and you are set. You can
 start and stop the controller and engines any many times as you want in the
 future, just make sure to tell the controller to reuse the file.
 
 .. note::
 
     You may ask the question: what ports does the controller listen on if you
     don't tell is to use specific ones? The default is to use high random port
     numbers. We do this for two reasons: i) to increase security through
     obscurity and ii) to multiple controllers on a given host to start and
     automatically use different ports.
 
 Log files
 ---------
 
 All of the components of IPython have log files associated with them.
 These log files can be extremely useful in debugging problems with
 IPython and can be found in the directory :file:`~/.ipython/profile_<name>/log`.
 Sending the log files to us will often help us to debug any problems.
 
 
 Configuring `ipcontroller`
 ---------------------------
 
 The IPython Controller takes its configuration from the file :file:`ipcontroller_config.py`
 in the active profile directory.
 
 Ports and addresses
 *******************
 
 In many cases, you will want to configure the Controller's network identity.  By default,
 the Controller listens only on loopback, which is the most secure but often impractical.
 To instruct the controller to listen on a specific interface, you can set the 
 :attr:`HubFactory.ip` trait.  To listen on all interfaces, simply specify:
 
 .. sourcecode:: python
 
     c.HubFactory.ip = '*'
 
 When connecting to a Controller that is listening on loopback or behind a firewall, it may
 be necessary to specify an SSH server to use for tunnels, and the external IP of the
 Controller. If you specified that the HubFactory listen on loopback, or all interfaces,
 then IPython will try to guess the external IP. If you are on a system with VM network
 devices, or many interfaces, this guess may be incorrect. In these cases, you will want
 to specify the 'location' of the Controller. This is the IP of the machine the Controller
 is on, as seen by the clients, engines, or the SSH server used to tunnel connections.
 
 For example, to set up a cluster with a Controller on a work node, using ssh tunnels
 through the login node, an example :file:`ipcontroller_config.py` might contain:
 
 .. sourcecode:: python
 
     # allow connections on all interfaces from engines
     # engines on the same node will use loopback, while engines
     # from other nodes will use an external IP
     c.HubFactory.ip = '*'
     
     # you typically only need to specify the location when there are extra
     # interfaces that may not be visible to peer nodes (e.g. VM interfaces)
     c.HubFactory.location = '10.0.1.5'
     # or to get an automatic value, try this:
     import socket
     ex_ip = socket.gethostbyname_ex(socket.gethostname())[-1][0]
     c.HubFactory.location = ex_ip
     
     # now instruct clients to use the login node for SSH tunnels:
     c.HubFactory.ssh_server = 'login.mycluster.net'
 
 After doing this, your :file:`ipcontroller-client.json` file will look something like this:
 
 .. this can be Python, despite the fact that it's actually JSON, because it's
 .. still valid Python
 
 .. sourcecode:: python
 
     {
       "url":"tcp:\/\/*:43447",
       "exec_key":"9c7779e4-d08a-4c3b-ba8e-db1f80b562c1",
       "ssh":"login.mycluster.net",
       "location":"10.0.1.5"
     }
 
 Then this file will be all you need for a client to connect to the controller, tunneling
 SSH connections through login.mycluster.net.
 
 Database Backend
 ****************
 
 The Hub stores all messages and results passed between Clients and Engines.
 For large and/or long-running clusters, it would be unreasonable to keep all
 of this information in memory. For this reason, we have two database backends:
 [MongoDB]_ via PyMongo_, and SQLite with the stdlib :py:mod:`sqlite`.
 
 MongoDB is our design target, and the dict-like model it uses has driven our design. As far
 as we are concerned, BSON can be considered essentially the same as JSON, adding support
 for binary data and datetime objects, and any new database backend must support the same
 data types.
 
 .. seealso::
 
     MongoDB `BSON doc <http://www.mongodb.org/display/DOCS/BSON>`_
 
 To use one of these backends, you must set the :attr:`HubFactory.db_class` trait:
 
 .. sourcecode:: python
 
     # for a simple dict-based in-memory implementation, use dictdb
     # This is the default and the fastest, since it doesn't involve the filesystem
     c.HubFactory.db_class = 'IPython.parallel.controller.dictdb.DictDB'
     
     # To use MongoDB:
     c.HubFactory.db_class = 'IPython.parallel.controller.mongodb.MongoDB'
     
     # and SQLite:
     c.HubFactory.db_class = 'IPython.parallel.controller.sqlitedb.SQLiteDB'
 
+    # You can use NoDB to disable the database altogether, in case you don't need
+    # to reuse tasks or results, and want to keep memory consumption under control.
+    c.HubFactory.db_class = 'IPython.parallel.controller.dictdb.NoDB'
+
 When using the proper databases, you can actually allow for tasks to persist from
 one session to the next by specifying the MongoDB database or SQLite table in
 which tasks are to be stored.  The default is to use a table named for the Hub's Session,
 which is a UUID, and thus different every time.
 
 .. sourcecode:: python
 
     # To keep persistant task history in MongoDB:
     c.MongoDB.database = 'tasks'
     
     # and in SQLite:
     c.SQLiteDB.table = 'tasks'
 
 
 Since MongoDB servers can be running remotely or configured to listen on a particular port,
 you can specify any arguments you may need to the PyMongo `Connection 
 <http://api.mongodb.org/python/1.9/api/pymongo/connection.html#pymongo.connection.Connection>`_:
 
 .. sourcecode:: python
 
     # positional args to pymongo.Connection
     c.MongoDB.connection_args = []
     
     # keyword args to pymongo.Connection
     c.MongoDB.connection_kwargs = {}
 
+But sometimes you are moving lots of data around quickly, and you don't need
+that information to be stored for later access, even by other Clients to this
+same session. For this case, we have a dummy database, which doesn't actually
+store anything. This lets the Hub stay small in memory, at the obvious expense
+of being able to access the information that would have been stored in the
+database (used for task resubmission, requesting results of tasks you didn't
+submit, etc.). To use this backend, simply pass ``--nodb`` to
+:command:`ipcontroller` on the command-line, or specify the :class:`NoDB` class
+in your :file:`ipcontroller_config.py` as described above.
+
+
+.. seealso::
+
+    For more information on the database backends, see the :ref:`db backend reference <parallel_db>`.
+
+
 .. _PyMongo: http://api.mongodb.org/python/1.9/
 
 Configuring `ipengine`
 -----------------------
 
 The IPython Engine takes its configuration from the file :file:`ipengine_config.py`
 
 The Engine itself also has some amount of configuration. Most of this
 has to do with initializing MPI or connecting to the controller.
 
 To instruct the Engine to initialize with an MPI environment set up by
 mpi4py, add:
 
 .. sourcecode:: python
 
     c.MPI.use = 'mpi4py'
 
 In this case, the Engine will use our default mpi4py init script to set up
 the MPI environment prior to exection.  We have default init scripts for
 mpi4py and pytrilinos.  If you want to specify your own code to be run
 at the beginning, specify `c.MPI.init_script`.
 
 You can also specify a file or python command to be run at startup of the
 Engine:
 
 .. sourcecode:: python
 
     c.IPEngineApp.startup_script = u'/path/to/my/startup.py'
     
     c.IPEngineApp.startup_command = 'import numpy, scipy, mpi4py'
 
 These commands/files will be run again, after each
 
 It's also useful on systems with shared filesystems to run the engines
 in some scratch directory.  This can be set with:
 
 .. sourcecode:: python
 
     c.IPEngineApp.work_dir = u'/path/to/scratch/'
 
 
 
 .. [MongoDB] MongoDB database http://www.mongodb.org
 
 .. [PBS] Portable Batch System http://www.openpbs.org
 
 .. [SSH] SSH-Agent http://en.wikipedia.org/wiki/ssh-agent