parallel docs, tests, default config updated to newconfig

IPython/parallel/apps/clusterdir.py

@@ -328,8 +328,8 b' class ClusterApplication(BaseIPythonApplication):'
 
     The cluster directory is resolved as follows:
 
-    * If the ``--cluster-dir`` option is given, it is used.
-    * If ``--cluster-dir`` is not given, the application directory is 
+    * If the ``cluster_dir`` option is given, it is used.
+    * If ``cluster_dir`` is not given, the application directory is
       resolve using the profile name as ``cluster_<profile>``. The search 
       path for this directory is then i) cwd if it is found there
       and ii) in ipython_dir otherwise.

IPython/parallel/apps/ipclusterapp.py

@@ -46,8 +46,7 b' from IPython.parallel.apps.clusterdir import ('
 default_config_file_name = u'ipcluster_config.py'
 
 
-_description = """\
-Start an IPython cluster for parallel computing.\n\n
+_description = """Start an IPython cluster for parallel computing.
 
 An IPython cluster consists of 1 controller and 1 or more engines.
 This command automates the startup of these processes using a wide
@@ -78,7 +77,8 b' NO_CLUSTER = 12'
 #-----------------------------------------------------------------------------
 # Main application
 #-----------------------------------------------------------------------------
-start_help = """
+start_help = """Start an IPython cluster for parallel computing
+
 Start an ipython cluster by its profile name or cluster
 directory. Cluster directories contain configuration, log and
 security related files and are named using the convention
@@ -88,7 +88,8 b' the cwd or the ipython directory, you can simply refer to it'
 using its profile name, 'ipcluster start n=4 profile=<profile>`,
 otherwise use the 'cluster_dir' option.
 """
-stop_help = """
+stop_help = """Stop a running IPython cluster
+
 Stop a running ipython cluster by its profile name or cluster
 directory. Cluster directories are named using the convention
 'cluster_<profile>'. If your cluster directory is in 
@@ -96,7 +97,8 b' the cwd or the ipython directory, you can simply refer to it'
 using its profile name, 'ipcluster stop profile=<profile>`, otherwise
 use the 'cluster_dir' option.
 """
-engines_help = """
+engines_help = """Start engines connected to an existing IPython cluster
+
 Start one or more engines to connect to an existing Cluster
 by profile name or cluster directory.
 Cluster directories contain configuration, log and
@@ -107,7 +109,8 b' the cwd or the ipython directory, you can simply refer to it'
 using its profile name, 'ipcluster engines n=4 profile=<profile>`,
 otherwise use the 'cluster_dir' option.
 """
-create_help = """
+create_help = """Create an ipcluster profile by name
+
 Create an ipython cluster directory by its profile name or
 cluster directory path. Cluster directories contain
 configuration, log and security related files and are named
@@ -119,7 +122,9 b' cluster directory by profile name,'
 `ipcluster create profile=mycluster`, which will put the directory
 in `<ipython_dir>/cluster_mycluster`.
 """
-list_help = """List all available clusters, by cluster directory, that can
+list_help = """List available cluster profiles
+
+List all available clusters, by cluster directory, that can
 be found in the current working directly or in the ipython
 directory. Cluster directories are named using the convention
 'cluster_<profile>'.

IPython/parallel/apps/ipcontrollerapp.py

@@ -79,8 +79,8 b' The IPython controller provides a gateway between the IPython engines and'
 clients. The controller needs to be started before the engines and can be
 configured using command line options or using a cluster directory. Cluster
 directories contain config, log and security files and are usually located in
-your ipython directory and named as "cluster_<profile>". See the --profile
-and --cluster-dir options for details.
+your ipython directory and named as "cluster_<profile>". See the `profile`
+and `cluster_dir` options for details.
 """
 
 
@@ -92,14 +92,16 b' and --cluster-dir options for details.'
 flags = {}
 flags.update(base_flags)
 flags.update({
-    'usethreads' : ( {'IPControllerApp' : {'usethreads' : True}},
+    'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
                     'Use threads instead of processes for the schedulers'),
-    'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
+    'sqlitedb' : ({'HubFactory' : Config({'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'})},
                     'use the SQLiteDB backend'),
-    'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
+    'mongodb' : ({'HubFactory' : Config({'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'})},
                     'use the MongoDB backend'),
-    'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
+    'dictdb' : ({'HubFactory' : Config({'db_class' : 'IPython.parallel.controller.dictdb.DictDB'})},
                     'use the in-memory DictDB backend'),
+    'reuse' : ({'IPControllerApp' : Config({'reuse_files' : True})},
+                    'reuse existing json connection files')
 })
 
 flags.update()
@@ -133,7 +135,7 b' class IPControllerApp(ClusterApplication):'
         help="import statements to be run at startup.  Necessary in some environments"
     )
 
-    usethreads = Bool(False, config=True,
+    use_threads = Bool(False, config=True,
         help='Use threads instead of processes for the schedulers',
         )
 
@@ -141,7 +143,7 b' class IPControllerApp(ClusterApplication):'
     children = List()
     mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')
 
-    def _usethreads_changed(self, name, old, new):
+    def _use_threads_changed(self, name, old, new):
         self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
 
     aliases = Dict(dict(
@@ -152,7 +154,7 b' class IPControllerApp(ClusterApplication):'
         reuse_files = 'IPControllerApp.reuse_files',
         secure = 'IPControllerApp.secure',
         ssh = 'IPControllerApp.ssh_server',
-        usethreads = 'IPControllerApp.usethreads',
+        use_threads = 'IPControllerApp.use_threads',
         import_statements = 'IPControllerApp.import_statements',
         location = 'IPControllerApp.location',
 
@@ -271,7 +273,7 b' class IPControllerApp(ClusterApplication):'
         mq = import_item(str(self.mq_class))
         
         hub = self.factory
-        # maybe_inproc = 'inproc://monitor' if self.usethreads else self.monitor_url
+        # maybe_inproc = 'inproc://monitor' if self.use_threads else self.monitor_url
         # IOPub relay (in a Process)
         q = mq(zmq.PUB, zmq.SUB, zmq.PUB, 'N/A','iopub')
         q.bind_in(hub.client_info['iopub'])

IPython/parallel/apps/ipengineapp.py

@@ -46,7 +46,7 b' from IPython.utils.traitlets import Bool, Unicode, Dict, List'
 #: The default config file name for this application
 default_config_file_name = u'ipengine_config.py'
 
-_description = """Start an IPython engine for parallel computing.\n\n
+_description = """Start an IPython engine for parallel computing.
 
 IPython engines run in parallel and perform computations on behalf of a client
 and controller. A controller needs to be started before the engines. The

IPython/parallel/apps/iploggerapp.py

@@ -36,14 +36,14 b' from IPython.parallel.apps.logwatcher import LogWatcher'
 #: The default config file name for this application
 default_config_file_name = u'iplogger_config.py'
 
-_description = """Start an IPython logger for parallel computing.\n\n
+_description = """Start an IPython logger for parallel computing.
 
 IPython controllers and engines (and your own processes) can broadcast log messages
 by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The
 logger can be configured using command line options or using a cluster
 directory. Cluster directories contain config, log and security files and are
 usually located in your ipython directory and named as "cluster_<profile>".
-See the --profile and --cluster-dir options for details.
+See the `profile` and `cluster_dir` options for details.
 """
 
 

IPython/parallel/engine/engine.py

@@ -141,7 +141,6 b' class EngineFactory(RegistrationFactory):'
             self.kernel.start()
             hb_addrs = [ disambiguate_url(addr, self.location) for addr in hb_addrs ]
             heart = Heart(*map(str, hb_addrs), heart_id=identity)
-            # ioloop.DelayedCallback(heart.start, 1000, self.loop).start()
             heart.start()
             
             

IPython/parallel/tests/__init__.py

@@ -48,7 +48,7 b' class TestProcessLauncher(LocalProcessLauncher):'
 def setup():
     cp = TestProcessLauncher()
     cp.cmd_and_args = ipcontroller_cmd_argv + \
-                ['--profile', 'iptest', '--log-level', '99', '-r']
+                ['profile=iptest', 'log_level=50', '--reuse']
     cp.start()
     launchers.append(cp)
     cluster_dir = os.path.join(get_ipython_dir(), 'cluster_iptest')
@@ -70,7 +70,7 b" def add_engines(n=1, profile='iptest'):"
     eps = []
     for i in range(n):
         ep = TestProcessLauncher()
-        ep.cmd_and_args = ipengine_cmd_argv + ['--profile', profile, '--log-level', '99']
+        ep.cmd_and_args = ipengine_cmd_argv + ['profile=%s'%profile, 'log_level=50']
         ep.start()
         launchers.append(ep)
         eps.append(ep)

IPython/parallel/tests/test_streamsession.py

@@ -48,11 +48,11 b' class TestSession(SessionTestCase):'
         self.assertTrue(s.unpack is ss.default_unpacker)
         self.assertEquals(s.username, os.environ.get('USER', 'username'))
         
-        s = ss.StreamSession(username=None)
+        s = ss.StreamSession()
         self.assertEquals(s.username, os.environ.get('USER', 'username'))
         
-        self.assertRaises(TypeError, ss.StreamSession, packer='hi')
-        self.assertRaises(TypeError, ss.StreamSession, unpacker='hi')
+        self.assertRaises(TypeError, ss.StreamSession, pack='hi')
+        self.assertRaises(TypeError, ss.StreamSession, unpack='hi')
         u = str(uuid.uuid4())
         s = ss.StreamSession(username='carrot', session=u)
         self.assertEquals(s.session, u)

docs/source/parallel/parallel_intro.txt

@@ -195,7 +195,7 b' simply start a controller and engines on a single host using the'
 :command:`ipcluster` command. To start a controller and 4 engines on your
 localhost, just do::
 
-    $ ipcluster start -n 4
+    $ ipcluster start n=4
 
 More details about starting the IPython controller and engines can be found
 :ref:`here <parallel_process>`

docs/source/parallel/parallel_mpi.txt

@@ -53,11 +53,11 b' these things to happen.'
 Automatic starting using :command:`mpiexec` and :command:`ipcluster`
 --------------------------------------------------------------------
 
-The easiest approach is to use the `mpiexec` mode of :command:`ipcluster`,
+The easiest approach is to use the `MPIExec` Launchers in :command:`ipcluster`,
 which will first start a controller and then a set of engines using
 :command:`mpiexec`::
 
-    $ ipcluster mpiexec -n 4
+    $ ipcluster start n=4 elauncher=MPIExecEngineSetLauncher
 
 This approach is best as interrupting :command:`ipcluster` will automatically
 stop and clean up the controller and engines.
@@ -68,14 +68,14 b' Manual starting using :command:`mpiexec`'
 If you want to start the IPython engines using the :command:`mpiexec`, just
 do::
 
-    $ mpiexec -n 4 ipengine --mpi=mpi4py
+    $ mpiexec n=4 ipengine mpi=mpi4py
 
 This requires that you already have a controller running and that the FURL
 files for the engines are in place. We also have built in support for
 PyTrilinos [PyTrilinos]_, which can be used (assuming is installed) by
 starting the engines with::
 
-    $ mpiexec -n 4 ipengine --mpi=pytrilinos
+    $ mpiexec n=4 ipengine mpi=pytrilinos
 
 Automatic starting using PBS and :command:`ipcluster`
 ------------------------------------------------------
@@ -110,7 +110,7 b' distributed array. Save the following text in a file called :file:`psum.py`:'
 
 Now, start an IPython cluster::
 
-    $ ipcluster start -p mpi -n 4
+    $ ipcluster start profile=mpi n=4
     
 .. note::
 

docs/source/parallel/parallel_multiengine.txt

@@ -19,7 +19,7 b' To follow along with this tutorial, you will need to start the IPython'
 controller and four IPython engines. The simplest way of doing this is to use
 the :command:`ipcluster` command::
 
-    $ ipcluster start -n 4
+    $ ipcluster start n=4
     
 For more detailed information about starting the controller and engines, see
 our :ref:`introduction <ip1par>` to using IPython for parallel computing.

docs/source/parallel/parallel_process.txt

@@ -57,7 +57,7 b' controller and engines in the following situations:'
 
 1. When the controller and engines are all run on localhost. This is useful
    for testing or running on a multicore computer.
-2. When engines are started using the :command:`mpirun` command that comes 
+2. When engines are started using the :command:`mpiexec` command that comes 
    with most MPI [MPI]_ implementations
 3. When engines are started using the PBS [PBS]_ batch system 
    (or other `qsub` systems, such as SGE).
@@ -80,9 +80,9 b' The simplest way to use ipcluster requires no configuration, and will'
 launch a controller and a number of engines on the local machine. For instance, 
 to start one controller and 4 engines on localhost, just do::
 
-    $ ipcluster start -n 4
+    $ ipcluster start n=4
 
-To see other command line options for the local mode, do::
+To see other command line options, do::
 
     $ ipcluster -h
 
@@ -92,12 +92,12 b' Configuring an IPython cluster'
 
 Cluster configurations are stored as `profiles`.  You can create a new profile with::
 
-    $ ipcluster create -p myprofile
+    $ ipcluster create profile=myprofile
 
 This will create the directory :file:`IPYTHONDIR/cluster_myprofile`, and populate it
 with the default configuration files for the three IPython cluster commands. Once
 you edit those files, you can continue to call ipcluster/ipcontroller/ipengine
-with no arguments beyond ``-p myprofile``, and any configuration will be maintained.
+with no arguments beyond ``p=myprofile``, and any configuration will be maintained.
 
 There is no limit to the number of profiles you can have, so you can maintain a profile for each
 of your common use cases. The default profile will be used whenever the
@@ -112,7 +112,8 b' Using various batch systems with :command:`ipcluster`'
 
 :command:`ipcluster` has a notion of Launchers that can start controllers
 and engines with various remote execution schemes.  Currently supported
-models include `mpiexec`, PBS-style (Torque, SGE), and Windows HPC Server.
+models include :command:`ssh`, :command`mpiexec`, PBS-style (Torque, SGE),
+and Windows HPC Server.
 
 .. note::
 
@@ -132,7 +133,7 b' The mpiexec/mpirun mode is useful if you:'
 
 If these are satisfied, you can create a new profile::
 
-    $ ipcluster create -p mpi
+    $ ipcluster create profile=mpi
 
 and edit the file :file:`IPYTHONDIR/cluster_mpi/ipcluster_config.py`.
 
@@ -140,11 +141,11 b' There, instruct ipcluster to use the MPIExec launchers by adding the lines:'
 
 .. sourcecode:: python
 
-    c.Global.engine_launcher = 'IPython.parallel.apps.launcher.MPIExecEngineSetLauncher'
+    c.IPClusterEnginesApp.engine_launcher = 'IPython.parallel.apps.launcher.MPIExecEngineSetLauncher'
 
 If the default MPI configuration is correct, then you can now start your cluster, with::
 
-    $ ipcluster start -n 4 -p mpi
+    $ ipcluster start n=4 profile=mpi
 
 This does the following:
 
@@ -155,7 +156,7 b' If you have a reason to also start the Controller with mpi, you can specify:'
 
 .. sourcecode:: python
 
-    c.Global.controller_launcher = 'IPython.parallel.apps.launcher.MPIExecControllerLauncher'
+    c.IPClusterStartApp.controller_launcher = 'IPython.parallel.apps.launcher.MPIExecControllerLauncher'
 
 .. note::
 
@@ -189,7 +190,7 b' The PBS mode uses the Portable Batch System [PBS]_ to start the engines.'
 
 As usual, we will start by creating a fresh profile::
 
-    $ ipcluster create -p pbs
+    $ ipcluster create profile=pbs
 
 And in :file:`ipcluster_config.py`, we will select the PBS launchers for the controller
 and engines:
@@ -213,7 +214,7 b' to specify your own. Here is a sample PBS script template:'
     cd $$PBS_O_WORKDIR
     export PATH=$$HOME/usr/local/bin
     export PYTHONPATH=$$HOME/usr/local/lib/python2.7/site-packages
-    /usr/local/bin/mpiexec -n ${n} ipengine --cluster_dir=${cluster_dir}
+    /usr/local/bin/mpiexec -n ${n} ipengine cluster_dir=${cluster_dir}
 
 There are a few important points about this template:
 
@@ -251,7 +252,7 b' The controller template should be similar, but simpler:'
     cd $$PBS_O_WORKDIR
     export PATH=$$HOME/usr/local/bin
     export PYTHONPATH=$$HOME/usr/local/lib/python2.7/site-packages
-    ipcontroller --cluster_dir=${cluster_dir}
+    ipcontroller cluster_dir=${cluster_dir}
 
 
 Once you have created these scripts, save them with names like
@@ -287,7 +288,7 b' connections on all its interfaces, by adding in :file:`ipcontroller_config`:'
 
 You can now run the cluster with::
 
-    $ ipcluster start -p pbs -n 128
+    $ ipcluster start profile=pbs n=128
 
 Additional configuration options can be found in the PBS section of :file:`ipcluster_config`.
 
@@ -312,7 +313,7 b' nodes and :command:`ipcontroller` can be run remotely as well, or on localhost.'
 
 As usual, we start by creating a clean profile::
 
-    $ ipcluster create -p ssh
+    $ ipcluster create profile= ssh
 
 To use this mode, select the SSH launchers in :file:`ipcluster_config.py`:
 
@@ -334,7 +335,7 b" The controller's remote location and configuration can be specified:"
     # Set the arguments to be passed to ipcontroller
     # note that remotely launched ipcontroller will not get the contents of 
     # the local ipcontroller_config.py unless it resides on the *remote host*
-    # in the location specified by the --cluster_dir argument.
+    # in the location specified by the `cluster_dir` argument.
     # c.SSHControllerLauncher.program_args = ['-r', '-ip', '0.0.0.0', '--cluster_dir', '/path/to/cd']
 
 .. note::
@@ -351,7 +352,7 b' on that host.'
 
     c.SSHEngineSetLauncher.engines = { 'host1.example.com' : 2,
                 'host2.example.com' : 5,
-                'host3.example.com' : (1, ['--cluster_dir', '/home/different/location']),
+                'host3.example.com' : (1, ['cluster_dir=/home/different/location']),
                 'host4.example.com' : 8 }
 
 * The `engines` dict, where the keys are the host we want to run engines on and
@@ -452,10 +453,10 b' you want to unlock the door and enter your house. As with your house, you want'
 to be able to create the key (or JSON file) once, and then simply use it at
 any point in the future.
 
-To do this, the only thing you have to do is specify the `-r` flag, so that
+To do this, the only thing you have to do is specify the `--reuse` flag, so that
 the connection information in the JSON files remains accurate::
 
-    $ ipcontroller -r
+    $ ipcontroller --reuse
 
 Then, just copy the JSON files over the first time and you are set. You can
 start and stop the controller and engines any many times as you want in the

docs/source/parallel/parallel_task.txt

@@ -24,7 +24,7 b' To follow along with this tutorial, you will need to start the IPython'
 controller and four IPython engines. The simplest way of doing this is to use
 the :command:`ipcluster` command::
 
-	$ ipcluster start -n 4
+	$ ipcluster start n=4
 	
 For more detailed information about starting the controller and engines, see
 our :ref:`introduction <ip1par>` to using IPython for parallel computing.
@@ -342,17 +342,17 b' Schedulers'
 
 There are a variety of valid ways to determine where jobs should be assigned in a 
 load-balancing situation.  In IPython, we support several standard schemes, and
-even make it easy to define your own.  The scheme can be selected via the ``--scheme``
-argument to :command:`ipcontroller`, or in the :attr:`HubFactory.scheme` attribute
+even make it easy to define your own.  The scheme can be selected via the ``scheme``
+argument to :command:`ipcontroller`, or in the :attr:`TaskScheduler.schemename` attribute
 of a controller config object.
 
 The built-in routing schemes:
 
 To select one of these schemes, simply do::
 
-    $ ipcontroller --scheme <schemename>
+    $ ipcontroller scheme=<schemename>
     for instance:
-    $ ipcontroller --scheme lru
+    $ ipcontroller scheme=lru
 
 lru: Least Recently Used
 

docs/source/parallel/parallel_winhpc.txt

@@ -162,7 +162,7 b' cluster using the Windows HPC Server 2008 job scheduler. To make sure that'
 to start an IPython cluster on your local host. To do this, open a Windows
 Command Prompt and type the following command::
 
-    ipcluster start -n 2
+    ipcluster start n=2
 
 You should see a number of messages printed to the screen, ending with 
 "IPython cluster: started". The result should look something like the following
@@ -179,11 +179,11 b' describe how to configure and run an IPython cluster on an actual compute'
 cluster running Windows HPC Server 2008. Here is an outline of the needed
 steps:
 
-1. Create a cluster profile using: ``ipcluster create -p mycluster``
+1. Create a cluster profile using: ``ipcluster create profile=mycluster``
 
 2. Edit configuration files in the directory :file:`.ipython\\cluster_mycluster`
 
-3. Start the cluster using: ``ipcluser start -p mycluster -n 32``
+3. Start the cluster using: ``ipcluser start profile=mycluster n=32``
 
 Creating a cluster profile
 --------------------------
@@ -204,7 +204,7 b' security keys. The naming convention for cluster directories is:'
 To create a new cluster profile (named "mycluster") and the associated cluster
 directory, type the following command at the Windows Command Prompt::
 
-    ipcluster create -p mycluster
+    ipcluster create profile=mycluster
 
 The output of this command is shown in the screenshot below. Notice how
 :command:`ipcluster` prints out the location of the newly created cluster
@@ -257,7 +257,7 b' Starting the cluster profile'
 Once a cluster profile has been configured, starting an IPython cluster using
 the profile is simple::
 
-    ipcluster start -p mycluster -n 32
+    ipcluster start profile=mycluster n=32
 
 The ``-n`` option tells :command:`ipcluster` how many engines to start (in
 this case 32). Stopping the cluster is as simple as typing Control-C.