From 080d3c719b13875b59f599e584c566a721037ce0 2011-06-20 23:40:14 From: MinRK Date: 2011-06-20 23:40:14 Subject: [PATCH] parallel docs, tests, default config updated to newconfig --- diff --git a/IPython/parallel/apps/clusterdir.py b/IPython/parallel/apps/clusterdir.py index edf4375..3766d26 100755 --- a/IPython/parallel/apps/clusterdir.py +++ b/IPython/parallel/apps/clusterdir.py @@ -328,8 +328,8 @@ class ClusterApplication(BaseIPythonApplication): The cluster directory is resolved as follows: - * If the ``--cluster-dir`` option is given, it is used. - * If ``--cluster-dir`` is not given, the application directory is + * If the ``cluster_dir`` option is given, it is used. + * If ``cluster_dir`` is not given, the application directory is resolve using the profile name as ``cluster_``. The search path for this directory is then i) cwd if it is found there and ii) in ipython_dir otherwise. diff --git a/IPython/parallel/apps/ipclusterapp.py b/IPython/parallel/apps/ipclusterapp.py index a70740d..35a4cbd 100755 --- a/IPython/parallel/apps/ipclusterapp.py +++ b/IPython/parallel/apps/ipclusterapp.py @@ -46,8 +46,7 @@ from IPython.parallel.apps.clusterdir import ( default_config_file_name = u'ipcluster_config.py' -_description = """\ -Start an IPython cluster for parallel computing.\n\n +_description = """Start an IPython cluster for parallel computing. An IPython cluster consists of 1 controller and 1 or more engines. This command automates the startup of these processes using a wide @@ -78,7 +77,8 @@ NO_CLUSTER = 12 #----------------------------------------------------------------------------- # Main application #----------------------------------------------------------------------------- -start_help = """ +start_help = """Start an IPython cluster for parallel computing + Start an ipython cluster by its profile name or cluster directory. Cluster directories contain configuration, log and security related files and are named using the convention @@ -88,7 +88,8 @@ the cwd or the ipython directory, you can simply refer to it using its profile name, 'ipcluster start n=4 profile=`, otherwise use the 'cluster_dir' option. """ -stop_help = """ +stop_help = """Stop a running IPython cluster + Stop a running ipython cluster by its profile name or cluster directory. Cluster directories are named using the convention 'cluster_'. If your cluster directory is in @@ -96,7 +97,8 @@ the cwd or the ipython directory, you can simply refer to it using its profile name, 'ipcluster stop profile=`, otherwise use the 'cluster_dir' option. """ -engines_help = """ +engines_help = """Start engines connected to an existing IPython cluster + Start one or more engines to connect to an existing Cluster by profile name or cluster directory. Cluster directories contain configuration, log and @@ -107,7 +109,8 @@ the cwd or the ipython directory, you can simply refer to it using its profile name, 'ipcluster engines n=4 profile=`, otherwise use the 'cluster_dir' option. """ -create_help = """ +create_help = """Create an ipcluster profile by name + Create an ipython cluster directory by its profile name or cluster directory path. Cluster directories contain configuration, log and security related files and are named @@ -119,7 +122,9 @@ cluster directory by profile name, `ipcluster create profile=mycluster`, which will put the directory in `/cluster_mycluster`. """ -list_help = """List all available clusters, by cluster directory, that can +list_help = """List available cluster profiles + +List all available clusters, by cluster directory, that can be found in the current working directly or in the ipython directory. Cluster directories are named using the convention 'cluster_'. diff --git a/IPython/parallel/apps/ipcontrollerapp.py b/IPython/parallel/apps/ipcontrollerapp.py index 25d80cb..9495101 100755 --- a/IPython/parallel/apps/ipcontrollerapp.py +++ b/IPython/parallel/apps/ipcontrollerapp.py @@ -79,8 +79,8 @@ The IPython controller provides a gateway between the IPython engines and clients. The controller needs to be started before the engines and can be configured using command line options or using a cluster directory. Cluster directories contain config, log and security files and are usually located in -your ipython directory and named as "cluster_". See the --profile -and --cluster-dir options for details. +your ipython directory and named as "cluster_". See the `profile` +and `cluster_dir` options for details. """ @@ -92,14 +92,16 @@ and --cluster-dir options for details. flags = {} flags.update(base_flags) flags.update({ - 'usethreads' : ( {'IPControllerApp' : {'usethreads' : True}}, + 'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}}, 'Use threads instead of processes for the schedulers'), - 'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}}, + 'sqlitedb' : ({'HubFactory' : Config({'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'})}, 'use the SQLiteDB backend'), - 'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}}, + 'mongodb' : ({'HubFactory' : Config({'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'})}, 'use the MongoDB backend'), - 'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}}, + 'dictdb' : ({'HubFactory' : Config({'db_class' : 'IPython.parallel.controller.dictdb.DictDB'})}, 'use the in-memory DictDB backend'), + 'reuse' : ({'IPControllerApp' : Config({'reuse_files' : True})}, + 'reuse existing json connection files') }) flags.update() @@ -133,7 +135,7 @@ class IPControllerApp(ClusterApplication): help="import statements to be run at startup. Necessary in some environments" ) - usethreads = Bool(False, config=True, + use_threads = Bool(False, config=True, help='Use threads instead of processes for the schedulers', ) @@ -141,7 +143,7 @@ class IPControllerApp(ClusterApplication): children = List() mq_class = Unicode('zmq.devices.ProcessMonitoredQueue') - def _usethreads_changed(self, name, old, new): + def _use_threads_changed(self, name, old, new): self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process') aliases = Dict(dict( @@ -152,7 +154,7 @@ class IPControllerApp(ClusterApplication): reuse_files = 'IPControllerApp.reuse_files', secure = 'IPControllerApp.secure', ssh = 'IPControllerApp.ssh_server', - usethreads = 'IPControllerApp.usethreads', + use_threads = 'IPControllerApp.use_threads', import_statements = 'IPControllerApp.import_statements', location = 'IPControllerApp.location', @@ -271,7 +273,7 @@ class IPControllerApp(ClusterApplication): mq = import_item(str(self.mq_class)) hub = self.factory - # maybe_inproc = 'inproc://monitor' if self.usethreads else self.monitor_url + # maybe_inproc = 'inproc://monitor' if self.use_threads else self.monitor_url # IOPub relay (in a Process) q = mq(zmq.PUB, zmq.SUB, zmq.PUB, 'N/A','iopub') q.bind_in(hub.client_info['iopub']) diff --git a/IPython/parallel/apps/ipengineapp.py b/IPython/parallel/apps/ipengineapp.py index c6afee1..e04274a 100755 --- a/IPython/parallel/apps/ipengineapp.py +++ b/IPython/parallel/apps/ipengineapp.py @@ -46,7 +46,7 @@ from IPython.utils.traitlets import Bool, Unicode, Dict, List #: The default config file name for this application default_config_file_name = u'ipengine_config.py' -_description = """Start an IPython engine for parallel computing.\n\n +_description = """Start an IPython engine for parallel computing. IPython engines run in parallel and perform computations on behalf of a client and controller. A controller needs to be started before the engines. The diff --git a/IPython/parallel/apps/iploggerapp.py b/IPython/parallel/apps/iploggerapp.py index c592781..8c98692 100755 --- a/IPython/parallel/apps/iploggerapp.py +++ b/IPython/parallel/apps/iploggerapp.py @@ -36,14 +36,14 @@ from IPython.parallel.apps.logwatcher import LogWatcher #: The default config file name for this application default_config_file_name = u'iplogger_config.py' -_description = """Start an IPython logger for parallel computing.\n\n +_description = """Start an IPython logger for parallel computing. IPython controllers and engines (and your own processes) can broadcast log messages by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The logger can be configured using command line options or using a cluster directory. Cluster directories contain config, log and security files and are usually located in your ipython directory and named as "cluster_". -See the --profile and --cluster-dir options for details. +See the `profile` and `cluster_dir` options for details. """ diff --git a/IPython/parallel/engine/engine.py b/IPython/parallel/engine/engine.py index 891f31a..be7fd96 100755 --- a/IPython/parallel/engine/engine.py +++ b/IPython/parallel/engine/engine.py @@ -141,7 +141,6 @@ class EngineFactory(RegistrationFactory): self.kernel.start() hb_addrs = [ disambiguate_url(addr, self.location) for addr in hb_addrs ] heart = Heart(*map(str, hb_addrs), heart_id=identity) - # ioloop.DelayedCallback(heart.start, 1000, self.loop).start() heart.start() diff --git a/IPython/parallel/tests/__init__.py b/IPython/parallel/tests/__init__.py index ae517ed..51690cf 100644 --- a/IPython/parallel/tests/__init__.py +++ b/IPython/parallel/tests/__init__.py @@ -48,7 +48,7 @@ class TestProcessLauncher(LocalProcessLauncher): def setup(): cp = TestProcessLauncher() cp.cmd_and_args = ipcontroller_cmd_argv + \ - ['--profile', 'iptest', '--log-level', '99', '-r'] + ['profile=iptest', 'log_level=50', '--reuse'] cp.start() launchers.append(cp) cluster_dir = os.path.join(get_ipython_dir(), 'cluster_iptest') @@ -70,7 +70,7 @@ def add_engines(n=1, profile='iptest'): eps = [] for i in range(n): ep = TestProcessLauncher() - ep.cmd_and_args = ipengine_cmd_argv + ['--profile', profile, '--log-level', '99'] + ep.cmd_and_args = ipengine_cmd_argv + ['profile=%s'%profile, 'log_level=50'] ep.start() launchers.append(ep) eps.append(ep) diff --git a/IPython/parallel/tests/test_streamsession.py b/IPython/parallel/tests/test_streamsession.py index 891afe0..051c11c 100644 --- a/IPython/parallel/tests/test_streamsession.py +++ b/IPython/parallel/tests/test_streamsession.py @@ -48,11 +48,11 @@ class TestSession(SessionTestCase): self.assertTrue(s.unpack is ss.default_unpacker) self.assertEquals(s.username, os.environ.get('USER', 'username')) - s = ss.StreamSession(username=None) + s = ss.StreamSession() self.assertEquals(s.username, os.environ.get('USER', 'username')) - self.assertRaises(TypeError, ss.StreamSession, packer='hi') - self.assertRaises(TypeError, ss.StreamSession, unpacker='hi') + self.assertRaises(TypeError, ss.StreamSession, pack='hi') + self.assertRaises(TypeError, ss.StreamSession, unpack='hi') u = str(uuid.uuid4()) s = ss.StreamSession(username='carrot', session=u) self.assertEquals(s.session, u) diff --git a/docs/source/parallel/parallel_intro.txt b/docs/source/parallel/parallel_intro.txt index 507b047..4cc93ab 100644 --- a/docs/source/parallel/parallel_intro.txt +++ b/docs/source/parallel/parallel_intro.txt @@ -195,7 +195,7 @@ simply start a controller and engines on a single host using the :command:`ipcluster` command. To start a controller and 4 engines on your localhost, just do:: - $ ipcluster start -n 4 + $ ipcluster start n=4 More details about starting the IPython controller and engines can be found :ref:`here ` diff --git a/docs/source/parallel/parallel_mpi.txt b/docs/source/parallel/parallel_mpi.txt index e0fee30..df8f0e6 100644 --- a/docs/source/parallel/parallel_mpi.txt +++ b/docs/source/parallel/parallel_mpi.txt @@ -53,11 +53,11 @@ these things to happen. Automatic starting using :command:`mpiexec` and :command:`ipcluster` -------------------------------------------------------------------- -The easiest approach is to use the `mpiexec` mode of :command:`ipcluster`, +The easiest approach is to use the `MPIExec` Launchers in :command:`ipcluster`, which will first start a controller and then a set of engines using :command:`mpiexec`:: - $ ipcluster mpiexec -n 4 + $ ipcluster start n=4 elauncher=MPIExecEngineSetLauncher This approach is best as interrupting :command:`ipcluster` will automatically stop and clean up the controller and engines. @@ -68,14 +68,14 @@ Manual starting using :command:`mpiexec` If you want to start the IPython engines using the :command:`mpiexec`, just do:: - $ mpiexec -n 4 ipengine --mpi=mpi4py + $ mpiexec n=4 ipengine mpi=mpi4py This requires that you already have a controller running and that the FURL files for the engines are in place. We also have built in support for PyTrilinos [PyTrilinos]_, which can be used (assuming is installed) by starting the engines with:: - $ mpiexec -n 4 ipengine --mpi=pytrilinos + $ mpiexec n=4 ipengine mpi=pytrilinos Automatic starting using PBS and :command:`ipcluster` ------------------------------------------------------ @@ -110,7 +110,7 @@ distributed array. Save the following text in a file called :file:`psum.py`: Now, start an IPython cluster:: - $ ipcluster start -p mpi -n 4 + $ ipcluster start profile=mpi n=4 .. note:: diff --git a/docs/source/parallel/parallel_multiengine.txt b/docs/source/parallel/parallel_multiengine.txt index b25bc9f..a1b394a 100644 --- a/docs/source/parallel/parallel_multiengine.txt +++ b/docs/source/parallel/parallel_multiengine.txt @@ -19,7 +19,7 @@ To follow along with this tutorial, you will need to start the IPython controller and four IPython engines. The simplest way of doing this is to use the :command:`ipcluster` command:: - $ ipcluster start -n 4 + $ ipcluster start n=4 For more detailed information about starting the controller and engines, see our :ref:`introduction ` to using IPython for parallel computing. diff --git a/docs/source/parallel/parallel_process.txt b/docs/source/parallel/parallel_process.txt index f02ebf8..944e5fc 100644 --- a/docs/source/parallel/parallel_process.txt +++ b/docs/source/parallel/parallel_process.txt @@ -57,7 +57,7 @@ controller and engines in the following situations: 1. When the controller and engines are all run on localhost. This is useful for testing or running on a multicore computer. -2. When engines are started using the :command:`mpirun` command that comes +2. When engines are started using the :command:`mpiexec` command that comes with most MPI [MPI]_ implementations 3. When engines are started using the PBS [PBS]_ batch system (or other `qsub` systems, such as SGE). @@ -80,9 +80,9 @@ The simplest way to use ipcluster requires no configuration, and will launch a controller and a number of engines on the local machine. For instance, to start one controller and 4 engines on localhost, just do:: - $ ipcluster start -n 4 + $ ipcluster start n=4 -To see other command line options for the local mode, do:: +To see other command line options, do:: $ ipcluster -h @@ -92,12 +92,12 @@ Configuring an IPython cluster Cluster configurations are stored as `profiles`. You can create a new profile with:: - $ ipcluster create -p myprofile + $ ipcluster create profile=myprofile This will create the directory :file:`IPYTHONDIR/cluster_myprofile`, and populate it with the default configuration files for the three IPython cluster commands. Once you edit those files, you can continue to call ipcluster/ipcontroller/ipengine -with no arguments beyond ``-p myprofile``, and any configuration will be maintained. +with no arguments beyond ``p=myprofile``, and any configuration will be maintained. There is no limit to the number of profiles you can have, so you can maintain a profile for each of your common use cases. The default profile will be used whenever the @@ -112,7 +112,8 @@ Using various batch systems with :command:`ipcluster` :command:`ipcluster` has a notion of Launchers that can start controllers and engines with various remote execution schemes. Currently supported -models include `mpiexec`, PBS-style (Torque, SGE), and Windows HPC Server. +models include :command:`ssh`, :command`mpiexec`, PBS-style (Torque, SGE), +and Windows HPC Server. .. note:: @@ -132,7 +133,7 @@ The mpiexec/mpirun mode is useful if you: If these are satisfied, you can create a new profile:: - $ ipcluster create -p mpi + $ ipcluster create profile=mpi and edit the file :file:`IPYTHONDIR/cluster_mpi/ipcluster_config.py`. @@ -140,11 +141,11 @@ There, instruct ipcluster to use the MPIExec launchers by adding the lines: .. sourcecode:: python - c.Global.engine_launcher = 'IPython.parallel.apps.launcher.MPIExecEngineSetLauncher' + c.IPClusterEnginesApp.engine_launcher = 'IPython.parallel.apps.launcher.MPIExecEngineSetLauncher' If the default MPI configuration is correct, then you can now start your cluster, with:: - $ ipcluster start -n 4 -p mpi + $ ipcluster start n=4 profile=mpi This does the following: @@ -155,7 +156,7 @@ If you have a reason to also start the Controller with mpi, you can specify: .. sourcecode:: python - c.Global.controller_launcher = 'IPython.parallel.apps.launcher.MPIExecControllerLauncher' + c.IPClusterStartApp.controller_launcher = 'IPython.parallel.apps.launcher.MPIExecControllerLauncher' .. note:: @@ -189,7 +190,7 @@ The PBS mode uses the Portable Batch System [PBS]_ to start the engines. As usual, we will start by creating a fresh profile:: - $ ipcluster create -p pbs + $ ipcluster create profile=pbs And in :file:`ipcluster_config.py`, we will select the PBS launchers for the controller and engines: @@ -213,7 +214,7 @@ to specify your own. Here is a sample PBS script template: cd $$PBS_O_WORKDIR export PATH=$$HOME/usr/local/bin export PYTHONPATH=$$HOME/usr/local/lib/python2.7/site-packages - /usr/local/bin/mpiexec -n ${n} ipengine --cluster_dir=${cluster_dir} + /usr/local/bin/mpiexec -n ${n} ipengine cluster_dir=${cluster_dir} There are a few important points about this template: @@ -251,7 +252,7 @@ The controller template should be similar, but simpler: cd $$PBS_O_WORKDIR export PATH=$$HOME/usr/local/bin export PYTHONPATH=$$HOME/usr/local/lib/python2.7/site-packages - ipcontroller --cluster_dir=${cluster_dir} + ipcontroller cluster_dir=${cluster_dir} Once you have created these scripts, save them with names like @@ -287,7 +288,7 @@ connections on all its interfaces, by adding in :file:`ipcontroller_config`: You can now run the cluster with:: - $ ipcluster start -p pbs -n 128 + $ ipcluster start profile=pbs n=128 Additional configuration options can be found in the PBS section of :file:`ipcluster_config`. @@ -312,7 +313,7 @@ nodes and :command:`ipcontroller` can be run remotely as well, or on localhost. As usual, we start by creating a clean profile:: - $ ipcluster create -p ssh + $ ipcluster create profile= ssh To use this mode, select the SSH launchers in :file:`ipcluster_config.py`: @@ -334,7 +335,7 @@ The controller's remote location and configuration can be specified: # Set the arguments to be passed to ipcontroller # note that remotely launched ipcontroller will not get the contents of # the local ipcontroller_config.py unless it resides on the *remote host* - # in the location specified by the --cluster_dir argument. + # in the location specified by the `cluster_dir` argument. # c.SSHControllerLauncher.program_args = ['-r', '-ip', '0.0.0.0', '--cluster_dir', '/path/to/cd'] .. note:: @@ -351,7 +352,7 @@ on that host. c.SSHEngineSetLauncher.engines = { 'host1.example.com' : 2, 'host2.example.com' : 5, - 'host3.example.com' : (1, ['--cluster_dir', '/home/different/location']), + 'host3.example.com' : (1, ['cluster_dir=/home/different/location']), 'host4.example.com' : 8 } * The `engines` dict, where the keys are the host we want to run engines on and @@ -452,10 +453,10 @@ you want to unlock the door and enter your house. As with your house, you want to be able to create the key (or JSON file) once, and then simply use it at any point in the future. -To do this, the only thing you have to do is specify the `-r` flag, so that +To do this, the only thing you have to do is specify the `--reuse` flag, so that the connection information in the JSON files remains accurate:: - $ ipcontroller -r + $ ipcontroller --reuse Then, just copy the JSON files over the first time and you are set. You can start and stop the controller and engines any many times as you want in the diff --git a/docs/source/parallel/parallel_task.txt b/docs/source/parallel/parallel_task.txt index e819d63..c3c18ed 100644 --- a/docs/source/parallel/parallel_task.txt +++ b/docs/source/parallel/parallel_task.txt @@ -24,7 +24,7 @@ To follow along with this tutorial, you will need to start the IPython controller and four IPython engines. The simplest way of doing this is to use the :command:`ipcluster` command:: - $ ipcluster start -n 4 + $ ipcluster start n=4 For more detailed information about starting the controller and engines, see our :ref:`introduction ` to using IPython for parallel computing. @@ -342,17 +342,17 @@ Schedulers There are a variety of valid ways to determine where jobs should be assigned in a load-balancing situation. In IPython, we support several standard schemes, and -even make it easy to define your own. The scheme can be selected via the ``--scheme`` -argument to :command:`ipcontroller`, or in the :attr:`HubFactory.scheme` attribute +even make it easy to define your own. The scheme can be selected via the ``scheme`` +argument to :command:`ipcontroller`, or in the :attr:`TaskScheduler.schemename` attribute of a controller config object. The built-in routing schemes: To select one of these schemes, simply do:: - $ ipcontroller --scheme + $ ipcontroller scheme= for instance: - $ ipcontroller --scheme lru + $ ipcontroller scheme=lru lru: Least Recently Used diff --git a/docs/source/parallel/parallel_winhpc.txt b/docs/source/parallel/parallel_winhpc.txt index 44ea429..2b0238d 100644 --- a/docs/source/parallel/parallel_winhpc.txt +++ b/docs/source/parallel/parallel_winhpc.txt @@ -162,7 +162,7 @@ cluster using the Windows HPC Server 2008 job scheduler. To make sure that to start an IPython cluster on your local host. To do this, open a Windows Command Prompt and type the following command:: - ipcluster start -n 2 + ipcluster start n=2 You should see a number of messages printed to the screen, ending with "IPython cluster: started". The result should look something like the following @@ -179,11 +179,11 @@ describe how to configure and run an IPython cluster on an actual compute cluster running Windows HPC Server 2008. Here is an outline of the needed steps: -1. Create a cluster profile using: ``ipcluster create -p mycluster`` +1. Create a cluster profile using: ``ipcluster create profile=mycluster`` 2. Edit configuration files in the directory :file:`.ipython\\cluster_mycluster` -3. Start the cluster using: ``ipcluser start -p mycluster -n 32`` +3. Start the cluster using: ``ipcluser start profile=mycluster n=32`` Creating a cluster profile -------------------------- @@ -204,7 +204,7 @@ security keys. The naming convention for cluster directories is: To create a new cluster profile (named "mycluster") and the associated cluster directory, type the following command at the Windows Command Prompt:: - ipcluster create -p mycluster + ipcluster create profile=mycluster The output of this command is shown in the screenshot below. Notice how :command:`ipcluster` prints out the location of the newly created cluster @@ -257,7 +257,7 @@ Starting the cluster profile Once a cluster profile has been configured, starting an IPython cluster using the profile is simple:: - ipcluster start -p mycluster -n 32 + ipcluster start profile=mycluster n=32 The ``-n`` option tells :command:`ipcluster` how many engines to start (in this case 32). Stopping the cluster is as simple as typing Control-C.