##// END OF EJS Templates
rebase IPython.parallel after removal of IPython.kernel...
MinRK -
Show More
@@ -11,8 +11,8 b' c = get_config()'
11 # - Start as a regular process on localhost.
11 # - Start as a regular process on localhost.
12 # - Start using mpiexec.
12 # - Start using mpiexec.
13 # - Start using the Windows HPC Server 2008 scheduler
13 # - Start using the Windows HPC Server 2008 scheduler
14 # - Start using PBS
14 # - Start using PBS/SGE
15 # - Start using SSH (currently broken)
15 # - Start using SSH
16
16
17
17
18 # The selected launchers can be configured below.
18 # The selected launchers can be configured below.
@@ -21,15 +21,18 b' c = get_config()'
21 # - LocalControllerLauncher
21 # - LocalControllerLauncher
22 # - MPIExecControllerLauncher
22 # - MPIExecControllerLauncher
23 # - PBSControllerLauncher
23 # - PBSControllerLauncher
24 # - SGEControllerLauncher
24 # - WindowsHPCControllerLauncher
25 # - WindowsHPCControllerLauncher
25 # c.Global.controller_launcher = 'IPython.kernel.launcher.LocalControllerLauncher'
26 # c.Global.controller_launcher = 'IPython.parallel.launcher.LocalControllerLauncher'
27 # c.Global.controller_launcher = 'IPython.parallel.launcher.PBSControllerLauncher'
26
28
27 # Options are:
29 # Options are:
28 # - LocalEngineSetLauncher
30 # - LocalEngineSetLauncher
29 # - MPIExecEngineSetLauncher
31 # - MPIExecEngineSetLauncher
30 # - PBSEngineSetLauncher
32 # - PBSEngineSetLauncher
33 # - SGEEngineSetLauncher
31 # - WindowsHPCEngineSetLauncher
34 # - WindowsHPCEngineSetLauncher
32 # c.Global.engine_launcher = 'IPython.kernel.launcher.LocalEngineSetLauncher'
35 # c.Global.engine_launcher = 'IPython.parallel.launcher.LocalEngineSetLauncher'
33
36
34 #-----------------------------------------------------------------------------
37 #-----------------------------------------------------------------------------
35 # Global configuration
38 # Global configuration
@@ -68,23 +71,23 b' c = get_config()'
68 # MPIExec launchers
71 # MPIExec launchers
69 #-----------------------------------------------------------------------------
72 #-----------------------------------------------------------------------------
70
73
71 # The mpiexec/mpirun command to use in started the controller.
74 # The mpiexec/mpirun command to use in both the controller and engines.
72 # c.MPIExecControllerLauncher.mpi_cmd = ['mpiexec']
75 # c.MPIExecLauncher.mpi_cmd = ['mpiexec']
73
76
74 # Additional arguments to pass to the actual mpiexec command.
77 # Additional arguments to pass to the actual mpiexec command.
78 # c.MPIExecLauncher.mpi_args = []
79
80 # The mpiexec/mpirun command and args can be overridden if they should be different
81 # for controller and engines.
82 # c.MPIExecControllerLauncher.mpi_cmd = ['mpiexec']
75 # c.MPIExecControllerLauncher.mpi_args = []
83 # c.MPIExecControllerLauncher.mpi_args = []
84 # c.MPIExecEngineSetLauncher.mpi_cmd = ['mpiexec']
85 # c.MPIExecEngineSetLauncher.mpi_args = []
76
86
77 # The command line argument to call the controller with.
87 # The command line argument to call the controller with.
78 # c.MPIExecControllerLauncher.controller_args = \
88 # c.MPIExecControllerLauncher.controller_args = \
79 # ['--log-to-file','--log-level', '40']
89 # ['--log-to-file','--log-level', '40']
80
90
81
82 # The mpiexec/mpirun command to use in started the controller.
83 # c.MPIExecEngineSetLauncher.mpi_cmd = ['mpiexec']
84
85 # Additional arguments to pass to the actual mpiexec command.
86 # c.MPIExecEngineSetLauncher.mpi_args = []
87
88 # Command line argument passed to the engines.
91 # Command line argument passed to the engines.
89 # c.MPIExecEngineSetLauncher.engine_args = ['--log-to-file','--log-level', '40']
92 # c.MPIExecEngineSetLauncher.engine_args = ['--log-to-file','--log-level', '40']
90
93
@@ -95,51 +98,105 b' c = get_config()'
95 # SSH launchers
98 # SSH launchers
96 #-----------------------------------------------------------------------------
99 #-----------------------------------------------------------------------------
97
100
98 # Todo
101 # ipclusterz can be used to launch controller and engines remotely via ssh.
102 # Note that currently ipclusterz does not do any file distribution, so if
103 # machines are not on a shared filesystem, config and json files must be
104 # distributed. For this reason, the reuse_files defaults to True on an
105 # ssh-launched Controller. This flag can be overridded by the program_args
106 # attribute of c.SSHControllerLauncher.
107
108 # set the ssh cmd for launching remote commands. The default is ['ssh']
109 # c.SSHLauncher.ssh_cmd = ['ssh']
110
111 # set the ssh cmd for launching remote commands. The default is ['ssh']
112 # c.SSHLauncher.ssh_args = ['tt']
113
114 # Set the user and hostname for the controller
115 # c.SSHControllerLauncher.hostname = 'controller.example.com'
116 # c.SSHControllerLauncher.user = os.environ.get('USER','username')
117
118 # Set the arguments to be passed to ipcontrollerz
119 # note that remotely launched ipcontrollerz will not get the contents of
120 # the local ipcontrollerz_config.py unless it resides on the *remote host*
121 # in the location specified by the --cluster_dir argument.
122 # c.SSHControllerLauncher.program_args = ['-r', '-ip', '0.0.0.0', '--cluster_dir', '/path/to/cd']
123
124 # Set the default args passed to ipenginez for SSH launched engines
125 # c.SSHEngineSetLauncher.engine_args = ['--mpi', 'mpi4py']
99
126
127 # SSH engines are launched as a dict of locations/n-engines.
128 # if a value is a tuple instead of an int, it is assumed to be of the form
129 # (n, [args]), setting the arguments to passed to ipenginez on `host`.
130 # otherwise, c.SSHEngineSetLauncher.engine_args will be used as the default.
131
132 # In this case, there will be 3 engines at my.example.com, and
133 # 2 at you@ipython.scipy.org with a special json connector location.
134 # c.SSHEngineSetLauncher.engines = {'my.example.com' : 3,
135 # 'you@ipython.scipy.org' : (2, ['-f', '/path/to/ipcontroller-engine.json']}
136 # }
100
137
101 #-----------------------------------------------------------------------------
138 #-----------------------------------------------------------------------------
102 # Unix batch (PBS) schedulers launchers
139 # Unix batch (PBS) schedulers launchers
103 #-----------------------------------------------------------------------------
140 #-----------------------------------------------------------------------------
104
141
142 # SGE and PBS are very similar. All configurables in this section called 'PBS*'
143 # also exist as 'SGE*'.
144
105 # The command line program to use to submit a PBS job.
145 # The command line program to use to submit a PBS job.
106 # c.PBSControllerLauncher.submit_command = 'qsub'
146 # c.PBSLauncher.submit_command = ['qsub']
107
147
108 # The command line program to use to delete a PBS job.
148 # The command line program to use to delete a PBS job.
109 # c.PBSControllerLauncher.delete_command = 'qdel'
149 # c.PBSLauncher.delete_command = ['qdel']
150
151 # The PBS queue in which the job should run
152 # c.PBSLauncher.queue = 'myqueue'
110
153
111 # A regular expression that takes the output of qsub and find the job id.
154 # A regular expression that takes the output of qsub and find the job id.
112 # c.PBSControllerLauncher.job_id_regexp = r'\d+'
155 # c.PBSLauncher.job_id_regexp = r'\d+'
156
157 # If for some reason the Controller and Engines have different options above, they
158 # can be set as c.PBSControllerLauncher.<option> etc.
159
160 # PBS and SGE have default templates, but you can specify your own, either as strings
161 # or from files, as described here:
113
162
114 # The batch submission script used to start the controller. This is where
163 # The batch submission script used to start the controller. This is where
115 # environment variables would be setup, etc. This string is interpolated using
164 # environment variables would be setup, etc. This string is interpreted using
116 # the Itpl module in IPython.external. Basically, you can use ${n} for the
165 # the Itpl module in IPython.external. Basically, you can use ${n} for the
117 # number of engine and ${cluster_dir} for the cluster_dir.
166 # number of engine and ${cluster_dir} for the cluster_dir.
118 # c.PBSControllerLauncher.batch_template = """"""
167 # c.PBSControllerLauncher.batch_template = """
168 # #PBS -N ipcontroller
169 # #PBS -q $queue
170 #
171 # ipcontrollerz --cluster-dir $cluster_dir
172 # """
173
174 # You can also load this template from a file
175 # c.PBSControllerLauncher.batch_template_file = u"/path/to/my/template.sh"
119
176
120 # The name of the instantiated batch script that will actually be used to
177 # The name of the instantiated batch script that will actually be used to
121 # submit the job. This will be written to the cluster directory.
178 # submit the job. This will be written to the cluster directory.
122 # c.PBSControllerLauncher.batch_file_name = u'pbs_batch_script_controller'
179 # c.PBSControllerLauncher.batch_file_name = u'pbs_controller'
123
124
125 # The command line program to use to submit a PBS job.
126 # c.PBSEngineSetLauncher.submit_command = 'qsub'
127
128 # The command line program to use to delete a PBS job.
129 # c.PBSEngineSetLauncher.delete_command = 'qdel'
130
131 # A regular expression that takes the output of qsub and find the job id.
132 # c.PBSEngineSetLauncher.job_id_regexp = r'\d+'
133
180
134 # The batch submission script used to start the engines. This is where
181 # The batch submission script used to start the engines. This is where
135 # environment variables would be setup, etc. This string is interpolated using
182 # environment variables would be setup, etc. This string is interpreted using
136 # the Itpl module in IPython.external. Basically, you can use ${n} for the
183 # the Itpl module in IPython.external. Basically, you can use ${n} for the
137 # number of engine and ${cluster_dir} for the cluster_dir.
184 # number of engine and ${cluster_dir} for the cluster_dir.
138 # c.PBSEngineSetLauncher.batch_template = """"""
185 # c.PBSEngineSetLauncher.batch_template = """
186 # #PBS -N ipcontroller
187 # #PBS -l nprocs=$n
188 #
189 # ipenginez --cluster-dir $cluster_dir$s
190 # """
191
192 # You can also load this template from a file
193 # c.PBSControllerLauncher.batch_template_file = u"/path/to/my/template.sh"
139
194
140 # The name of the instantiated batch script that will actually be used to
195 # The name of the instantiated batch script that will actually be used to
141 # submit the job. This will be written to the cluster directory.
196 # submit the job. This will be written to the cluster directory.
142 # c.PBSEngineSetLauncher.batch_file_name = u'pbs_batch_script_engines'
197 # c.PBSEngineSetLauncher.batch_file_name = u'pbs_engines'
198
199
143
200
144 #-----------------------------------------------------------------------------
201 #-----------------------------------------------------------------------------
145 # Windows HPC Server 2008 launcher configuration
202 # Windows HPC Server 2008 launcher configuration
@@ -25,112 +25,156 b' c = get_config()'
25 # be imported in the controller for pickling to work.
25 # be imported in the controller for pickling to work.
26 # c.Global.import_statements = ['import math']
26 # c.Global.import_statements = ['import math']
27
27
28 # Reuse the controller's FURL files. If False, FURL files are regenerated
28 # Reuse the controller's JSON files. If False, JSON files are regenerated
29 # each time the controller is run. If True, they will be reused, *but*, you
29 # each time the controller is run. If True, they will be reused, *but*, you
30 # also must set the network ports by hand. If set, this will override the
30 # also must set the network ports by hand. If set, this will override the
31 # values set for the client and engine connections below.
31 # values set for the client and engine connections below.
32 # c.Global.reuse_furls = True
32 # c.Global.reuse_files = True
33
33
34 # Enable SSL encryption on all connections to the controller. If set, this
34 # Enable exec_key authentication on all messages. Default is True
35 # will override the values set for the client and engine connections below.
36 # c.Global.secure = True
35 # c.Global.secure = True
37
36
38 # The working directory for the process. The application will use os.chdir
37 # The working directory for the process. The application will use os.chdir
39 # to change to this directory before starting.
38 # to change to this directory before starting.
40 # c.Global.work_dir = os.getcwd()
39 # c.Global.work_dir = os.getcwd()
41
40
41 # The log url for logging to an `iploggerz` application. This will override
42 # log-to-file.
43 # c.Global.log_url = 'tcp://127.0.0.1:20202'
44
45 # The specific external IP that is used to disambiguate multi-interface URLs.
46 # The default behavior is to guess from external IPs gleaned from `socket`.
47 # c.Global.location = '192.168.1.123'
48
49 # The ssh server remote clients should use to connect to this controller.
50 # It must be a machine that can see the interface specified in client_ip.
51 # The default for client_ip is localhost, in which case the sshserver must
52 # be an external IP of the controller machine.
53 # c.Global.sshserver = 'controller.example.com'
54
55 # the url to use for registration. If set, this overrides engine-ip,
56 # engine-transport client-ip,client-transport, and regport.
57 # c.RegistrationFactory.url = 'tcp://*:12345'
58
59 # the port to use for registration. Clients and Engines both use this
60 # port for registration.
61 # c.RegistrationFactory.regport = 10101
62
42 #-----------------------------------------------------------------------------
63 #-----------------------------------------------------------------------------
43 # Configure the client services
64 # Configure the Task Scheduler
44 #-----------------------------------------------------------------------------
65 #-----------------------------------------------------------------------------
45
66
46 # Basic client service config attributes
67 # The routing scheme. 'pure' will use the pure-ZMQ scheduler. Any other
68 # value will use a Python scheduler with various routing schemes.
69 # python schemes are: lru, weighted, random, twobin. Default is 'weighted'.
70 # Note that the pure ZMQ scheduler does not support many features, such as
71 # dying engines, dependencies, or engine-subset load-balancing.
72 # c.ControllerFactory.scheme = 'pure'
47
73
48 # The network interface the controller will listen on for client connections.
74 # The pure ZMQ scheduler can limit the number of outstanding tasks per engine
49 # This should be an IP address or hostname of the controller's host. The empty
75 # by using the ZMQ HWM option. This allows engines with long-running tasks
50 # string means listen on all interfaces.
76 # to not steal too many tasks from other engines. The default is 0, which
51 # c.FCClientServiceFactory.ip = ''
77 # means agressively distribute messages, never waiting for them to finish.
78 # c.ControllerFactory.hwm = 1
52
79
53 # The TCP/IP port the controller will listen on for client connections. If 0
80 # Whether to use Threads or Processes to start the Schedulers. Threads will
54 # a random port will be used. If the controller's host has a firewall running
81 # use less resources, but potentially reduce throughput. Default is to
55 # it must allow incoming traffic on this port.
82 # use processes. Note that the a Python scheduler will always be in a Process.
56 # c.FCClientServiceFactory.port = 0
83 # c.ControllerFactory.usethreads
57
84
58 # The client learns how to connect to the controller by looking at the
85 #-----------------------------------------------------------------------------
59 # location field embedded in the FURL. If this field is empty, all network
86 # Configure the Hub
60 # interfaces that the controller is listening on will be listed. To have the
87 #-----------------------------------------------------------------------------
61 # client connect on a particular interface, list it here.
88
62 # c.FCClientServiceFactory.location = ''
89 # Which class to use for the db backend. Currently supported are DictDB (the
90 # default), and MongoDB. Uncomment this line to enable MongoDB, which will
91 # slow-down the Hub's responsiveness, but also reduce its memory footprint.
92 # c.HubFactory.db_class = 'IPython.parallel.mongodb.MongoDB'
63
93
64 # Use SSL encryption for the client connection.
94 # The heartbeat ping frequency. This is the frequency (in ms) at which the
65 # c.FCClientServiceFactory.secure = True
95 # Hub pings engines for heartbeats. This determines how quickly the Hub
96 # will react to engines coming and going. A lower number means faster response
97 # time, but more network activity. The default is 100ms
98 # c.HubFactory.ping = 100
66
99
67 # Reuse the client FURL each time the controller is started. If set, you must
100 # HubFactory queue port pairs, to set by name: mux, iopub, control, task. Set
68 # also pick a specific network port above (FCClientServiceFactory.port).
101 # each as a tuple of length 2 of ints. The default is to find random
69 # c.FCClientServiceFactory.reuse_furls = False
102 # available ports
103 # c.HubFactory.mux = (10102,10112)
70
104
71 #-----------------------------------------------------------------------------
105 #-----------------------------------------------------------------------------
72 # Configure the engine services
106 # Configure the client connections
73 #-----------------------------------------------------------------------------
107 #-----------------------------------------------------------------------------
74
108
75 # Basic config attributes for the engine services.
109 # Basic client connection config attributes
76
110
77 # The network interface the controller will listen on for engine connections.
111 # The network interface the controller will listen on for client connections.
78 # This should be an IP address or hostname of the controller's host. The empty
112 # This should be an IP address or interface on the controller. An asterisk
79 # string means listen on all interfaces.
113 # means listen on all interfaces. The transport can be any transport
80 # c.FCEngineServiceFactory.ip = ''
114 # supported by zeromq (tcp,epgm,pgm,ib,ipc):
115 # c.HubFactory.client_ip = '*'
116 # c.HubFactory.client_transport = 'tcp'
81
117
82 # The TCP/IP port the controller will listen on for engine connections. If 0
118 # individual client ports to configure by name: query_port, notifier_port
83 # a random port will be used. If the controller's host has a firewall running
119 # c.HubFactory.query_port = 12345
84 # it must allow incoming traffic on this port.
85 # c.FCEngineServiceFactory.port = 0
86
120
87 # The engine learns how to connect to the controller by looking at the
121 #-----------------------------------------------------------------------------
88 # location field embedded in the FURL. If this field is empty, all network
122 # Configure the engine connections
89 # interfaces that the controller is listening on will be listed. To have the
123 #-----------------------------------------------------------------------------
90 # client connect on a particular interface, list it here.
91 # c.FCEngineServiceFactory.location = ''
92
124
93 # Use SSL encryption for the engine connection.
125 # Basic config attributes for the engine connections.
94 # c.FCEngineServiceFactory.secure = True
95
126
96 # Reuse the client FURL each time the controller is started. If set, you must
127 # The network interface the controller will listen on for engine connections.
97 # also pick a specific network port above (FCClientServiceFactory.port).
128 # This should be an IP address or interface on the controller. An asterisk
98 # c.FCEngineServiceFactory.reuse_furls = False
129 # means listen on all interfaces. The transport can be any transport
130 # supported by zeromq (tcp,epgm,pgm,ib,ipc):
131 # c.HubFactory.engine_ip = '*'
132 # c.HubFactory.engine_transport = 'tcp'
133
134 # set the engine heartbeat ports to use:
135 # c.HubFactory.hb = (10303,10313)
99
136
100 #-----------------------------------------------------------------------------
137 #-----------------------------------------------------------------------------
101 # Developer level configuration attributes
138 # Configure the TaskRecord database backend
102 #-----------------------------------------------------------------------------
139 #-----------------------------------------------------------------------------
103
140
104 # You shouldn't have to modify anything in this section. These attributes
141 # For memory/persistance reasons, tasks can be stored out-of-memory in a database.
105 # are more for developers who want to change the behavior of the controller
142 # Currently, only sqlite and mongodb are supported as backends, but the interface
106 # at a fundamental level.
143 # is fairly simple, so advanced developers could write their own backend.
107
144
108 # c.FCClientServiceFactory.cert_file = u'ipcontroller-client.pem'
145 # ----- in-memory configuration --------
109
146 # this line restores the default behavior: in-memory storage of all results.
110 # default_client_interfaces = Config()
147 # c.HubFactory.db_class = 'IPython.parallel.dictdb.DictDB'
111 # default_client_interfaces.Task.interface_chain = [
148
112 # 'IPython.kernel.task.ITaskController',
149 # ----- sqlite configuration --------
113 # 'IPython.kernel.taskfc.IFCTaskController'
150 # use this line to activate sqlite:
114 # ]
151 # c.HubFactory.db_class = 'IPython.parallel.sqlitedb.SQLiteDB'
115 #
152
116 # default_client_interfaces.Task.furl_file = u'ipcontroller-tc.furl'
153 # You can specify the name of the db-file. By default, this will be located
117 #
154 # in the active cluster_dir, e.g. ~/.ipython/clusterz_default/tasks.db
118 # default_client_interfaces.MultiEngine.interface_chain = [
155 # c.SQLiteDB.filename = 'tasks.db'
119 # 'IPython.kernel.multiengine.IMultiEngine',
156
120 # 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
157 # You can also specify the location of the db-file, if you want it to be somewhere
121 # ]
158 # other than the cluster_dir.
122 #
159 # c.SQLiteDB.location = '/scratch/'
123 # default_client_interfaces.MultiEngine.furl_file = u'ipcontroller-mec.furl'
160
124 #
161 # This will specify the name of the table for the controller to use. The default
125 # c.FCEngineServiceFactory.interfaces = default_client_interfaces
162 # behavior is to use the session ID of the SessionFactory object (a uuid). Overriding
126
163 # this will result in results persisting for multiple sessions.
127 # c.FCEngineServiceFactory.cert_file = u'ipcontroller-engine.pem'
164 # c.SQLiteDB.table = 'results'
128
165
129 # default_engine_interfaces = Config()
166 # ----- mongodb configuration --------
130 # default_engine_interfaces.Default.interface_chain = [
167 # use this line to activate mongodb:
131 # 'IPython.kernel.enginefc.IFCControllerBase'
168 # c.HubFactory.db_class = 'IPython.parallel.mongodb.MongoDB'
132 # ]
169
133 #
170 # You can specify the args and kwargs pymongo will use when creating the Connection.
134 # default_engine_interfaces.Default.furl_file = u'ipcontroller-engine.furl'
171 # For more information on what these options might be, see pymongo documentation.
135 #
172 # c.MongoDB.connection_kwargs = {}
136 # c.FCEngineServiceFactory.interfaces = default_engine_interfaces
173 # c.MongoDB.connection_args = []
174
175 # This will specify the name of the mongo database for the controller to use. The default
176 # behavior is to use the session ID of the SessionFactory object (a uuid). Overriding
177 # this will result in task results persisting through multiple sessions.
178 # c.MongoDB.database = 'ipythondb'
179
180
@@ -29,10 +29,10 b' c = get_config()'
29 # c.Global.connect_delay = 0.1
29 # c.Global.connect_delay = 0.1
30 # c.Global.connect_max_tries = 15
30 # c.Global.connect_max_tries = 15
31
31
32 # By default, the engine will look for the controller's FURL file in its own
32 # By default, the engine will look for the controller's JSON file in its own
33 # cluster directory. Sometimes, the FURL file will be elsewhere and this
33 # cluster directory. Sometimes, the JSON file will be elsewhere and this
34 # attribute can be set to the full path of the FURL file.
34 # attribute can be set to the full path of the JSON file.
35 # c.Global.furl_file = u''
35 # c.Global.url_file = u'/path/to/my/ipcontroller-engine.json'
36
36
37 # The working directory for the process. The application will use os.chdir
37 # The working directory for the process. The application will use os.chdir
38 # to change to this directory before starting.
38 # to change to this directory before starting.
@@ -78,12 +78,7 b' c = get_config()'
78
78
79 # You should not have to change these attributes.
79 # You should not have to change these attributes.
80
80
81 # c.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
81 # c.Global.url_file_name = u'ipcontroller-engine.furl'
82
83 # c.Global.furl_file_name = u'ipcontroller-engine.furl'
84
85
86
87
82
88
83
89
84
@@ -138,8 +138,8 b' class ClusterDir(Configurable):'
138
138
139 def copy_all_config_files(self, path=None, overwrite=False):
139 def copy_all_config_files(self, path=None, overwrite=False):
140 """Copy all config files into the active cluster directory."""
140 """Copy all config files into the active cluster directory."""
141 for f in [u'ipcontrollerz_config.py', u'ipenginez_config.py',
141 for f in [u'ipcontroller_config.py', u'ipengine_config.py',
142 u'ipclusterz_config.py']:
142 u'ipcluster_config.py']:
143 self.copy_config_file(f, path=path, overwrite=overwrite)
143 self.copy_config_file(f, path=path, overwrite=overwrite)
144
144
145 @classmethod
145 @classmethod
@@ -164,11 +164,11 b' class ClusterDir(Configurable):'
164 The path (directory) to put the cluster directory in.
164 The path (directory) to put the cluster directory in.
165 profile : str
165 profile : str
166 The name of the profile. The name of the cluster directory will
166 The name of the profile. The name of the cluster directory will
167 be "clusterz_<profile>".
167 be "cluster_<profile>".
168 """
168 """
169 if not os.path.isdir(path):
169 if not os.path.isdir(path):
170 raise ClusterDirError('Directory not found: %s' % path)
170 raise ClusterDirError('Directory not found: %s' % path)
171 cluster_dir = os.path.join(path, u'clusterz_' + profile)
171 cluster_dir = os.path.join(path, u'cluster_' + profile)
172 return ClusterDir(location=cluster_dir)
172 return ClusterDir(location=cluster_dir)
173
173
174 @classmethod
174 @classmethod
@@ -190,9 +190,9 b' class ClusterDir(Configurable):'
190 The IPython directory to use.
190 The IPython directory to use.
191 profile : unicode or str
191 profile : unicode or str
192 The name of the profile. The name of the cluster directory
192 The name of the profile. The name of the cluster directory
193 will be "clusterz_<profile>".
193 will be "cluster_<profile>".
194 """
194 """
195 dirname = u'clusterz_' + profile
195 dirname = u'cluster_' + profile
196 cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
196 cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
197 if cluster_dir_paths:
197 if cluster_dir_paths:
198 cluster_dir_paths = cluster_dir_paths.split(':')
198 cluster_dir_paths = cluster_dir_paths.split(':')
@@ -37,7 +37,7 b' from IPython.parallel.clusterdir import ('
37 #-----------------------------------------------------------------------------
37 #-----------------------------------------------------------------------------
38
38
39
39
40 default_config_file_name = u'ipclusterz_config.py'
40 default_config_file_name = u'ipcluster_config.py'
41
41
42
42
43 _description = """\
43 _description = """\
@@ -47,9 +47,9 b' An IPython cluster consists of 1 controller and 1 or more engines.'
47 This command automates the startup of these processes using a wide
47 This command automates the startup of these processes using a wide
48 range of startup methods (SSH, local processes, PBS, mpiexec,
48 range of startup methods (SSH, local processes, PBS, mpiexec,
49 Windows HPC Server 2008). To start a cluster with 4 engines on your
49 Windows HPC Server 2008). To start a cluster with 4 engines on your
50 local host simply do 'ipclusterz start -n 4'. For more complex usage
50 local host simply do 'ipcluster start -n 4'. For more complex usage
51 you will typically do 'ipclusterz create -p mycluster', then edit
51 you will typically do 'ipcluster create -p mycluster', then edit
52 configuration files, followed by 'ipclusterz start -p mycluster -n 4'.
52 configuration files, followed by 'ipcluster start -p mycluster -n 4'.
53 """
53 """
54
54
55
55
@@ -108,9 +108,9 b' class IPClusterAppConfigLoader(ClusterDirConfigLoader):'
108 title='ipcluster subcommands',
108 title='ipcluster subcommands',
109 description=
109 description=
110 """ipcluster has a variety of subcommands. The general way of
110 """ipcluster has a variety of subcommands. The general way of
111 running ipcluster is 'ipclusterz <cmd> [options]'. To get help
111 running ipcluster is 'ipcluster <cmd> [options]'. To get help
112 on a particular subcommand do 'ipclusterz <cmd> -h'."""
112 on a particular subcommand do 'ipcluster <cmd> -h'."""
113 # help="For more help, type 'ipclusterz <cmd> -h'",
113 # help="For more help, type 'ipcluster <cmd> -h'",
114 )
114 )
115
115
116 # The "list" subcommand parser
116 # The "list" subcommand parser
@@ -123,7 +123,7 b' class IPClusterAppConfigLoader(ClusterDirConfigLoader):'
123 """List all available clusters, by cluster directory, that can
123 """List all available clusters, by cluster directory, that can
124 be found in the current working directly or in the ipython
124 be found in the current working directly or in the ipython
125 directory. Cluster directories are named using the convention
125 directory. Cluster directories are named using the convention
126 'clusterz_<profile>'."""
126 'cluster_<profile>'."""
127 )
127 )
128
128
129 # The "create" subcommand parser
129 # The "create" subcommand parser
@@ -136,13 +136,13 b' class IPClusterAppConfigLoader(ClusterDirConfigLoader):'
136 """Create an ipython cluster directory by its profile name or
136 """Create an ipython cluster directory by its profile name or
137 cluster directory path. Cluster directories contain
137 cluster directory path. Cluster directories contain
138 configuration, log and security related files and are named
138 configuration, log and security related files and are named
139 using the convention 'clusterz_<profile>'. By default they are
139 using the convention 'cluster_<profile>'. By default they are
140 located in your ipython directory. Once created, you will
140 located in your ipython directory. Once created, you will
141 probably need to edit the configuration files in the cluster
141 probably need to edit the configuration files in the cluster
142 directory to configure your cluster. Most users will create a
142 directory to configure your cluster. Most users will create a
143 cluster directory by profile name,
143 cluster directory by profile name,
144 'ipclusterz create -p mycluster', which will put the directory
144 'ipcluster create -p mycluster', which will put the directory
145 in '<ipython_dir>/clusterz_mycluster'.
145 in '<ipython_dir>/cluster_mycluster'.
146 """
146 """
147 )
147 )
148 paa = parser_create.add_argument
148 paa = parser_create.add_argument
@@ -162,10 +162,10 b' class IPClusterAppConfigLoader(ClusterDirConfigLoader):'
162 """Start an ipython cluster by its profile name or cluster
162 """Start an ipython cluster by its profile name or cluster
163 directory. Cluster directories contain configuration, log and
163 directory. Cluster directories contain configuration, log and
164 security related files and are named using the convention
164 security related files and are named using the convention
165 'clusterz_<profile>' and should be creating using the 'start'
165 'cluster_<profile>' and should be creating using the 'start'
166 subcommand of 'ipcluster'. If your cluster directory is in
166 subcommand of 'ipcluster'. If your cluster directory is in
167 the cwd or the ipython directory, you can simply refer to it
167 the cwd or the ipython directory, you can simply refer to it
168 using its profile name, 'ipclusterz start -n 4 -p <profile>`,
168 using its profile name, 'ipcluster start -n 4 -p <profile>`,
169 otherwise use the '--cluster-dir' option.
169 otherwise use the '--cluster-dir' option.
170 """
170 """
171 )
171 )
@@ -200,9 +200,9 b' class IPClusterAppConfigLoader(ClusterDirConfigLoader):'
200 description=
200 description=
201 """Stop a running ipython cluster by its profile name or cluster
201 """Stop a running ipython cluster by its profile name or cluster
202 directory. Cluster directories are named using the convention
202 directory. Cluster directories are named using the convention
203 'clusterz_<profile>'. If your cluster directory is in
203 'cluster_<profile>'. If your cluster directory is in
204 the cwd or the ipython directory, you can simply refer to it
204 the cwd or the ipython directory, you can simply refer to it
205 using its profile name, 'ipclusterz stop -p <profile>`, otherwise
205 using its profile name, 'ipcluster stop -p <profile>`, otherwise
206 use the '--cluster-dir' option.
206 use the '--cluster-dir' option.
207 """
207 """
208 )
208 )
@@ -223,10 +223,10 b' class IPClusterAppConfigLoader(ClusterDirConfigLoader):'
223 by profile name or cluster directory.
223 by profile name or cluster directory.
224 Cluster directories contain configuration, log and
224 Cluster directories contain configuration, log and
225 security related files and are named using the convention
225 security related files and are named using the convention
226 'clusterz_<profile>' and should be creating using the 'start'
226 'cluster_<profile>' and should be creating using the 'start'
227 subcommand of 'ipcluster'. If your cluster directory is in
227 subcommand of 'ipcluster'. If your cluster directory is in
228 the cwd or the ipython directory, you can simply refer to it
228 the cwd or the ipython directory, you can simply refer to it
229 using its profile name, 'ipclusterz engines -n 4 -p <profile>`,
229 using its profile name, 'ipcluster engines -n 4 -p <profile>`,
230 otherwise use the '--cluster-dir' option.
230 otherwise use the '--cluster-dir' option.
231 """
231 """
232 )
232 )
@@ -249,7 +249,7 b' class IPClusterAppConfigLoader(ClusterDirConfigLoader):'
249
249
250 class IPClusterApp(ApplicationWithClusterDir):
250 class IPClusterApp(ApplicationWithClusterDir):
251
251
252 name = u'ipclusterz'
252 name = u'ipcluster'
253 description = _description
253 description = _description
254 usage = None
254 usage = None
255 command_line_loader = IPClusterAppConfigLoader
255 command_line_loader = IPClusterAppConfigLoader
@@ -286,8 +286,8 b' class IPClusterApp(ApplicationWithClusterDir):'
286 except ClusterDirError:
286 except ClusterDirError:
287 raise ClusterDirError(
287 raise ClusterDirError(
288 "Could not find a cluster directory. A cluster dir must "
288 "Could not find a cluster directory. A cluster dir must "
289 "be created before running 'ipclusterz start'. Do "
289 "be created before running 'ipcluster start'. Do "
290 "'ipclusterz create -h' or 'ipclusterz list -h' for more "
290 "'ipcluster create -h' or 'ipcluster list -h' for more "
291 "information about creating and listing cluster dirs."
291 "information about creating and listing cluster dirs."
292 )
292 )
293 elif subcommand=='engines':
293 elif subcommand=='engines':
@@ -297,8 +297,8 b' class IPClusterApp(ApplicationWithClusterDir):'
297 except ClusterDirError:
297 except ClusterDirError:
298 raise ClusterDirError(
298 raise ClusterDirError(
299 "Could not find a cluster directory. A cluster dir must "
299 "Could not find a cluster directory. A cluster dir must "
300 "be created before running 'ipclusterz start'. Do "
300 "be created before running 'ipcluster start'. Do "
301 "'ipclusterz create -h' or 'ipclusterz list -h' for more "
301 "'ipcluster create -h' or 'ipcluster list -h' for more "
302 "information about creating and listing cluster dirs."
302 "information about creating and listing cluster dirs."
303 )
303 )
304
304
@@ -322,9 +322,9 b' class IPClusterApp(ApplicationWithClusterDir):'
322 files = os.listdir(path)
322 files = os.listdir(path)
323 for f in files:
323 for f in files:
324 full_path = os.path.join(path, f)
324 full_path = os.path.join(path, f)
325 if os.path.isdir(full_path) and f.startswith('clusterz_'):
325 if os.path.isdir(full_path) and f.startswith('cluster_'):
326 profile = full_path.split('_')[-1]
326 profile = full_path.split('_')[-1]
327 start_cmd = 'ipclusterz start -p %s -n 4' % profile
327 start_cmd = 'ipcluster start -p %s -n 4' % profile
328 print start_cmd + " ==> " + full_path
328 print start_cmd + " ==> " + full_path
329
329
330 def pre_construct(self):
330 def pre_construct(self):
@@ -498,7 +498,7 b' class IPClusterApp(ApplicationWithClusterDir):'
498 else:
498 else:
499 self.log.critical(
499 self.log.critical(
500 'Cluster is already running with [pid=%s]. '
500 'Cluster is already running with [pid=%s]. '
501 'use "ipclusterz stop" to stop the cluster.' % pid
501 'use "ipcluster stop" to stop the cluster.' % pid
502 )
502 )
503 # Here I exit with a unusual exit status that other processes
503 # Here I exit with a unusual exit status that other processes
504 # can watch for to learn how I existed.
504 # can watch for to learn how I existed.
@@ -506,7 +506,7 b' class IPClusterApp(ApplicationWithClusterDir):'
506
506
507 # Now log and daemonize
507 # Now log and daemonize
508 self.log.info(
508 self.log.info(
509 'Starting ipclusterz with [daemon=%r]' % config.Global.daemonize
509 'Starting ipcluster with [daemon=%r]' % config.Global.daemonize
510 )
510 )
511 # TODO: Get daemonize working on Windows or as a Windows Server.
511 # TODO: Get daemonize working on Windows or as a Windows Server.
512 if config.Global.daemonize:
512 if config.Global.daemonize:
@@ -48,7 +48,7 b' from IPython.utils.traitlets import Instance, Unicode'
48
48
49
49
50 #: The default config file name for this application
50 #: The default config file name for this application
51 default_config_file_name = u'ipcontrollerz_config.py'
51 default_config_file_name = u'ipcontroller_config.py'
52
52
53
53
54 _description = """Start the IPython controller for parallel computing.
54 _description = """Start the IPython controller for parallel computing.
@@ -57,7 +57,7 b' The IPython controller provides a gateway between the IPython engines and'
57 clients. The controller needs to be started before the engines and can be
57 clients. The controller needs to be started before the engines and can be
58 configured using command line options or using a cluster directory. Cluster
58 configured using command line options or using a cluster directory. Cluster
59 directories contain config, log and security files and are usually located in
59 directories contain config, log and security files and are usually located in
60 your ipython directory and named as "clusterz_<profile>". See the --profile
60 your ipython directory and named as "cluster_<profile>". See the --profile
61 and --cluster-dir options for details.
61 and --cluster-dir options for details.
62 """
62 """
63
63
@@ -251,7 +251,7 b' class IPControllerAppConfigLoader(ClusterDirConfigLoader):'
251
251
252 class IPControllerApp(ApplicationWithClusterDir):
252 class IPControllerApp(ApplicationWithClusterDir):
253
253
254 name = u'ipcontrollerz'
254 name = u'ipcontroller'
255 description = _description
255 description = _description
256 command_line_loader = IPControllerAppConfigLoader
256 command_line_loader = IPControllerAppConfigLoader
257 default_config_file_name = default_config_file_name
257 default_config_file_name = default_config_file_name
@@ -40,7 +40,7 b' from IPython.utils.importstring import import_item'
40 #-----------------------------------------------------------------------------
40 #-----------------------------------------------------------------------------
41
41
42 #: The default config file name for this application
42 #: The default config file name for this application
43 default_config_file_name = u'ipenginez_config.py'
43 default_config_file_name = u'ipengine_config.py'
44
44
45
45
46 mpi4py_init = """from mpi4py import MPI as mpi
46 mpi4py_init = """from mpi4py import MPI as mpi
@@ -64,7 +64,7 b' IPython engines run in parallel and perform computations on behalf of a client'
64 and controller. A controller needs to be started before the engines. The
64 and controller. A controller needs to be started before the engines. The
65 engine can be configured using command line options or using a cluster
65 engine can be configured using command line options or using a cluster
66 directory. Cluster directories contain config, log and security files and are
66 directory. Cluster directories contain config, log and security files and are
67 usually located in your ipython directory and named as "clusterz_<profile>".
67 usually located in your ipython directory and named as "cluster_<profile>".
68 See the --profile and --cluster-dir options for details.
68 See the --profile and --cluster-dir options for details.
69 """
69 """
70
70
@@ -124,7 +124,7 b' class IPEngineAppConfigLoader(ClusterDirConfigLoader):'
124
124
125 class IPEngineApp(ApplicationWithClusterDir):
125 class IPEngineApp(ApplicationWithClusterDir):
126
126
127 name = u'ipenginez'
127 name = u'ipengine'
128 description = _description
128 description = _description
129 command_line_loader = IPEngineAppConfigLoader
129 command_line_loader = IPEngineAppConfigLoader
130 default_config_file_name = default_config_file_name
130 default_config_file_name = default_config_file_name
@@ -39,7 +39,7 b' IPython controllers and engines (and your own processes) can broadcast log messa'
39 by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The
39 by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The
40 logger can be configured using command line options or using a cluster
40 logger can be configured using command line options or using a cluster
41 directory. Cluster directories contain config, log and security files and are
41 directory. Cluster directories contain config, log and security files and are
42 usually located in your ipython directory and named as "clusterz_<profile>".
42 usually located in your ipython directory and named as "cluster_<profile>".
43 See the --profile and --cluster-dir options for details.
43 See the --profile and --cluster-dir options for details.
44 """
44 """
45
45
@@ -63,15 +63,15 b' except ImportError:'
63 #-----------------------------------------------------------------------------
63 #-----------------------------------------------------------------------------
64
64
65
65
66 ipclusterz_cmd_argv = pycmd2argv(get_ipython_module_path(
66 ipcluster_cmd_argv = pycmd2argv(get_ipython_module_path(
67 'IPython.parallel.ipclusterapp'
67 'IPython.parallel.ipclusterapp'
68 ))
68 ))
69
69
70 ipenginez_cmd_argv = pycmd2argv(get_ipython_module_path(
70 ipengine_cmd_argv = pycmd2argv(get_ipython_module_path(
71 'IPython.parallel.ipengineapp'
71 'IPython.parallel.ipengineapp'
72 ))
72 ))
73
73
74 ipcontrollerz_cmd_argv = pycmd2argv(get_ipython_module_path(
74 ipcontroller_cmd_argv = pycmd2argv(get_ipython_module_path(
75 'IPython.parallel.ipcontrollerapp'
75 'IPython.parallel.ipcontrollerapp'
76 ))
76 ))
77
77
@@ -304,7 +304,7 b' class LocalProcessLauncher(BaseLauncher):'
304 class LocalControllerLauncher(LocalProcessLauncher):
304 class LocalControllerLauncher(LocalProcessLauncher):
305 """Launch a controller as a regular external process."""
305 """Launch a controller as a regular external process."""
306
306
307 controller_cmd = List(ipcontrollerz_cmd_argv, config=True)
307 controller_cmd = List(ipcontroller_cmd_argv, config=True)
308 # Command line arguments to ipcontroller.
308 # Command line arguments to ipcontroller.
309 controller_args = List(['--log-to-file','--log-level', str(logging.INFO)], config=True)
309 controller_args = List(['--log-to-file','--log-level', str(logging.INFO)], config=True)
310
310
@@ -322,7 +322,7 b' class LocalControllerLauncher(LocalProcessLauncher):'
322 class LocalEngineLauncher(LocalProcessLauncher):
322 class LocalEngineLauncher(LocalProcessLauncher):
323 """Launch a single engine as a regular externall process."""
323 """Launch a single engine as a regular externall process."""
324
324
325 engine_cmd = List(ipenginez_cmd_argv, config=True)
325 engine_cmd = List(ipengine_cmd_argv, config=True)
326 # Command line arguments for ipengine.
326 # Command line arguments for ipengine.
327 engine_args = List(
327 engine_args = List(
328 ['--log-to-file','--log-level', str(logging.INFO)], config=True
328 ['--log-to-file','--log-level', str(logging.INFO)], config=True
@@ -443,7 +443,7 b' class MPIExecLauncher(LocalProcessLauncher):'
443 class MPIExecControllerLauncher(MPIExecLauncher):
443 class MPIExecControllerLauncher(MPIExecLauncher):
444 """Launch a controller using mpiexec."""
444 """Launch a controller using mpiexec."""
445
445
446 controller_cmd = List(ipcontrollerz_cmd_argv, config=True)
446 controller_cmd = List(ipcontroller_cmd_argv, config=True)
447 # Command line arguments to ipcontroller.
447 # Command line arguments to ipcontroller.
448 controller_args = List(['--log-to-file','--log-level', str(logging.INFO)], config=True)
448 controller_args = List(['--log-to-file','--log-level', str(logging.INFO)], config=True)
449 n = Int(1, config=False)
449 n = Int(1, config=False)
@@ -462,7 +462,7 b' class MPIExecControllerLauncher(MPIExecLauncher):'
462
462
463 class MPIExecEngineSetLauncher(MPIExecLauncher):
463 class MPIExecEngineSetLauncher(MPIExecLauncher):
464
464
465 program = List(ipenginez_cmd_argv, config=True)
465 program = List(ipengine_cmd_argv, config=True)
466 # Command line arguments for ipengine.
466 # Command line arguments for ipengine.
467 program_args = List(
467 program_args = List(
468 ['--log-to-file','--log-level', str(logging.INFO)], config=True
468 ['--log-to-file','--log-level', str(logging.INFO)], config=True
@@ -531,13 +531,13 b' class SSHLauncher(LocalProcessLauncher):'
531
531
532 class SSHControllerLauncher(SSHLauncher):
532 class SSHControllerLauncher(SSHLauncher):
533
533
534 program = List(ipcontrollerz_cmd_argv, config=True)
534 program = List(ipcontroller_cmd_argv, config=True)
535 # Command line arguments to ipcontroller.
535 # Command line arguments to ipcontroller.
536 program_args = List(['-r', '--log-to-file','--log-level', str(logging.INFO)], config=True)
536 program_args = List(['-r', '--log-to-file','--log-level', str(logging.INFO)], config=True)
537
537
538
538
539 class SSHEngineLauncher(SSHLauncher):
539 class SSHEngineLauncher(SSHLauncher):
540 program = List(ipenginez_cmd_argv, config=True)
540 program = List(ipengine_cmd_argv, config=True)
541 # Command line arguments for ipengine.
541 # Command line arguments for ipengine.
542 program_args = List(
542 program_args = List(
543 ['--log-to-file','--log-level', str(logging.INFO)], config=True
543 ['--log-to-file','--log-level', str(logging.INFO)], config=True
@@ -883,9 +883,9 b' class PBSControllerLauncher(PBSLauncher):'
883 batch_file_name = CUnicode(u'pbs_controller', config=True)
883 batch_file_name = CUnicode(u'pbs_controller', config=True)
884 default_template= CUnicode("""#!/bin/sh
884 default_template= CUnicode("""#!/bin/sh
885 #PBS -V
885 #PBS -V
886 #PBS -N ipcontrollerz
886 #PBS -N ipcontroller
887 %s --log-to-file --cluster-dir $cluster_dir
887 %s --log-to-file --cluster-dir $cluster_dir
888 """%(' '.join(ipcontrollerz_cmd_argv)))
888 """%(' '.join(ipcontroller_cmd_argv)))
889
889
890 def start(self, cluster_dir):
890 def start(self, cluster_dir):
891 """Start the controller by profile or cluster_dir."""
891 """Start the controller by profile or cluster_dir."""
@@ -898,9 +898,9 b' class PBSEngineSetLauncher(PBSLauncher):'
898 batch_file_name = CUnicode(u'pbs_engines', config=True)
898 batch_file_name = CUnicode(u'pbs_engines', config=True)
899 default_template= CUnicode(u"""#!/bin/sh
899 default_template= CUnicode(u"""#!/bin/sh
900 #PBS -V
900 #PBS -V
901 #PBS -N ipenginez
901 #PBS -N ipengine
902 %s --cluster-dir $cluster_dir
902 %s --cluster-dir $cluster_dir
903 """%(' '.join(ipenginez_cmd_argv)))
903 """%(' '.join(ipengine_cmd_argv)))
904
904
905 def start(self, n, cluster_dir):
905 def start(self, n, cluster_dir):
906 """Start n engines by profile or cluster_dir."""
906 """Start n engines by profile or cluster_dir."""
@@ -922,9 +922,9 b' class SGEControllerLauncher(SGELauncher):'
922 batch_file_name = CUnicode(u'sge_controller', config=True)
922 batch_file_name = CUnicode(u'sge_controller', config=True)
923 default_template= CUnicode(u"""#$$ -V
923 default_template= CUnicode(u"""#$$ -V
924 #$$ -S /bin/sh
924 #$$ -S /bin/sh
925 #$$ -N ipcontrollerz
925 #$$ -N ipcontroller
926 %s --log-to-file --cluster-dir $cluster_dir
926 %s --log-to-file --cluster-dir $cluster_dir
927 """%(' '.join(ipcontrollerz_cmd_argv)))
927 """%(' '.join(ipcontroller_cmd_argv)))
928
928
929 def start(self, cluster_dir):
929 def start(self, cluster_dir):
930 """Start the controller by profile or cluster_dir."""
930 """Start the controller by profile or cluster_dir."""
@@ -936,9 +936,9 b' class SGEEngineSetLauncher(SGELauncher):'
936 batch_file_name = CUnicode(u'sge_engines', config=True)
936 batch_file_name = CUnicode(u'sge_engines', config=True)
937 default_template = CUnicode("""#$$ -V
937 default_template = CUnicode("""#$$ -V
938 #$$ -S /bin/sh
938 #$$ -S /bin/sh
939 #$$ -N ipenginez
939 #$$ -N ipengine
940 %s --cluster-dir $cluster_dir
940 %s --cluster-dir $cluster_dir
941 """%(' '.join(ipenginez_cmd_argv)))
941 """%(' '.join(ipengine_cmd_argv)))
942
942
943 def start(self, n, cluster_dir):
943 def start(self, n, cluster_dir):
944 """Start n engines by profile or cluster_dir."""
944 """Start n engines by profile or cluster_dir."""
@@ -954,7 +954,7 b' class SGEEngineSetLauncher(SGELauncher):'
954 class IPClusterLauncher(LocalProcessLauncher):
954 class IPClusterLauncher(LocalProcessLauncher):
955 """Launch the ipcluster program in an external process."""
955 """Launch the ipcluster program in an external process."""
956
956
957 ipcluster_cmd = List(ipclusterz_cmd_argv, config=True)
957 ipcluster_cmd = List(ipcluster_cmd_argv, config=True)
958 # Command line arguments to pass to ipcluster.
958 # Command line arguments to pass to ipcluster.
959 ipcluster_args = List(
959 ipcluster_args = List(
960 ['--clean-logs', '--log-to-file', '--log-level', str(logging.INFO)], config=True)
960 ['--clean-logs', '--log-to-file', '--log-level', str(logging.INFO)], config=True)
@@ -1,5 +1,5 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 """A simple logger object that consolidates messages incoming from ipclusterz processes."""
2 """A simple logger object that consolidates messages incoming from ipcluster processes."""
3
3
4 #-----------------------------------------------------------------------------
4 #-----------------------------------------------------------------------------
5 # Copyright (C) 2011 The IPython Development Team
5 # Copyright (C) 2011 The IPython Development Team
1 NO CONTENT: file renamed from IPython/parallel/scripts/ipclusterz to IPython/parallel/scripts/ipcluster
NO CONTENT: file renamed from IPython/parallel/scripts/ipclusterz to IPython/parallel/scripts/ipcluster
1 NO CONTENT: file renamed from IPython/parallel/scripts/ipcontrollerz to IPython/parallel/scripts/ipcontroller
NO CONTENT: file renamed from IPython/parallel/scripts/ipcontrollerz to IPython/parallel/scripts/ipcontroller
1 NO CONTENT: file renamed from IPython/parallel/scripts/ipenginez to IPython/parallel/scripts/ipengine
NO CONTENT: file renamed from IPython/parallel/scripts/ipenginez to IPython/parallel/scripts/ipengine
1 NO CONTENT: file renamed from IPython/parallel/scripts/iploggerz to IPython/parallel/scripts/iplogger
NO CONTENT: file renamed from IPython/parallel/scripts/iploggerz to IPython/parallel/scripts/iplogger
@@ -23,7 +23,7 b' blackhole = tempfile.TemporaryFile()'
23 # nose setup/teardown
23 # nose setup/teardown
24
24
25 def setup():
25 def setup():
26 cp = Popen('ipcontrollerz --profile iptest -r --log-level 10 --log-to-file'.split(), stdout=blackhole, stderr=STDOUT)
26 cp = Popen('ipcontroller --profile iptest -r --log-level 10 --log-to-file'.split(), stdout=blackhole, stderr=STDOUT)
27 processes.append(cp)
27 processes.append(cp)
28 time.sleep(.5)
28 time.sleep(.5)
29 add_engines(1)
29 add_engines(1)
@@ -38,7 +38,7 b" def add_engines(n=1, profile='iptest'):"
38 base = len(rc)
38 base = len(rc)
39 eps = []
39 eps = []
40 for i in range(n):
40 for i in range(n):
41 ep = Popen(['ipenginez']+ ['--profile', profile, '--log-level', '10', '--log-to-file'], stdout=blackhole, stderr=STDOUT)
41 ep = Popen(['ipengine']+ ['--profile', profile, '--log-level', '10', '--log-to-file'], stdout=blackhole, stderr=STDOUT)
42 # ep.start()
42 # ep.start()
43 processes.append(ep)
43 processes.append(ep)
44 eps.append(ep)
44 eps.append(ep)
@@ -13,7 +13,7 b''
13
13
14 from unittest import TestCase
14 from unittest import TestCase
15
15
16 from IPython.testing.parametric import parametric
16 from IPython.testing.decorators import parametric
17 from IPython.utils import newserialized as ns
17 from IPython.utils import newserialized as ns
18 from IPython.utils.pickleutil import can, uncan, CannedObject, CannedFunction
18 from IPython.utils.pickleutil import can, uncan, CannedObject, CannedFunction
19 from IPython.parallel.tests.clienttest import skip_without
19 from IPython.parallel.tests.clienttest import skip_without
@@ -185,6 +185,7 b' def make_exclude():'
185
185
186 if not have['zmq']:
186 if not have['zmq']:
187 exclusions.append(ipjoin('zmq'))
187 exclusions.append(ipjoin('zmq'))
188 exclusions.append(ipjoin('parallel'))
188
189
189 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
190 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
190 if sys.platform == 'win32':
191 if sys.platform == 'win32':
@@ -19,8 +19,7 b' Contents'
19 whatsnew/index.txt
19 whatsnew/index.txt
20 install/index.txt
20 install/index.txt
21 interactive/index.txt
21 interactive/index.txt
22 .. parallel/index.txt
22 parallel/index.txt
23 parallelz/index.txt
24 config/index.txt
23 config/index.txt
25 development/index.txt
24 development/index.txt
26 api/index.txt
25 api/index.txt
@@ -9,16 +9,16 b' install all of its dependencies.'
9
9
10
10
11 Please let us know if you have problems installing IPython or any of its
11 Please let us know if you have problems installing IPython or any of its
12 dependencies. Officially, IPython requires Python version 2.5 or 2.6. We
12 dependencies. Officially, IPython requires Python version 2.6 or 2.7. There
13 have *not* yet started to port IPython to Python 3.0.
13 is an experimental port of IPython for Python3 `on GitHub
14 <https://github.com/ipython/ipython-py3k>`_
14
15
15 .. warning::
16 .. warning::
16
17
17 Officially, IPython supports Python versions 2.5 and 2.6.
18 Officially, IPython supports Python versions 2.6 and 2.7.
18
19
19 IPython 0.10 has only been well tested with Python 2.5 and 2.6. Parts of
20 IPython 0.11 has a hard syntax dependency on 2.6, and will no longer work
20 it may work with Python 2.4, but we do not officially support Python 2.4
21 on Python <= 2.5.
21 anymore. If you need to use 2.4, you can still run IPython 0.9.
22
22
23 Some of the installation approaches use the :mod:`setuptools` package and its
23 Some of the installation approaches use the :mod:`setuptools` package and its
24 :command:`easy_install` command line program. In many scenarios, this provides
24 :command:`easy_install` command line program. In many scenarios, this provides
@@ -38,9 +38,9 b' optional dependencies:'
38
38
39 .. code-block:: bash
39 .. code-block:: bash
40
40
41 $ easy_install ipython[kernel,security,test]
41 $ easy_install ipython[zmq,test]
42
42
43 This will get Twisted, zope.interface and Foolscap, which are needed for
43 This will get pyzmq, which is needed for
44 IPython's parallel computing features as well as the nose package, which will
44 IPython's parallel computing features as well as the nose package, which will
45 enable you to run IPython's test suite.
45 enable you to run IPython's test suite.
46
46
@@ -221,8 +221,7 b' On Windows, you will need the PyReadline module. PyReadline is a separate,'
221 Windows only implementation of readline that uses native Windows calls through
221 Windows only implementation of readline that uses native Windows calls through
222 :mod:`ctypes`. The easiest way of installing PyReadline is you use the binary
222 :mod:`ctypes`. The easiest way of installing PyReadline is you use the binary
223 installer available `here <http://ipython.scipy.org/dist/>`_. The :mod:`ctypes`
223 installer available `here <http://ipython.scipy.org/dist/>`_. The :mod:`ctypes`
224 module, which comes with Python 2.5 and greater, is required by PyReadline. It
224 module, which comes with Python 2.5 and greater, is required by PyReadline.
225 is available for Python 2.4 at http://python.net/crew/theller/ctypes.
226
225
227 nose
226 nose
228 ----
227 ----
@@ -267,91 +266,30 b" The `pexpect <http://www.noah.org/wiki/Pexpect>`_ package is used in IPython's"
267
266
268 Windows users are out of luck as pexpect does not run there.
267 Windows users are out of luck as pexpect does not run there.
269
268
270 Dependencies for IPython.kernel (parallel computing)
269 Dependencies for IPython.parallel (parallel computing)
271 ====================================================
270 ======================================================
272
271
273 The IPython kernel provides a nice architecture for parallel computing. The
272 :mod:`IPython.kernel` has been replaced by :mod:`IPython.parallel`,
274 main focus of this architecture is on interactive parallel computing. These
273 which uses ZeroMQ for all communication.
275 features require a number of additional packages:
276
274
277 * zope.interface (yep, we use interfaces)
275 IPython.parallel provides a nice architecture for parallel computing. The
278 * Twisted (asynchronous networking framework)
276 main focus of this architecture is on interactive parallel computing. These
279 * Foolscap (a nice, secure network protocol)
277 features require just one package: pyzmq. See the next section for pyzmq
280 * pyOpenSSL (security for network connections)
278 details.
281
279
282 On a Unix style platform (including OS X), if you want to use
280 On a Unix style platform (including OS X), if you want to use
283 :mod:`setuptools`, you can just do:
281 :mod:`setuptools`, you can just do:
284
282
285 .. code-block:: bash
283 .. code-block:: bash
286
284
287 $ easy_install ipython[kernel] # the first three
285 $ easy_install ipython[zmq] # will include pyzmq
288 $ easy_install ipython[security] # pyOpenSSL
289
290 zope.interface and Twisted
291 --------------------------
292
293 Twisted [Twisted]_ and zope.interface [ZopeInterface]_ are used for networking
294 related things. On Unix style platforms (including OS X), the simplest way of
295 getting the these is to use :command:`easy_install`:
296
297 .. code-block:: bash
298
299 $ easy_install zope.interface
300 $ easy_install Twisted
301
286
302 Of course, you can also download the source tarballs from the Twisted website
287 Security in IPython.parallel is provided by SSH tunnels. By default, Linux
303 [Twisted]_ and the
288 and OSX clients will use the shell ssh command, but on Windows, we also
304 `zope.interface page at PyPI <http://pypi.python.org/pypi/zope.interface>`_
289 support tunneling with paramiko [paramiko]_.
305 and do the usual ``python setup.py install`` if you prefer.
306
290
307 Windows is a bit different. For zope.interface and Twisted, simply get the
291 Dependencies for IPython.zmq
308 latest binary ``.exe`` installer from the Twisted website. This installer
292 ============================
309 includes both zope.interface and Twisted and should just work.
310
311 Foolscap
312 --------
313
314 Foolscap [Foolscap]_ uses Twisted to provide a very nice secure RPC protocol that we use to implement our parallel computing features.
315
316 On all platforms a simple:
317
318 .. code-block:: bash
319
320 $ easy_install foolscap
321
322 should work. You can also download the source tarballs from the `Foolscap
323 website <http://foolscap.lothar.com/trac>`_ and do ``python setup.py install``
324 if you prefer.
325
326 pyOpenSSL
327 ---------
328
329 IPython does not work with version 0.7 of pyOpenSSL [pyOpenSSL]_. It is known
330 to work with version 0.6 and will likely work with the more recent 0.8 and 0.9
331 versions. There are a couple of options for getting this:
332
333 1. Most Linux distributions have packages for pyOpenSSL.
334 2. The built-in Python 2.5 on OS X 10.5 already has it installed.
335 3. There are source tarballs on the pyOpenSSL website. On Unix-like
336 platforms, these can be built using ``python seutp.py install``.
337 4. There is also a binary ``.exe`` Windows installer on the
338 `pyOpenSSL website <http://pyopenssl.sourceforge.net/>`_.
339
340 Dependencies for IPython.frontend (the IPython GUI)
341 ===================================================
342
343 wxPython
344 --------
345
346 Starting with IPython 0.9, IPython has a new :mod:`IPython.frontend` package
347 that has a nice wxPython based IPython GUI. As you would expect, this GUI
348 requires wxPython. Most Linux distributions have wxPython packages available
349 and the built-in Python on OS X comes with wxPython preinstalled. For Windows,
350 a binary installer is available on the `wxPython website
351 <http://www.wxpython.org/>`_.
352
353 Dependencies for IPython.zmq (new parallel)
354 ===========================================
355
293
356 pyzmq
294 pyzmq
357 -----
295 -----
@@ -359,9 +297,11 b' pyzmq'
359 IPython 0.11 introduced some new functionality, including a two-process
297 IPython 0.11 introduced some new functionality, including a two-process
360 execution model using ZeroMQ for communication [ZeroMQ]_. The Python bindings
298 execution model using ZeroMQ for communication [ZeroMQ]_. The Python bindings
361 to ZeroMQ are found in the pyzmq project, which is easy_install-able once you
299 to ZeroMQ are found in the pyzmq project, which is easy_install-able once you
362 have ZeroMQ installed. :mod:`IPython.kernel` is also in the process of being
300 have ZeroMQ installed (or even if you don't).
363 replaced by :mod:`IPython.zmq.parallel`, which uses ZeroMQ for all
301
364 communication.
302 IPython.zmq depends on pyzmq >= 2.0.10.1, but IPython.parallel requires the more
303 recent 2.1.4. 2.1.4 also has binary releases for OSX and Windows, that do not
304 require prior installation of libzmq.
365
305
366 Dependencies for ipython-qtconsole (new GUI)
306 Dependencies for ipython-qtconsole (new GUI)
367 ============================================
307 ============================================
@@ -377,11 +317,12 b' which can be installed from the'
377 pygments
317 pygments
378 --------
318 --------
379
319
380 The syntax-highlighting in ``ipython-qtconsole`` is done with the pygments project, which is easy_install-able.
320 The syntax-highlighting in ``ipython-qtconsole`` is done with the pygments project,
321 which is easy_install-able.
381
322
382 .. [Twisted] Twisted matrix. http://twistedmatrix.org
323 .. [Twisted] Twisted matrix. http://twistedmatrix.org
383 .. [ZopeInterface] http://pypi.python.org/pypi/zope.interface
324 .. [ZopeInterface] http://pypi.python.org/pypi/zope.interface
384 .. [Foolscap] Foolscap network protocol. http://foolscap.lothar.com/trac
325 .. [Foolscap] Foolscap network protocol. http://foolscap.lothar.com/trac
385 .. [pyOpenSSL] pyOpenSSL. http://pyopenssl.sourceforge.net
326 .. [pyOpenSSL] pyOpenSSL. http://pyopenssl.sourceforge.net
386 .. [ZeroMQ] ZeroMQ. http://www.zeromq.org
327 .. [ZeroMQ] ZeroMQ. http://www.zeromq.org
387
328 .. [paramiko] paramiko. https://github.com/robey/paramiko
1 NO CONTENT: file renamed from docs/source/parallelz/asian_call.pdf to docs/source/parallel/asian_call.pdf
NO CONTENT: file renamed from docs/source/parallelz/asian_call.pdf to docs/source/parallel/asian_call.pdf
1 NO CONTENT: file renamed from docs/source/parallelz/asian_call.png to docs/source/parallel/asian_call.png
NO CONTENT: file renamed from docs/source/parallelz/asian_call.png to docs/source/parallel/asian_call.png
1 NO CONTENT: file renamed from docs/source/parallelz/asian_put.pdf to docs/source/parallel/asian_put.pdf
NO CONTENT: file renamed from docs/source/parallelz/asian_put.pdf to docs/source/parallel/asian_put.pdf
1 NO CONTENT: file renamed from docs/source/parallelz/asian_put.png to docs/source/parallel/asian_put.png
NO CONTENT: file renamed from docs/source/parallelz/asian_put.png to docs/source/parallel/asian_put.png
1 NO CONTENT: file renamed from docs/source/parallelz/dag_dependencies.txt to docs/source/parallel/dag_dependencies.txt
NO CONTENT: file renamed from docs/source/parallelz/dag_dependencies.txt to docs/source/parallel/dag_dependencies.txt
1 NO CONTENT: file renamed from docs/source/parallelz/dagdeps.pdf to docs/source/parallel/dagdeps.pdf
NO CONTENT: file renamed from docs/source/parallelz/dagdeps.pdf to docs/source/parallel/dagdeps.pdf
1 NO CONTENT: file renamed from docs/source/parallelz/dagdeps.png to docs/source/parallel/dagdeps.png
NO CONTENT: file renamed from docs/source/parallelz/dagdeps.png to docs/source/parallel/dagdeps.png
1 NO CONTENT: file renamed from docs/source/parallelz/hpc_job_manager.pdf to docs/source/parallel/hpc_job_manager.pdf
NO CONTENT: file renamed from docs/source/parallelz/hpc_job_manager.pdf to docs/source/parallel/hpc_job_manager.pdf
1 NO CONTENT: file renamed from docs/source/parallelz/hpc_job_manager.png to docs/source/parallel/hpc_job_manager.png
NO CONTENT: file renamed from docs/source/parallelz/hpc_job_manager.png to docs/source/parallel/hpc_job_manager.png
@@ -4,9 +4,19 b''
4 Using IPython for parallel computing
4 Using IPython for parallel computing
5 ====================================
5 ====================================
6
6
7 The twisted-based :mod:`IPython.kernel` has been removed, in favor of
7 .. toctree::
8 the new 0MQ-based :mod:`IPython.parallel`, whose merge into master is imminent.
8 :maxdepth: 2
9
10 parallel_intro.txt
11 parallel_process.txt
12 parallel_multiengine.txt
13 parallel_task.txt
14 parallel_mpi.txt
15 parallel_security.txt
16 parallel_winhpc.txt
17 parallel_demos.txt
18 dag_dependencies.txt
19 parallel_details.txt
20 parallel_transition.txt
21
9
22
10 Until that code is merged, it can be found in the `newparallel branch
11 <https://github.com/ipython/ipython/tree/newparallel>`_, and its draft documentation can be
12 found `here <http://minrk.github.com/ipython-doc/newparallel>`_. No newline at end of file
1 NO CONTENT: file renamed from docs/source/parallelz/ipcluster_create.pdf to docs/source/parallel/ipcluster_create.pdf
NO CONTENT: file renamed from docs/source/parallelz/ipcluster_create.pdf to docs/source/parallel/ipcluster_create.pdf
1 NO CONTENT: file renamed from docs/source/parallelz/ipcluster_create.png to docs/source/parallel/ipcluster_create.png
NO CONTENT: file renamed from docs/source/parallelz/ipcluster_create.png to docs/source/parallel/ipcluster_create.png
1 NO CONTENT: file renamed from docs/source/parallelz/ipcluster_start.pdf to docs/source/parallel/ipcluster_start.pdf
NO CONTENT: file renamed from docs/source/parallelz/ipcluster_start.pdf to docs/source/parallel/ipcluster_start.pdf
1 NO CONTENT: file renamed from docs/source/parallelz/ipcluster_start.png to docs/source/parallel/ipcluster_start.png
NO CONTENT: file renamed from docs/source/parallelz/ipcluster_start.png to docs/source/parallel/ipcluster_start.png
1 NO CONTENT: file renamed from docs/source/parallelz/ipython_shell.pdf to docs/source/parallel/ipython_shell.pdf
NO CONTENT: file renamed from docs/source/parallelz/ipython_shell.pdf to docs/source/parallel/ipython_shell.pdf
1 NO CONTENT: file renamed from docs/source/parallelz/ipython_shell.png to docs/source/parallel/ipython_shell.png
NO CONTENT: file renamed from docs/source/parallelz/ipython_shell.png to docs/source/parallel/ipython_shell.png
1 NO CONTENT: file renamed from docs/source/parallelz/mec_simple.pdf to docs/source/parallel/mec_simple.pdf
NO CONTENT: file renamed from docs/source/parallelz/mec_simple.pdf to docs/source/parallel/mec_simple.pdf
1 NO CONTENT: file renamed from docs/source/parallelz/mec_simple.png to docs/source/parallel/mec_simple.png
NO CONTENT: file renamed from docs/source/parallelz/mec_simple.png to docs/source/parallel/mec_simple.png
@@ -110,7 +110,7 b' results. The code to run this calculation in parallel is contained in'
110 :file:`docs/examples/newparallel/parallelpi.py`. This code can be run in parallel
110 :file:`docs/examples/newparallel/parallelpi.py`. This code can be run in parallel
111 using IPython by following these steps:
111 using IPython by following these steps:
112
112
113 1. Use :command:`ipclusterz` to start 15 engines. We used an 8 core (2 quad
113 1. Use :command:`ipcluster` to start 15 engines. We used an 8 core (2 quad
114 core CPUs) cluster with hyperthreading enabled which makes the 8 cores
114 core CPUs) cluster with hyperthreading enabled which makes the 8 cores
115 looks like 16 (1 controller + 15 engines) in the OS. However, the maximum
115 looks like 16 (1 controller + 15 engines) in the OS. However, the maximum
116 speedup we can observe is still only 8x.
116 speedup we can observe is still only 8x.
@@ -230,7 +230,7 b' plot using Matplotlib.'
230 .. literalinclude:: ../../examples/newparallel/mcdriver.py
230 .. literalinclude:: ../../examples/newparallel/mcdriver.py
231 :language: python
231 :language: python
232
232
233 To use this code, start an IPython cluster using :command:`ipclusterz`, open
233 To use this code, start an IPython cluster using :command:`ipcluster`, open
234 IPython in the pylab mode with the file :file:`mcdriver.py` in your current
234 IPython in the pylab mode with the file :file:`mcdriver.py` in your current
235 working directory and then type:
235 working directory and then type:
236
236
1 NO CONTENT: file renamed from docs/source/parallelz/parallel_details.txt to docs/source/parallel/parallel_details.txt
NO CONTENT: file renamed from docs/source/parallelz/parallel_details.txt to docs/source/parallel/parallel_details.txt
@@ -156,7 +156,7 b' To connect and authenticate to the controller an engine or client needs'
156 some information that the controller has stored in a JSON file.
156 some information that the controller has stored in a JSON file.
157 Thus, the JSON files need to be copied to a location where
157 Thus, the JSON files need to be copied to a location where
158 the clients and engines can find them. Typically, this is the
158 the clients and engines can find them. Typically, this is the
159 :file:`~/.ipython/clusterz_default/security` directory on the host where the
159 :file:`~/.ipython/cluster_default/security` directory on the host where the
160 client/engine is running (which could be a different host than the controller).
160 client/engine is running (which could be a different host than the controller).
161 Once the JSON files are copied over, everything should work fine.
161 Once the JSON files are copied over, everything should work fine.
162
162
@@ -192,10 +192,10 b' Getting Started'
192 To use IPython for parallel computing, you need to start one instance of the
192 To use IPython for parallel computing, you need to start one instance of the
193 controller and one or more instances of the engine. Initially, it is best to
193 controller and one or more instances of the engine. Initially, it is best to
194 simply start a controller and engines on a single host using the
194 simply start a controller and engines on a single host using the
195 :command:`ipclusterz` command. To start a controller and 4 engines on your
195 :command:`ipcluster` command. To start a controller and 4 engines on your
196 localhost, just do::
196 localhost, just do::
197
197
198 $ ipclusterz start -n 4
198 $ ipcluster start -n 4
199
199
200 More details about starting the IPython controller and engines can be found
200 More details about starting the IPython controller and engines can be found
201 :ref:`here <parallel_process>`
201 :ref:`here <parallel_process>`
@@ -218,7 +218,7 b' everything is working correctly, try the following commands:'
218
218
219
219
220 When a client is created with no arguments, the client tries to find the corresponding JSON file
220 When a client is created with no arguments, the client tries to find the corresponding JSON file
221 in the local `~/.ipython/clusterz_default/security` directory. Or if you specified a profile,
221 in the local `~/.ipython/cluster_default/security` directory. Or if you specified a profile,
222 you can use that with the Client. This should cover most cases:
222 you can use that with the Client. This should cover most cases:
223
223
224 .. sourcecode:: ipython
224 .. sourcecode:: ipython
@@ -50,16 +50,16 b' To use code that calls MPI, there are typically two things that MPI requires.'
50 There are a couple of ways that you can start the IPython engines and get
50 There are a couple of ways that you can start the IPython engines and get
51 these things to happen.
51 these things to happen.
52
52
53 Automatic starting using :command:`mpiexec` and :command:`ipclusterz`
53 Automatic starting using :command:`mpiexec` and :command:`ipcluster`
54 --------------------------------------------------------------------
54 --------------------------------------------------------------------
55
55
56 The easiest approach is to use the `mpiexec` mode of :command:`ipclusterz`,
56 The easiest approach is to use the `mpiexec` mode of :command:`ipcluster`,
57 which will first start a controller and then a set of engines using
57 which will first start a controller and then a set of engines using
58 :command:`mpiexec`::
58 :command:`mpiexec`::
59
59
60 $ ipclusterz mpiexec -n 4
60 $ ipcluster mpiexec -n 4
61
61
62 This approach is best as interrupting :command:`ipclusterz` will automatically
62 This approach is best as interrupting :command:`ipcluster` will automatically
63 stop and clean up the controller and engines.
63 stop and clean up the controller and engines.
64
64
65 Manual starting using :command:`mpiexec`
65 Manual starting using :command:`mpiexec`
@@ -68,20 +68,20 b' Manual starting using :command:`mpiexec`'
68 If you want to start the IPython engines using the :command:`mpiexec`, just
68 If you want to start the IPython engines using the :command:`mpiexec`, just
69 do::
69 do::
70
70
71 $ mpiexec -n 4 ipenginez --mpi=mpi4py
71 $ mpiexec -n 4 ipengine --mpi=mpi4py
72
72
73 This requires that you already have a controller running and that the FURL
73 This requires that you already have a controller running and that the FURL
74 files for the engines are in place. We also have built in support for
74 files for the engines are in place. We also have built in support for
75 PyTrilinos [PyTrilinos]_, which can be used (assuming is installed) by
75 PyTrilinos [PyTrilinos]_, which can be used (assuming is installed) by
76 starting the engines with::
76 starting the engines with::
77
77
78 $ mpiexec -n 4 ipenginez --mpi=pytrilinos
78 $ mpiexec -n 4 ipengine --mpi=pytrilinos
79
79
80 Automatic starting using PBS and :command:`ipclusterz`
80 Automatic starting using PBS and :command:`ipcluster`
81 ------------------------------------------------------
81 ------------------------------------------------------
82
82
83 The :command:`ipclusterz` command also has built-in integration with PBS. For
83 The :command:`ipcluster` command also has built-in integration with PBS. For
84 more information on this approach, see our documentation on :ref:`ipclusterz
84 more information on this approach, see our documentation on :ref:`ipcluster
85 <parallel_process>`.
85 <parallel_process>`.
86
86
87 Actually using MPI
87 Actually using MPI
@@ -110,7 +110,7 b' distributed array. Save the following text in a file called :file:`psum.py`:'
110
110
111 Now, start an IPython cluster::
111 Now, start an IPython cluster::
112
112
113 $ ipclusterz start -p mpi -n 4
113 $ ipcluster start -p mpi -n 4
114
114
115 .. note::
115 .. note::
116
116
@@ -17,9 +17,9 b' Starting the IPython controller and engines'
17
17
18 To follow along with this tutorial, you will need to start the IPython
18 To follow along with this tutorial, you will need to start the IPython
19 controller and four IPython engines. The simplest way of doing this is to use
19 controller and four IPython engines. The simplest way of doing this is to use
20 the :command:`ipclusterz` command::
20 the :command:`ipcluster` command::
21
21
22 $ ipclusterz start -n 4
22 $ ipcluster start -n 4
23
23
24 For more detailed information about starting the controller and engines, see
24 For more detailed information about starting the controller and engines, see
25 our :ref:`introduction <ip1par>` to using IPython for parallel computing.
25 our :ref:`introduction <ip1par>` to using IPython for parallel computing.
@@ -37,7 +37,7 b' module and then create a :class:`.Client` instance:'
37 In [2]: rc = Client()
37 In [2]: rc = Client()
38
38
39 This form assumes that the default connection information (stored in
39 This form assumes that the default connection information (stored in
40 :file:`ipcontroller-client.json` found in :file:`IPYTHON_DIR/clusterz_default/security`) is
40 :file:`ipcontroller-client.json` found in :file:`IPYTHON_DIR/cluster_default/security`) is
41 accurate. If the controller was started on a remote machine, you must copy that connection
41 accurate. If the controller was started on a remote machine, you must copy that connection
42 file to the client machine, or enter its contents as arguments to the Client constructor:
42 file to the client machine, or enter its contents as arguments to the Client constructor:
43
43
1 NO CONTENT: file renamed from docs/source/parallelz/parallel_pi.pdf to docs/source/parallel/parallel_pi.pdf
NO CONTENT: file renamed from docs/source/parallelz/parallel_pi.pdf to docs/source/parallel/parallel_pi.pdf
1 NO CONTENT: file renamed from docs/source/parallelz/parallel_pi.png to docs/source/parallel/parallel_pi.png
NO CONTENT: file renamed from docs/source/parallelz/parallel_pi.png to docs/source/parallel/parallel_pi.png
@@ -11,12 +11,12 b' Because of this, there are many different possibilities.'
11
11
12 Broadly speaking, there are two ways of going about starting a controller and engines:
12 Broadly speaking, there are two ways of going about starting a controller and engines:
13
13
14 * In an automated manner using the :command:`ipclusterz` command.
14 * In an automated manner using the :command:`ipcluster` command.
15 * In a more manual way using the :command:`ipcontrollerz` and
15 * In a more manual way using the :command:`ipcontroller` and
16 :command:`ipenginez` commands.
16 :command:`ipengine` commands.
17
17
18 This document describes both of these methods. We recommend that new users
18 This document describes both of these methods. We recommend that new users
19 start with the :command:`ipclusterz` command as it simplifies many common usage
19 start with the :command:`ipcluster` command as it simplifies many common usage
20 cases.
20 cases.
21
21
22 General considerations
22 General considerations
@@ -30,29 +30,29 b' matter which method you use to start your IPython cluster.'
30 Let's say that you want to start the controller on ``host0`` and engines on
30 Let's say that you want to start the controller on ``host0`` and engines on
31 hosts ``host1``-``hostn``. The following steps are then required:
31 hosts ``host1``-``hostn``. The following steps are then required:
32
32
33 1. Start the controller on ``host0`` by running :command:`ipcontrollerz` on
33 1. Start the controller on ``host0`` by running :command:`ipcontroller` on
34 ``host0``.
34 ``host0``.
35 2. Move the JSON file (:file:`ipcontroller-engine.json`) created by the
35 2. Move the JSON file (:file:`ipcontroller-engine.json`) created by the
36 controller from ``host0`` to hosts ``host1``-``hostn``.
36 controller from ``host0`` to hosts ``host1``-``hostn``.
37 3. Start the engines on hosts ``host1``-``hostn`` by running
37 3. Start the engines on hosts ``host1``-``hostn`` by running
38 :command:`ipenginez`. This command has to be told where the JSON file
38 :command:`ipengine`. This command has to be told where the JSON file
39 (:file:`ipcontroller-engine.json`) is located.
39 (:file:`ipcontroller-engine.json`) is located.
40
40
41 At this point, the controller and engines will be connected. By default, the JSON files
41 At this point, the controller and engines will be connected. By default, the JSON files
42 created by the controller are put into the :file:`~/.ipython/clusterz_default/security`
42 created by the controller are put into the :file:`~/.ipython/cluster_default/security`
43 directory. If the engines share a filesystem with the controller, step 2 can be skipped as
43 directory. If the engines share a filesystem with the controller, step 2 can be skipped as
44 the engines will automatically look at that location.
44 the engines will automatically look at that location.
45
45
46 The final step required to actually use the running controller from a client is to move
46 The final step required to actually use the running controller from a client is to move
47 the JSON file :file:`ipcontroller-client.json` from ``host0`` to any host where clients
47 the JSON file :file:`ipcontroller-client.json` from ``host0`` to any host where clients
48 will be run. If these file are put into the :file:`~/.ipython/clusterz_default/security`
48 will be run. If these file are put into the :file:`~/.ipython/cluster_default/security`
49 directory of the client's host, they will be found automatically. Otherwise, the full path
49 directory of the client's host, they will be found automatically. Otherwise, the full path
50 to them has to be passed to the client's constructor.
50 to them has to be passed to the client's constructor.
51
51
52 Using :command:`ipclusterz`
52 Using :command:`ipcluster`
53 ===========================
53 ===========================
54
54
55 The :command:`ipclusterz` command provides a simple way of starting a
55 The :command:`ipcluster` command provides a simple way of starting a
56 controller and engines in the following situations:
56 controller and engines in the following situations:
57
57
58 1. When the controller and engines are all run on localhost. This is useful
58 1. When the controller and engines are all run on localhost. This is useful
@@ -67,24 +67,24 b' controller and engines in the following situations:'
67
67
68 .. note::
68 .. note::
69
69
70 Currently :command:`ipclusterz` requires that the
70 Currently :command:`ipcluster` requires that the
71 :file:`~/.ipython/cluster_<profile>/security` directory live on a shared filesystem that is
71 :file:`~/.ipython/cluster_<profile>/security` directory live on a shared filesystem that is
72 seen by both the controller and engines. If you don't have a shared file
72 seen by both the controller and engines. If you don't have a shared file
73 system you will need to use :command:`ipcontrollerz` and
73 system you will need to use :command:`ipcontroller` and
74 :command:`ipenginez` directly.
74 :command:`ipengine` directly.
75
75
76 Under the hood, :command:`ipclusterz` just uses :command:`ipcontrollerz`
76 Under the hood, :command:`ipcluster` just uses :command:`ipcontroller`
77 and :command:`ipenginez` to perform the steps described above.
77 and :command:`ipengine` to perform the steps described above.
78
78
79 The simplest way to use ipclusterz requires no configuration, and will
79 The simplest way to use ipcluster requires no configuration, and will
80 launch a controller and a number of engines on the local machine. For instance,
80 launch a controller and a number of engines on the local machine. For instance,
81 to start one controller and 4 engines on localhost, just do::
81 to start one controller and 4 engines on localhost, just do::
82
82
83 $ ipclusterz start -n 4
83 $ ipcluster start -n 4
84
84
85 To see other command line options for the local mode, do::
85 To see other command line options for the local mode, do::
86
86
87 $ ipclusterz -h
87 $ ipcluster -h
88
88
89
89
90 Configuring an IPython cluster
90 Configuring an IPython cluster
@@ -92,25 +92,25 b' Configuring an IPython cluster'
92
92
93 Cluster configurations are stored as `profiles`. You can create a new profile with::
93 Cluster configurations are stored as `profiles`. You can create a new profile with::
94
94
95 $ ipclusterz create -p myprofile
95 $ ipcluster create -p myprofile
96
96
97 This will create the directory :file:`IPYTHONDIR/clusterz_myprofile`, and populate it
97 This will create the directory :file:`IPYTHONDIR/cluster_myprofile`, and populate it
98 with the default configuration files for the three IPython cluster commands. Once
98 with the default configuration files for the three IPython cluster commands. Once
99 you edit those files, you can continue to call ipclusterz/ipcontrollerz/ipenginez
99 you edit those files, you can continue to call ipcluster/ipcontroller/ipengine
100 with no arguments beyond ``-p myprofile``, and any configuration will be maintained.
100 with no arguments beyond ``-p myprofile``, and any configuration will be maintained.
101
101
102 There is no limit to the number of profiles you can have, so you can maintain a profile for each
102 There is no limit to the number of profiles you can have, so you can maintain a profile for each
103 of your common use cases. The default profile will be used whenever the
103 of your common use cases. The default profile will be used whenever the
104 profile argument is not specified, so edit :file:`IPYTHONDIR/clusterz_default/*_config.py` to
104 profile argument is not specified, so edit :file:`IPYTHONDIR/cluster_default/*_config.py` to
105 represent your most common use case.
105 represent your most common use case.
106
106
107 The configuration files are loaded with commented-out settings and explanations,
107 The configuration files are loaded with commented-out settings and explanations,
108 which should cover most of the available possibilities.
108 which should cover most of the available possibilities.
109
109
110 Using various batch systems with :command:`ipclusterz`
110 Using various batch systems with :command:`ipcluster`
111 ------------------------------------------------------
111 ------------------------------------------------------
112
112
113 :command:`ipclusterz` has a notion of Launchers that can start controllers
113 :command:`ipcluster` has a notion of Launchers that can start controllers
114 and engines with various remote execution schemes. Currently supported
114 and engines with various remote execution schemes. Currently supported
115 models include `mpiexec`, PBS-style (Torque, SGE), and Windows HPC Server.
115 models include `mpiexec`, PBS-style (Torque, SGE), and Windows HPC Server.
116
116
@@ -120,7 +120,7 b' models include `mpiexec`, PBS-style (Torque, SGE), and Windows HPC Server.'
120 users can subclass and configure them to fit their own system that we
120 users can subclass and configure them to fit their own system that we
121 have not yet supported (such as Condor)
121 have not yet supported (such as Condor)
122
122
123 Using :command:`ipclusterz` in mpiexec/mpirun mode
123 Using :command:`ipcluster` in mpiexec/mpirun mode
124 --------------------------------------------------
124 --------------------------------------------------
125
125
126
126
@@ -132,11 +132,11 b' The mpiexec/mpirun mode is useful if you:'
132
132
133 If these are satisfied, you can create a new profile::
133 If these are satisfied, you can create a new profile::
134
134
135 $ ipclusterz create -p mpi
135 $ ipcluster create -p mpi
136
136
137 and edit the file :file:`IPYTHONDIR/clusterz_mpi/ipclusterz_config.py`.
137 and edit the file :file:`IPYTHONDIR/cluster_mpi/ipcluster_config.py`.
138
138
139 There, instruct ipclusterz to use the MPIExec launchers by adding the lines:
139 There, instruct ipcluster to use the MPIExec launchers by adding the lines:
140
140
141 .. sourcecode:: python
141 .. sourcecode:: python
142
142
@@ -144,7 +144,7 b' There, instruct ipclusterz to use the MPIExec launchers by adding the lines:'
144
144
145 If the default MPI configuration is correct, then you can now start your cluster, with::
145 If the default MPI configuration is correct, then you can now start your cluster, with::
146
146
147 $ ipclusterz start -n 4 -p mpi
147 $ ipcluster start -n 4 -p mpi
148
148
149 This does the following:
149 This does the following:
150
150
@@ -166,7 +166,7 b' On newer MPI implementations (such as OpenMPI), this will work even if you'
166 don't make any calls to MPI or call :func:`MPI_Init`. However, older MPI
166 don't make any calls to MPI or call :func:`MPI_Init`. However, older MPI
167 implementations actually require each process to call :func:`MPI_Init` upon
167 implementations actually require each process to call :func:`MPI_Init` upon
168 starting. The easiest way of having this done is to install the mpi4py
168 starting. The easiest way of having this done is to install the mpi4py
169 [mpi4py]_ package and then specify the ``c.MPI.use`` option in :file:`ipenginez_config.py`:
169 [mpi4py]_ package and then specify the ``c.MPI.use`` option in :file:`ipengine_config.py`:
170
170
171 .. sourcecode:: python
171 .. sourcecode:: python
172
172
@@ -177,21 +177,21 b' having problems with this, you will likely have to use a custom Python'
177 executable that itself calls :func:`MPI_Init` at the appropriate time.
177 executable that itself calls :func:`MPI_Init` at the appropriate time.
178 Fortunately, mpi4py comes with such a custom Python executable that is easy to
178 Fortunately, mpi4py comes with such a custom Python executable that is easy to
179 install and use. However, this custom Python executable approach will not work
179 install and use. However, this custom Python executable approach will not work
180 with :command:`ipclusterz` currently.
180 with :command:`ipcluster` currently.
181
181
182 More details on using MPI with IPython can be found :ref:`here <parallelmpi>`.
182 More details on using MPI with IPython can be found :ref:`here <parallelmpi>`.
183
183
184
184
185 Using :command:`ipclusterz` in PBS mode
185 Using :command:`ipcluster` in PBS mode
186 ---------------------------------------
186 ---------------------------------------
187
187
188 The PBS mode uses the Portable Batch System [PBS]_ to start the engines.
188 The PBS mode uses the Portable Batch System [PBS]_ to start the engines.
189
189
190 As usual, we will start by creating a fresh profile::
190 As usual, we will start by creating a fresh profile::
191
191
192 $ ipclusterz create -p pbs
192 $ ipcluster create -p pbs
193
193
194 And in :file:`ipclusterz_config.py`, we will select the PBS launchers for the controller
194 And in :file:`ipcluster_config.py`, we will select the PBS launchers for the controller
195 and engines:
195 and engines:
196
196
197 .. sourcecode:: python
197 .. sourcecode:: python
@@ -213,7 +213,7 b' to specify your own. Here is a sample PBS script template:'
213 cd $$PBS_O_WORKDIR
213 cd $$PBS_O_WORKDIR
214 export PATH=$$HOME/usr/local/bin
214 export PATH=$$HOME/usr/local/bin
215 export PYTHONPATH=$$HOME/usr/local/lib/python2.7/site-packages
215 export PYTHONPATH=$$HOME/usr/local/lib/python2.7/site-packages
216 /usr/local/bin/mpiexec -n ${n} ipenginez --cluster_dir=${cluster_dir}
216 /usr/local/bin/mpiexec -n ${n} ipengine --cluster_dir=${cluster_dir}
217
217
218 There are a few important points about this template:
218 There are a few important points about this template:
219
219
@@ -232,8 +232,8 b' There are a few important points about this template:'
232 environment variables in the template, or in SGE, where the config lines start
232 environment variables in the template, or in SGE, where the config lines start
233 with ``#$``, which will have to be ``#$$``.
233 with ``#$``, which will have to be ``#$$``.
234
234
235 4. Any options to :command:`ipenginez` can be given in the batch script
235 4. Any options to :command:`ipengine` can be given in the batch script
236 template, or in :file:`ipenginez_config.py`.
236 template, or in :file:`ipengine_config.py`.
237
237
238 5. Depending on the configuration of you system, you may have to set
238 5. Depending on the configuration of you system, you may have to set
239 environment variables in the script template.
239 environment variables in the script template.
@@ -251,11 +251,11 b' The controller template should be similar, but simpler:'
251 cd $$PBS_O_WORKDIR
251 cd $$PBS_O_WORKDIR
252 export PATH=$$HOME/usr/local/bin
252 export PATH=$$HOME/usr/local/bin
253 export PYTHONPATH=$$HOME/usr/local/lib/python2.7/site-packages
253 export PYTHONPATH=$$HOME/usr/local/lib/python2.7/site-packages
254 ipcontrollerz --cluster_dir=${cluster_dir}
254 ipcontroller --cluster_dir=${cluster_dir}
255
255
256
256
257 Once you have created these scripts, save them with names like
257 Once you have created these scripts, save them with names like
258 :file:`pbs.engine.template`. Now you can load them into the :file:`ipclusterz_config` with:
258 :file:`pbs.engine.template`. Now you can load them into the :file:`ipcluster_config` with:
259
259
260 .. sourcecode:: python
260 .. sourcecode:: python
261
261
@@ -264,12 +264,12 b' Once you have created these scripts, save them with names like'
264 c.PBSControllerLauncher.batch_template_file = "pbs.controller.template"
264 c.PBSControllerLauncher.batch_template_file = "pbs.controller.template"
265
265
266
266
267 Alternately, you can just define the templates as strings inside :file:`ipclusterz_config`.
267 Alternately, you can just define the templates as strings inside :file:`ipcluster_config`.
268
268
269 Whether you are using your own templates or our defaults, the extra configurables available are
269 Whether you are using your own templates or our defaults, the extra configurables available are
270 the number of engines to launch (``$n``, and the batch system queue to which the jobs are to be
270 the number of engines to launch (``$n``, and the batch system queue to which the jobs are to be
271 submitted (``$queue``)). These are configurables, and can be specified in
271 submitted (``$queue``)). These are configurables, and can be specified in
272 :file:`ipclusterz_config`:
272 :file:`ipcluster_config`:
273
273
274 .. sourcecode:: python
274 .. sourcecode:: python
275
275
@@ -279,7 +279,7 b' submitted (``$queue``)). These are configurables, and can be specified in'
279 Note that assuming you are running PBS on a multi-node cluster, the Controller's default behavior
279 Note that assuming you are running PBS on a multi-node cluster, the Controller's default behavior
280 of listening only on localhost is likely too restrictive. In this case, also assuming the
280 of listening only on localhost is likely too restrictive. In this case, also assuming the
281 nodes are safely behind a firewall, you can simply instruct the Controller to listen for
281 nodes are safely behind a firewall, you can simply instruct the Controller to listen for
282 connections on all its interfaces, by adding in :file:`ipcontrollerz_config`:
282 connections on all its interfaces, by adding in :file:`ipcontroller_config`:
283
283
284 .. sourcecode:: python
284 .. sourcecode:: python
285
285
@@ -287,9 +287,9 b' connections on all its interfaces, by adding in :file:`ipcontrollerz_config`:'
287
287
288 You can now run the cluster with::
288 You can now run the cluster with::
289
289
290 $ ipclusterz start -p pbs -n 128
290 $ ipcluster start -p pbs -n 128
291
291
292 Additional configuration options can be found in the PBS section of :file:`ipclusterz_config`.
292 Additional configuration options can be found in the PBS section of :file:`ipcluster_config`.
293
293
294 .. note::
294 .. note::
295
295
@@ -298,12 +298,12 b' Additional configuration options can be found in the PBS section of :file:`ipclu'
298 and with further configuration in similar batch systems like Condor.
298 and with further configuration in similar batch systems like Condor.
299
299
300
300
301 Using :command:`ipclusterz` in SSH mode
301 Using :command:`ipcluster` in SSH mode
302 ---------------------------------------
302 ---------------------------------------
303
303
304
304
305 The SSH mode uses :command:`ssh` to execute :command:`ipenginez` on remote
305 The SSH mode uses :command:`ssh` to execute :command:`ipengine` on remote
306 nodes and :command:`ipcontrollerz` can be run remotely as well, or on localhost.
306 nodes and :command:`ipcontroller` can be run remotely as well, or on localhost.
307
307
308 .. note::
308 .. note::
309
309
@@ -312,9 +312,9 b' nodes and :command:`ipcontrollerz` can be run remotely as well, or on localhost.'
312
312
313 As usual, we start by creating a clean profile::
313 As usual, we start by creating a clean profile::
314
314
315 $ ipclusterz create -p ssh
315 $ ipcluster create -p ssh
316
316
317 To use this mode, select the SSH launchers in :file:`ipclusterz_config.py`:
317 To use this mode, select the SSH launchers in :file:`ipcluster_config.py`:
318
318
319 .. sourcecode:: python
319 .. sourcecode:: python
320
320
@@ -331,9 +331,9 b" The controller's remote location and configuration can be specified:"
331 # c.SSHControllerLauncher.hostname = 'controller.example.com'
331 # c.SSHControllerLauncher.hostname = 'controller.example.com'
332 # c.SSHControllerLauncher.user = os.environ.get('USER','username')
332 # c.SSHControllerLauncher.user = os.environ.get('USER','username')
333
333
334 # Set the arguments to be passed to ipcontrollerz
334 # Set the arguments to be passed to ipcontroller
335 # note that remotely launched ipcontrollerz will not get the contents of
335 # note that remotely launched ipcontroller will not get the contents of
336 # the local ipcontrollerz_config.py unless it resides on the *remote host*
336 # the local ipcontroller_config.py unless it resides on the *remote host*
337 # in the location specified by the --cluster_dir argument.
337 # in the location specified by the --cluster_dir argument.
338 # c.SSHControllerLauncher.program_args = ['-r', '-ip', '0.0.0.0', '--cluster_dir', '/path/to/cd']
338 # c.SSHControllerLauncher.program_args = ['-r', '-ip', '0.0.0.0', '--cluster_dir', '/path/to/cd']
339
339
@@ -357,46 +357,46 b' on that host.'
357 * The `engines` dict, where the keys are the host we want to run engines on and
357 * The `engines` dict, where the keys are the host we want to run engines on and
358 the value is the number of engines to run on that host.
358 the value is the number of engines to run on that host.
359 * on host3, the value is a tuple, where the number of engines is first, and the arguments
359 * on host3, the value is a tuple, where the number of engines is first, and the arguments
360 to be passed to :command:`ipenginez` are the second element.
360 to be passed to :command:`ipengine` are the second element.
361
361
362 For engines without explicitly specified arguments, the default arguments are set in
362 For engines without explicitly specified arguments, the default arguments are set in
363 a single location:
363 a single location:
364
364
365 .. sourcecode:: python
365 .. sourcecode:: python
366
366
367 c.SSHEngineSetLauncher.engine_args = ['--cluster_dir', '/path/to/clusterz_ssh']
367 c.SSHEngineSetLauncher.engine_args = ['--cluster_dir', '/path/to/cluster_ssh']
368
368
369 Current limitations of the SSH mode of :command:`ipclusterz` are:
369 Current limitations of the SSH mode of :command:`ipcluster` are:
370
370
371 * Untested on Windows. Would require a working :command:`ssh` on Windows.
371 * Untested on Windows. Would require a working :command:`ssh` on Windows.
372 Also, we are using shell scripts to setup and execute commands on remote
372 Also, we are using shell scripts to setup and execute commands on remote
373 hosts.
373 hosts.
374 * No file movement -
374 * No file movement -
375
375
376 Using the :command:`ipcontrollerz` and :command:`ipenginez` commands
376 Using the :command:`ipcontroller` and :command:`ipengine` commands
377 ====================================================================
377 ====================================================================
378
378
379 It is also possible to use the :command:`ipcontrollerz` and :command:`ipenginez`
379 It is also possible to use the :command:`ipcontroller` and :command:`ipengine`
380 commands to start your controller and engines. This approach gives you full
380 commands to start your controller and engines. This approach gives you full
381 control over all aspects of the startup process.
381 control over all aspects of the startup process.
382
382
383 Starting the controller and engine on your local machine
383 Starting the controller and engine on your local machine
384 --------------------------------------------------------
384 --------------------------------------------------------
385
385
386 To use :command:`ipcontrollerz` and :command:`ipenginez` to start things on your
386 To use :command:`ipcontroller` and :command:`ipengine` to start things on your
387 local machine, do the following.
387 local machine, do the following.
388
388
389 First start the controller::
389 First start the controller::
390
390
391 $ ipcontrollerz
391 $ ipcontroller
392
392
393 Next, start however many instances of the engine you want using (repeatedly)
393 Next, start however many instances of the engine you want using (repeatedly)
394 the command::
394 the command::
395
395
396 $ ipenginez
396 $ ipengine
397
397
398 The engines should start and automatically connect to the controller using the
398 The engines should start and automatically connect to the controller using the
399 JSON files in :file:`~/.ipython/clusterz_default/security`. You are now ready to use the
399 JSON files in :file:`~/.ipython/cluster_default/security`. You are now ready to use the
400 controller and engines from IPython.
400 controller and engines from IPython.
401
401
402 .. warning::
402 .. warning::
@@ -418,18 +418,18 b' Starting the controller and engines on different hosts'
418 When the controller and engines are running on different hosts, things are
418 When the controller and engines are running on different hosts, things are
419 slightly more complicated, but the underlying ideas are the same:
419 slightly more complicated, but the underlying ideas are the same:
420
420
421 1. Start the controller on a host using :command:`ipcontrollerz`.
421 1. Start the controller on a host using :command:`ipcontroller`.
422 2. Copy :file:`ipcontroller-engine.json` from :file:`~/.ipython/cluster_<profile>/security` on
422 2. Copy :file:`ipcontroller-engine.json` from :file:`~/.ipython/cluster_<profile>/security` on
423 the controller's host to the host where the engines will run.
423 the controller's host to the host where the engines will run.
424 3. Use :command:`ipenginez` on the engine's hosts to start the engines.
424 3. Use :command:`ipengine` on the engine's hosts to start the engines.
425
425
426 The only thing you have to be careful of is to tell :command:`ipenginez` where
426 The only thing you have to be careful of is to tell :command:`ipengine` where
427 the :file:`ipcontroller-engine.json` file is located. There are two ways you
427 the :file:`ipcontroller-engine.json` file is located. There are two ways you
428 can do this:
428 can do this:
429
429
430 * Put :file:`ipcontroller-engine.json` in the :file:`~/.ipython/cluster_<profile>/security`
430 * Put :file:`ipcontroller-engine.json` in the :file:`~/.ipython/cluster_<profile>/security`
431 directory on the engine's host, where it will be found automatically.
431 directory on the engine's host, where it will be found automatically.
432 * Call :command:`ipenginez` with the ``--file=full_path_to_the_file``
432 * Call :command:`ipengine` with the ``--file=full_path_to_the_file``
433 flag.
433 flag.
434
434
435 The ``--file`` flag works like this::
435 The ``--file`` flag works like this::
@@ -455,7 +455,7 b' any point in the future.'
455 To do this, the only thing you have to do is specify the `-r` flag, so that
455 To do this, the only thing you have to do is specify the `-r` flag, so that
456 the connection information in the JSON files remains accurate::
456 the connection information in the JSON files remains accurate::
457
457
458 $ ipcontrollerz -r
458 $ ipcontroller -r
459
459
460 Then, just copy the JSON files over the first time and you are set. You can
460 Then, just copy the JSON files over the first time and you are set. You can
461 start and stop the controller and engines any many times as you want in the
461 start and stop the controller and engines any many times as you want in the
@@ -478,7 +478,7 b' IPython and can be found in the directory :file:`~/.ipython/cluster_<profile>/lo'
478 Sending the log files to us will often help us to debug any problems.
478 Sending the log files to us will often help us to debug any problems.
479
479
480
480
481 Configuring `ipcontrollerz`
481 Configuring `ipcontroller`
482 ---------------------------
482 ---------------------------
483
483
484 Ports and addresses
484 Ports and addresses
@@ -493,7 +493,7 b' Database Backend'
493
493
494
494
495
495
496 Configuring `ipenginez`
496 Configuring `ipengine`
497 -----------------------
497 -----------------------
498
498
499 .. note::
499 .. note::
@@ -130,7 +130,7 b' way.'
130 There is exactly one key per cluster - it must be the same everywhere. Typically, the
130 There is exactly one key per cluster - it must be the same everywhere. Typically, the
131 controller creates this key, and stores it in the private connection files
131 controller creates this key, and stores it in the private connection files
132 `ipython-{engine|client}.json`. These files are typically stored in the
132 `ipython-{engine|client}.json`. These files are typically stored in the
133 `~/.ipython/clusterz_<profile>/security` directory, and are maintained as readable only by
133 `~/.ipython/cluster_<profile>/security` directory, and are maintained as readable only by
134 the owner, just as is common practice with a user's keys in their `.ssh` directory.
134 the owner, just as is common practice with a user's keys in their `.ssh` directory.
135
135
136 .. warning::
136 .. warning::
@@ -22,9 +22,9 b' Starting the IPython controller and engines'
22
22
23 To follow along with this tutorial, you will need to start the IPython
23 To follow along with this tutorial, you will need to start the IPython
24 controller and four IPython engines. The simplest way of doing this is to use
24 controller and four IPython engines. The simplest way of doing this is to use
25 the :command:`ipclusterz` command::
25 the :command:`ipcluster` command::
26
26
27 $ ipclusterz start -n 4
27 $ ipcluster start -n 4
28
28
29 For more detailed information about starting the controller and engines, see
29 For more detailed information about starting the controller and engines, see
30 our :ref:`introduction <ip1par>` to using IPython for parallel computing.
30 our :ref:`introduction <ip1par>` to using IPython for parallel computing.
@@ -321,16 +321,16 b' Schedulers'
321 There are a variety of valid ways to determine where jobs should be assigned in a
321 There are a variety of valid ways to determine where jobs should be assigned in a
322 load-balancing situation. In IPython, we support several standard schemes, and
322 load-balancing situation. In IPython, we support several standard schemes, and
323 even make it easy to define your own. The scheme can be selected via the ``--scheme``
323 even make it easy to define your own. The scheme can be selected via the ``--scheme``
324 argument to :command:`ipcontrollerz`, or in the :attr:`HubFactory.scheme` attribute
324 argument to :command:`ipcontroller`, or in the :attr:`HubFactory.scheme` attribute
325 of a controller config object.
325 of a controller config object.
326
326
327 The built-in routing schemes:
327 The built-in routing schemes:
328
328
329 To select one of these schemes, simply do::
329 To select one of these schemes, simply do::
330
330
331 $ ipcontrollerz --scheme <schemename>
331 $ ipcontroller --scheme <schemename>
332 for instance:
332 for instance:
333 $ ipcontrollerz --scheme lru
333 $ ipcontroller --scheme lru
334
334
335 lru: Least Recently Used
335 lru: Least Recently Used
336
336
1 NO CONTENT: file renamed from docs/source/parallelz/parallel_transition.txt to docs/source/parallel/parallel_transition.txt
NO CONTENT: file renamed from docs/source/parallelz/parallel_transition.txt to docs/source/parallel/parallel_transition.txt
@@ -144,25 +144,25 b' in parallel on the engines from within the IPython shell using an appropriate'
144 client. This includes the ability to interact with, plot and visualize data
144 client. This includes the ability to interact with, plot and visualize data
145 from the engines.
145 from the engines.
146
146
147 IPython has a command line program called :command:`ipclusterz` that automates
147 IPython has a command line program called :command:`ipcluster` that automates
148 all aspects of starting the controller and engines on the compute nodes.
148 all aspects of starting the controller and engines on the compute nodes.
149 :command:`ipclusterz` has full support for the Windows HPC job scheduler,
149 :command:`ipcluster` has full support for the Windows HPC job scheduler,
150 meaning that :command:`ipclusterz` can use this job scheduler to start the
150 meaning that :command:`ipcluster` can use this job scheduler to start the
151 controller and engines. In our experience, the Windows HPC job scheduler is
151 controller and engines. In our experience, the Windows HPC job scheduler is
152 particularly well suited for interactive applications, such as IPython. Once
152 particularly well suited for interactive applications, such as IPython. Once
153 :command:`ipclusterz` is configured properly, a user can start an IPython
153 :command:`ipcluster` is configured properly, a user can start an IPython
154 cluster from their local workstation almost instantly, without having to log
154 cluster from their local workstation almost instantly, without having to log
155 on to the head node (as is typically required by Unix based job schedulers).
155 on to the head node (as is typically required by Unix based job schedulers).
156 This enables a user to move seamlessly between serial and parallel
156 This enables a user to move seamlessly between serial and parallel
157 computations.
157 computations.
158
158
159 In this section we show how to use :command:`ipclusterz` to start an IPython
159 In this section we show how to use :command:`ipcluster` to start an IPython
160 cluster using the Windows HPC Server 2008 job scheduler. To make sure that
160 cluster using the Windows HPC Server 2008 job scheduler. To make sure that
161 :command:`ipclusterz` is installed and working properly, you should first try
161 :command:`ipcluster` is installed and working properly, you should first try
162 to start an IPython cluster on your local host. To do this, open a Windows
162 to start an IPython cluster on your local host. To do this, open a Windows
163 Command Prompt and type the following command::
163 Command Prompt and type the following command::
164
164
165 ipclusterz start -n 2
165 ipcluster start -n 2
166
166
167 You should see a number of messages printed to the screen, ending with
167 You should see a number of messages printed to the screen, ending with
168 "IPython cluster: started". The result should look something like the following
168 "IPython cluster: started". The result should look something like the following
@@ -174,12 +174,12 b' At this point, the controller and two engines are running on your local host.'
174 This configuration is useful for testing and for situations where you want to
174 This configuration is useful for testing and for situations where you want to
175 take advantage of multiple cores on your local computer.
175 take advantage of multiple cores on your local computer.
176
176
177 Now that we have confirmed that :command:`ipclusterz` is working properly, we
177 Now that we have confirmed that :command:`ipcluster` is working properly, we
178 describe how to configure and run an IPython cluster on an actual compute
178 describe how to configure and run an IPython cluster on an actual compute
179 cluster running Windows HPC Server 2008. Here is an outline of the needed
179 cluster running Windows HPC Server 2008. Here is an outline of the needed
180 steps:
180 steps:
181
181
182 1. Create a cluster profile using: ``ipclusterz create -p mycluster``
182 1. Create a cluster profile using: ``ipcluster create -p mycluster``
183
183
184 2. Edit configuration files in the directory :file:`.ipython\\cluster_mycluster`
184 2. Edit configuration files in the directory :file:`.ipython\\cluster_mycluster`
185
185
@@ -191,7 +191,7 b' Creating a cluster profile'
191 In most cases, you will have to create a cluster profile to use IPython on a
191 In most cases, you will have to create a cluster profile to use IPython on a
192 cluster. A cluster profile is a name (like "mycluster") that is associated
192 cluster. A cluster profile is a name (like "mycluster") that is associated
193 with a particular cluster configuration. The profile name is used by
193 with a particular cluster configuration. The profile name is used by
194 :command:`ipclusterz` when working with the cluster.
194 :command:`ipcluster` when working with the cluster.
195
195
196 Associated with each cluster profile is a cluster directory. This cluster
196 Associated with each cluster profile is a cluster directory. This cluster
197 directory is a specially named directory (typically located in the
197 directory is a specially named directory (typically located in the
@@ -204,10 +204,10 b' security keys. The naming convention for cluster directories is:'
204 To create a new cluster profile (named "mycluster") and the associated cluster
204 To create a new cluster profile (named "mycluster") and the associated cluster
205 directory, type the following command at the Windows Command Prompt::
205 directory, type the following command at the Windows Command Prompt::
206
206
207 ipclusterz create -p mycluster
207 ipcluster create -p mycluster
208
208
209 The output of this command is shown in the screenshot below. Notice how
209 The output of this command is shown in the screenshot below. Notice how
210 :command:`ipclusterz` prints out the location of the newly created cluster
210 :command:`ipcluster` prints out the location of the newly created cluster
211 directory.
211 directory.
212
212
213 .. image:: ipcluster_create.*
213 .. image:: ipcluster_create.*
@@ -218,19 +218,19 b' Configuring a cluster profile'
218 Next, you will need to configure the newly created cluster profile by editing
218 Next, you will need to configure the newly created cluster profile by editing
219 the following configuration files in the cluster directory:
219 the following configuration files in the cluster directory:
220
220
221 * :file:`ipclusterz_config.py`
221 * :file:`ipcluster_config.py`
222 * :file:`ipcontroller_config.py`
222 * :file:`ipcontroller_config.py`
223 * :file:`ipengine_config.py`
223 * :file:`ipengine_config.py`
224
224
225 When :command:`ipclusterz` is run, these configuration files are used to
225 When :command:`ipcluster` is run, these configuration files are used to
226 determine how the engines and controller will be started. In most cases,
226 determine how the engines and controller will be started. In most cases,
227 you will only have to set a few of the attributes in these files.
227 you will only have to set a few of the attributes in these files.
228
228
229 To configure :command:`ipclusterz` to use the Windows HPC job scheduler, you
229 To configure :command:`ipcluster` to use the Windows HPC job scheduler, you
230 will need to edit the following attributes in the file
230 will need to edit the following attributes in the file
231 :file:`ipclusterz_config.py`::
231 :file:`ipcluster_config.py`::
232
232
233 # Set these at the top of the file to tell ipclusterz to use the
233 # Set these at the top of the file to tell ipcluster to use the
234 # Windows HPC job scheduler.
234 # Windows HPC job scheduler.
235 c.Global.controller_launcher = \
235 c.Global.controller_launcher = \
236 'IPython.parallel.launcher.WindowsHPCControllerLauncher'
236 'IPython.parallel.launcher.WindowsHPCControllerLauncher'
@@ -257,15 +257,15 b' Starting the cluster profile'
257 Once a cluster profile has been configured, starting an IPython cluster using
257 Once a cluster profile has been configured, starting an IPython cluster using
258 the profile is simple::
258 the profile is simple::
259
259
260 ipclusterz start -p mycluster -n 32
260 ipcluster start -p mycluster -n 32
261
261
262 The ``-n`` option tells :command:`ipclusterz` how many engines to start (in
262 The ``-n`` option tells :command:`ipcluster` how many engines to start (in
263 this case 32). Stopping the cluster is as simple as typing Control-C.
263 this case 32). Stopping the cluster is as simple as typing Control-C.
264
264
265 Using the HPC Job Manager
265 Using the HPC Job Manager
266 -------------------------
266 -------------------------
267
267
268 When ``ipclusterz start`` is run the first time, :command:`ipclusterz` creates
268 When ``ipcluster start`` is run the first time, :command:`ipcluster` creates
269 two XML job description files in the cluster directory:
269 two XML job description files in the cluster directory:
270
270
271 * :file:`ipcontroller_job.xml`
271 * :file:`ipcontroller_job.xml`
@@ -273,8 +273,8 b' two XML job description files in the cluster directory:'
273
273
274 Once these files have been created, they can be imported into the HPC Job
274 Once these files have been created, they can be imported into the HPC Job
275 Manager application. Then, the controller and engines for that profile can be
275 Manager application. Then, the controller and engines for that profile can be
276 started using the HPC Job Manager directly, without using :command:`ipclusterz`.
276 started using the HPC Job Manager directly, without using :command:`ipcluster`.
277 However, anytime the cluster profile is re-configured, ``ipclusterz start``
277 However, anytime the cluster profile is re-configured, ``ipcluster start``
278 must be run again to regenerate the XML job description files. The
278 must be run again to regenerate the XML job description files. The
279 following screenshot shows what the HPC Job Manager interface looks like
279 following screenshot shows what the HPC Job Manager interface looks like
280 with a running IPython cluster.
280 with a running IPython cluster.
1 NO CONTENT: file renamed from docs/source/parallelz/simpledag.pdf to docs/source/parallel/simpledag.pdf
NO CONTENT: file renamed from docs/source/parallelz/simpledag.pdf to docs/source/parallel/simpledag.pdf
1 NO CONTENT: file renamed from docs/source/parallelz/simpledag.png to docs/source/parallel/simpledag.png
NO CONTENT: file renamed from docs/source/parallelz/simpledag.png to docs/source/parallel/simpledag.png
1 NO CONTENT: file renamed from docs/source/parallelz/single_digits.pdf to docs/source/parallel/single_digits.pdf
NO CONTENT: file renamed from docs/source/parallelz/single_digits.pdf to docs/source/parallel/single_digits.pdf
1 NO CONTENT: file renamed from docs/source/parallelz/single_digits.png to docs/source/parallel/single_digits.png
NO CONTENT: file renamed from docs/source/parallelz/single_digits.png to docs/source/parallel/single_digits.png
1 NO CONTENT: file renamed from docs/source/parallelz/two_digit_counts.pdf to docs/source/parallel/two_digit_counts.pdf
NO CONTENT: file renamed from docs/source/parallelz/two_digit_counts.pdf to docs/source/parallel/two_digit_counts.pdf
1 NO CONTENT: file renamed from docs/source/parallelz/two_digit_counts.png to docs/source/parallel/two_digit_counts.png
NO CONTENT: file renamed from docs/source/parallelz/two_digit_counts.png to docs/source/parallel/two_digit_counts.png
1 NO CONTENT: file renamed from docs/source/parallelz/winhpc_index.txt to docs/source/parallel/winhpc_index.txt
NO CONTENT: file renamed from docs/source/parallelz/winhpc_index.txt to docs/source/parallel/winhpc_index.txt
@@ -215,16 +215,16 b" if 'setuptools' in sys.modules:"
215 'ipython = IPython.frontend.terminal.ipapp:launch_new_instance',
215 'ipython = IPython.frontend.terminal.ipapp:launch_new_instance',
216 'ipython-qtconsole = IPython.frontend.qt.console.ipythonqt:main',
216 'ipython-qtconsole = IPython.frontend.qt.console.ipythonqt:main',
217 'pycolor = IPython.utils.PyColorize:main',
217 'pycolor = IPython.utils.PyColorize:main',
218 'ipcontrollerz = IPython.parallel.ipcontrollerapp:launch_new_instance',
218 'ipcontroller = IPython.parallel.ipcontrollerapp:launch_new_instance',
219 'ipenginez = IPython.parallel.ipengineapp:launch_new_instance',
219 'ipengine = IPython.parallel.ipengineapp:launch_new_instance',
220 'iploggerz = IPython.parallel.iploggerapp:launch_new_instance',
220 'iplogger = IPython.parallel.iploggerapp:launch_new_instance',
221 'ipclusterz = IPython.parallel.ipclusterapp:launch_new_instance',
221 'ipcluster = IPython.parallel.ipclusterapp:launch_new_instance',
222 'iptest = IPython.testing.iptest:main',
222 'iptest = IPython.testing.iptest:main',
223 'irunner = IPython.lib.irunner:main'
223 'irunner = IPython.lib.irunner:main'
224 ]
224 ]
225 }
225 }
226 setup_args['extras_require'] = dict(
226 setup_args['extras_require'] = dict(
227 zmq = 'pyzmq>=2.0.10',
227 zmq = 'pyzmq>=2.0.10.1',
228 doc='Sphinx>=0.3',
228 doc='Sphinx>=0.3',
229 test='nose>=0.10.1',
229 test='nose>=0.10.1',
230 security='pyOpenSSL>=0.6'
230 security='pyOpenSSL>=0.6'
@@ -127,6 +127,7 b' def find_packages():'
127 add_package(packages, 'frontend.qt.console', tests=True)
127 add_package(packages, 'frontend.qt.console', tests=True)
128 add_package(packages, 'frontend.terminal', tests=True)
128 add_package(packages, 'frontend.terminal', tests=True)
129 add_package(packages, 'lib', tests=True)
129 add_package(packages, 'lib', tests=True)
130 add_package(packages, 'parallel', tests=True)
130 add_package(packages, 'quarantine', tests=True)
131 add_package(packages, 'quarantine', tests=True)
131 add_package(packages, 'scripts')
132 add_package(packages, 'scripts')
132 add_package(packages, 'testing', tests=True)
133 add_package(packages, 'testing', tests=True)
@@ -134,7 +135,6 b' def find_packages():'
134 add_package(packages, 'utils', tests=True)
135 add_package(packages, 'utils', tests=True)
135 add_package(packages, 'zmq')
136 add_package(packages, 'zmq')
136 add_package(packages, 'zmq.pylab')
137 add_package(packages, 'zmq.pylab')
137 add_package(packages, 'parallel')
138 return packages
138 return packages
139
139
140 #---------------------------------------------------------------------------
140 #---------------------------------------------------------------------------
@@ -265,10 +265,10 b' def find_scripts():'
265 parallel_scripts = pjoin('IPython','parallel','scripts')
265 parallel_scripts = pjoin('IPython','parallel','scripts')
266 main_scripts = pjoin('IPython','scripts')
266 main_scripts = pjoin('IPython','scripts')
267 scripts = [
267 scripts = [
268 pjoin(parallel_scripts, 'ipenginez'),
268 pjoin(parallel_scripts, 'ipengine'),
269 pjoin(parallel_scripts, 'ipcontrollerz'),
269 pjoin(parallel_scripts, 'ipcontroller'),
270 pjoin(parallel_scripts, 'ipclusterz'),
270 pjoin(parallel_scripts, 'ipcluster'),
271 pjoin(parallel_scripts, 'iploggerz'),
271 pjoin(parallel_scripts, 'iplogger'),
272 pjoin(main_scripts, 'ipython'),
272 pjoin(main_scripts, 'ipython'),
273 pjoin(main_scripts, 'ipython-qtconsole'),
273 pjoin(main_scripts, 'ipython-qtconsole'),
274 pjoin(main_scripts, 'pycolor'),
274 pjoin(main_scripts, 'pycolor'),
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now