Show More
@@ -50,6 +50,7 b' except ImportError:' | |||||
50 | from zmq.eventloop import ioloop |
|
50 | from zmq.eventloop import ioloop | |
51 |
|
51 | |||
52 | # from IPython.config.configurable import Configurable |
|
52 | # from IPython.config.configurable import Configurable | |
|
53 | from IPython.utils.text import EvalFormatter | |||
53 | from IPython.utils.traitlets import Any, Int, List, Unicode, Dict, Instance |
|
54 | from IPython.utils.traitlets import Any, Int, List, Unicode, Dict, Instance | |
54 | from IPython.utils.path import get_ipython_module_path |
|
55 | from IPython.utils.path import get_ipython_module_path | |
55 | from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError |
|
56 | from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError | |
@@ -839,6 +840,8 b' class BatchSystemLauncher(BaseLauncher):' | |||||
839 | batch_file = Unicode(u'') |
|
840 | batch_file = Unicode(u'') | |
840 | # the format dict used with batch_template: |
|
841 | # the format dict used with batch_template: | |
841 | context = Dict() |
|
842 | context = Dict() | |
|
843 | # the Formatter instance for rendering the templates: | |||
|
844 | formatter = Instance(EvalFormatter, (), {}) | |||
842 |
|
845 | |||
843 |
|
846 | |||
844 | def find_args(self): |
|
847 | def find_args(self): | |
@@ -888,7 +891,7 b' class BatchSystemLauncher(BaseLauncher):' | |||||
888 | firstline, rest = self.batch_template.split('\n',1) |
|
891 | firstline, rest = self.batch_template.split('\n',1) | |
889 | self.batch_template = u'\n'.join([firstline, self.queue_template, rest]) |
|
892 | self.batch_template = u'\n'.join([firstline, self.queue_template, rest]) | |
890 |
|
893 | |||
891 |
script_as_string = self. |
|
894 | script_as_string = self.formatter.format(self.batch_template, **self.context) | |
892 | self.log.info('Writing instantiated batch script: %s' % self.batch_file) |
|
895 | self.log.info('Writing instantiated batch script: %s' % self.batch_file) | |
893 |
|
896 | |||
894 | with open(self.batch_file, 'w') as f: |
|
897 | with open(self.batch_file, 'w') as f: |
@@ -19,6 +19,7 b' import __main__' | |||||
19 | import os |
|
19 | import os | |
20 | import re |
|
20 | import re | |
21 | import shutil |
|
21 | import shutil | |
|
22 | from string import Formatter | |||
22 |
|
23 | |||
23 | from IPython.external.path import path |
|
24 | from IPython.external.path import path | |
24 |
|
25 | |||
@@ -519,3 +520,41 b' def format_screen(strng):' | |||||
519 | strng = par_re.sub('',strng) |
|
520 | strng = par_re.sub('',strng) | |
520 | return strng |
|
521 | return strng | |
521 |
|
522 | |||
|
523 | ||||
|
524 | class EvalFormatter(Formatter): | |||
|
525 | """A String Formatter that allows evaluation of simple expressions. | |||
|
526 | ||||
|
527 | Any time a format key is not found in the kwargs, | |||
|
528 | it will be tried as an expression in the kwargs namespace. | |||
|
529 | ||||
|
530 | This is to be used in templating cases, such as the parallel batch | |||
|
531 | script templates, where simple arithmetic on arguments is useful. | |||
|
532 | ||||
|
533 | Examples | |||
|
534 | -------- | |||
|
535 | ||||
|
536 | In [1]: f = EvalFormatter() | |||
|
537 | In [2]: f.format('{n/4}', n=8) | |||
|
538 | Out[2]: '2' | |||
|
539 | ||||
|
540 | In [3]: f.format('{range(3)}') | |||
|
541 | Out[3]: '[0, 1, 2]' | |||
|
542 | ||||
|
543 | In [4]: f.format('{3*2}') | |||
|
544 | Out[4]: '6' | |||
|
545 | """ | |||
|
546 | ||||
|
547 | def get_value(self, key, args, kwargs): | |||
|
548 | if isinstance(key, (int, long)): | |||
|
549 | return args[key] | |||
|
550 | elif key in kwargs: | |||
|
551 | return kwargs[key] | |||
|
552 | else: | |||
|
553 | # evaluate the expression using kwargs as namespace | |||
|
554 | try: | |||
|
555 | return eval(key, kwargs) | |||
|
556 | except Exception: | |||
|
557 | # classify all bad expressions as key errors | |||
|
558 | raise KeyError(key) | |||
|
559 | ||||
|
560 |
@@ -208,35 +208,32 b' to specify your own. Here is a sample PBS script template:' | |||||
208 | #PBS -N ipython |
|
208 | #PBS -N ipython | |
209 | #PBS -j oe |
|
209 | #PBS -j oe | |
210 | #PBS -l walltime=00:10:00 |
|
210 | #PBS -l walltime=00:10:00 | |
211 |
#PBS -l nodes= |
|
211 | #PBS -l nodes={n/4}:ppn=4 | |
212 |
#PBS -q |
|
212 | #PBS -q {queue} | |
213 |
|
213 | |||
214 |
cd $ |
|
214 | cd $PBS_O_WORKDIR | |
215 |
export PATH=$ |
|
215 | export PATH=$HOME/usr/local/bin | |
216 |
export PYTHONPATH=$ |
|
216 | export PYTHONPATH=$HOME/usr/local/lib/python2.7/site-packages | |
217 |
/usr/local/bin/mpiexec -n |
|
217 | /usr/local/bin/mpiexec -n {n} ipengine profile_dir={profile_dir} | |
218 |
|
218 | |||
219 | There are a few important points about this template: |
|
219 | There are a few important points about this template: | |
220 |
|
220 | |||
221 |
1. This template will be rendered at runtime using IPython's : |
|
221 | 1. This template will be rendered at runtime using IPython's :class:`EvalFormatter`. | |
222 | template engine. |
|
222 | This is simply a subclass of :class:`string.Formatter` that allows simple expressions | |
|
223 | on keys. | |||
223 |
|
224 | |||
224 | 2. Instead of putting in the actual number of engines, use the notation |
|
225 | 2. Instead of putting in the actual number of engines, use the notation | |
225 |
`` |
|
226 | ``{n}`` to indicate the number of engines to be started. You can also use | |
226 |
expressions like `` |
|
227 | expressions like ``{n/4}`` in the template to indicate the number of nodes. | |
227 |
|
|
228 | There will always be ``{n}`` and ``{profile_dir}`` variables passed to the formatter. | |
228 | These allow the batch system to know how many engines, and where the configuration |
|
229 | These allow the batch system to know how many engines, and where the configuration | |
229 |
files reside. The same is true for the batch queue, with the template variable |
|
230 | files reside. The same is true for the batch queue, with the template variable | |
|
231 | ``{queue}``. | |||
230 |
|
232 | |||
231 | 3. Because ``$`` is a special character used by the template engine, you must |
|
233 | 3. Any options to :command:`ipengine` can be given in the batch script | |
232 | escape any ``$`` by using ``$$``. This is important when referring to |
|
|||
233 | environment variables in the template, or in SGE, where the config lines start |
|
|||
234 | with ``#$``, which will have to be ``#$$``. |
|
|||
235 |
|
||||
236 | 4. Any options to :command:`ipengine` can be given in the batch script |
|
|||
237 | template, or in :file:`ipengine_config.py`. |
|
234 | template, or in :file:`ipengine_config.py`. | |
238 |
|
235 | |||
239 |
|
|
236 | 4. Depending on the configuration of you system, you may have to set | |
240 | environment variables in the script template. |
|
237 | environment variables in the script template. | |
241 |
|
238 | |||
242 | The controller template should be similar, but simpler: |
|
239 | The controller template should be similar, but simpler: | |
@@ -247,12 +244,12 b' The controller template should be similar, but simpler:' | |||||
247 | #PBS -j oe |
|
244 | #PBS -j oe | |
248 | #PBS -l walltime=00:10:00 |
|
245 | #PBS -l walltime=00:10:00 | |
249 | #PBS -l nodes=1:ppn=4 |
|
246 | #PBS -l nodes=1:ppn=4 | |
250 |
#PBS -q |
|
247 | #PBS -q {queue} | |
251 |
|
248 | |||
252 |
cd $ |
|
249 | cd $PBS_O_WORKDIR | |
253 |
export PATH=$ |
|
250 | export PATH=$HOME/usr/local/bin | |
254 |
export PYTHONPATH=$ |
|
251 | export PYTHONPATH=$HOME/usr/local/lib/python2.7/site-packages | |
255 |
ipcontroller |
|
252 | ipcontroller profile_dir={profile_dir} | |
256 |
|
253 | |||
257 |
|
254 | |||
258 | Once you have created these scripts, save them with names like |
|
255 | Once you have created these scripts, save them with names like | |
@@ -268,14 +265,14 b' Once you have created these scripts, save them with names like' | |||||
268 | Alternately, you can just define the templates as strings inside :file:`ipcluster_config`. |
|
265 | Alternately, you can just define the templates as strings inside :file:`ipcluster_config`. | |
269 |
|
266 | |||
270 | Whether you are using your own templates or our defaults, the extra configurables available are |
|
267 | Whether you are using your own templates or our defaults, the extra configurables available are | |
271 |
the number of engines to launch (`` |
|
268 | the number of engines to launch (``{n}``, and the batch system queue to which the jobs are to be | |
272 |
submitted (`` |
|
269 | submitted (``{queue}``)). These are configurables, and can be specified in | |
273 | :file:`ipcluster_config`: |
|
270 | :file:`ipcluster_config`: | |
274 |
|
271 | |||
275 | .. sourcecode:: python |
|
272 | .. sourcecode:: python | |
276 |
|
273 | |||
277 | c.PBSLauncher.queue = 'veryshort.q' |
|
274 | c.PBSLauncher.queue = 'veryshort.q' | |
278 |
c. |
|
275 | c.IPClusterEnginesApp.n = 64 | |
279 |
|
276 | |||
280 | Note that assuming you are running PBS on a multi-node cluster, the Controller's default behavior |
|
277 | Note that assuming you are running PBS on a multi-node cluster, the Controller's default behavior | |
281 | of listening only on localhost is likely too restrictive. In this case, also assuming the |
|
278 | of listening only on localhost is likely too restrictive. In this case, also assuming the | |
@@ -335,8 +332,8 b" The controller's remote location and configuration can be specified:" | |||||
335 | # Set the arguments to be passed to ipcontroller |
|
332 | # Set the arguments to be passed to ipcontroller | |
336 | # note that remotely launched ipcontroller will not get the contents of |
|
333 | # note that remotely launched ipcontroller will not get the contents of | |
337 | # the local ipcontroller_config.py unless it resides on the *remote host* |
|
334 | # the local ipcontroller_config.py unless it resides on the *remote host* | |
338 |
# in the location specified by the ` |
|
335 | # in the location specified by the `profile_dir` argument. | |
339 |
# c.SSHControllerLauncher.program_args = ['- |
|
336 | # c.SSHControllerLauncher.program_args = ['--reuse', 'ip=0.0.0.0', 'profile_dir=/path/to/cd'] | |
340 |
|
337 | |||
341 | .. note:: |
|
338 | .. note:: | |
342 |
|
339 | |||
@@ -352,7 +349,7 b' on that host.' | |||||
352 |
|
349 | |||
353 | c.SSHEngineSetLauncher.engines = { 'host1.example.com' : 2, |
|
350 | c.SSHEngineSetLauncher.engines = { 'host1.example.com' : 2, | |
354 | 'host2.example.com' : 5, |
|
351 | 'host2.example.com' : 5, | |
355 |
'host3.example.com' : (1, [' |
|
352 | 'host3.example.com' : (1, ['profile_dir=/home/different/location']), | |
356 | 'host4.example.com' : 8 } |
|
353 | 'host4.example.com' : 8 } | |
357 |
|
354 | |||
358 | * The `engines` dict, where the keys are the host we want to run engines on and |
|
355 | * The `engines` dict, where the keys are the host we want to run engines on and | |
@@ -365,7 +362,7 b' a single location:' | |||||
365 |
|
362 | |||
366 | .. sourcecode:: python |
|
363 | .. sourcecode:: python | |
367 |
|
364 | |||
368 |
c.SSHEngineSetLauncher.engine_args = [' |
|
365 | c.SSHEngineSetLauncher.engine_args = ['profile_dir=/path/to/cluster_ssh'] | |
369 |
|
366 | |||
370 | Current limitations of the SSH mode of :command:`ipcluster` are: |
|
367 | Current limitations of the SSH mode of :command:`ipcluster` are: | |
371 |
|
368 |
General Comments 0
You need to be logged in to leave comments.
Login now