##// END OF EJS Templates
add EvalFormatter for batch system (PBS) launcher templates...
MinRK -
Show More
@@ -50,6 +50,7 b' except ImportError:'
50 from zmq.eventloop import ioloop
50 from zmq.eventloop import ioloop
51
51
52 # from IPython.config.configurable import Configurable
52 # from IPython.config.configurable import Configurable
53 from IPython.utils.text import EvalFormatter
53 from IPython.utils.traitlets import Any, Int, List, Unicode, Dict, Instance
54 from IPython.utils.traitlets import Any, Int, List, Unicode, Dict, Instance
54 from IPython.utils.path import get_ipython_module_path
55 from IPython.utils.path import get_ipython_module_path
55 from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError
56 from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError
@@ -839,6 +840,8 b' class BatchSystemLauncher(BaseLauncher):'
839 batch_file = Unicode(u'')
840 batch_file = Unicode(u'')
840 # the format dict used with batch_template:
841 # the format dict used with batch_template:
841 context = Dict()
842 context = Dict()
843 # the Formatter instance for rendering the templates:
844 formatter = Instance(EvalFormatter, (), {})
842
845
843
846
844 def find_args(self):
847 def find_args(self):
@@ -888,7 +891,7 b' class BatchSystemLauncher(BaseLauncher):'
888 firstline, rest = self.batch_template.split('\n',1)
891 firstline, rest = self.batch_template.split('\n',1)
889 self.batch_template = u'\n'.join([firstline, self.queue_template, rest])
892 self.batch_template = u'\n'.join([firstline, self.queue_template, rest])
890
893
891 script_as_string = self.batch_template.format(**self.context)
894 script_as_string = self.formatter.format(self.batch_template, **self.context)
892 self.log.info('Writing instantiated batch script: %s' % self.batch_file)
895 self.log.info('Writing instantiated batch script: %s' % self.batch_file)
893
896
894 with open(self.batch_file, 'w') as f:
897 with open(self.batch_file, 'w') as f:
@@ -19,6 +19,7 b' import __main__'
19 import os
19 import os
20 import re
20 import re
21 import shutil
21 import shutil
22 from string import Formatter
22
23
23 from IPython.external.path import path
24 from IPython.external.path import path
24
25
@@ -519,3 +520,41 b' def format_screen(strng):'
519 strng = par_re.sub('',strng)
520 strng = par_re.sub('',strng)
520 return strng
521 return strng
521
522
523
524 class EvalFormatter(Formatter):
525 """A String Formatter that allows evaluation of simple expressions.
526
527 Any time a format key is not found in the kwargs,
528 it will be tried as an expression in the kwargs namespace.
529
530 This is to be used in templating cases, such as the parallel batch
531 script templates, where simple arithmetic on arguments is useful.
532
533 Examples
534 --------
535
536 In [1]: f = EvalFormatter()
537 In [2]: f.format('{n/4}', n=8)
538 Out[2]: '2'
539
540 In [3]: f.format('{range(3)}')
541 Out[3]: '[0, 1, 2]'
542
543 In [4]: f.format('{3*2}')
544 Out[4]: '6'
545 """
546
547 def get_value(self, key, args, kwargs):
548 if isinstance(key, (int, long)):
549 return args[key]
550 elif key in kwargs:
551 return kwargs[key]
552 else:
553 # evaluate the expression using kwargs as namespace
554 try:
555 return eval(key, kwargs)
556 except Exception:
557 # classify all bad expressions as key errors
558 raise KeyError(key)
559
560
@@ -208,35 +208,32 b' to specify your own. Here is a sample PBS script template:'
208 #PBS -N ipython
208 #PBS -N ipython
209 #PBS -j oe
209 #PBS -j oe
210 #PBS -l walltime=00:10:00
210 #PBS -l walltime=00:10:00
211 #PBS -l nodes=${n/4}:ppn=4
211 #PBS -l nodes={n/4}:ppn=4
212 #PBS -q $queue
212 #PBS -q {queue}
213
213
214 cd $$PBS_O_WORKDIR
214 cd $PBS_O_WORKDIR
215 export PATH=$$HOME/usr/local/bin
215 export PATH=$HOME/usr/local/bin
216 export PYTHONPATH=$$HOME/usr/local/lib/python2.7/site-packages
216 export PYTHONPATH=$HOME/usr/local/lib/python2.7/site-packages
217 /usr/local/bin/mpiexec -n ${n} ipengine cluster_dir=${cluster_dir}
217 /usr/local/bin/mpiexec -n {n} ipengine profile_dir={profile_dir}
218
218
219 There are a few important points about this template:
219 There are a few important points about this template:
220
220
221 1. This template will be rendered at runtime using IPython's :mod:`Itpl`
221 1. This template will be rendered at runtime using IPython's :class:`EvalFormatter`.
222 template engine.
222 This is simply a subclass of :class:`string.Formatter` that allows simple expressions
223 on keys.
223
224
224 2. Instead of putting in the actual number of engines, use the notation
225 2. Instead of putting in the actual number of engines, use the notation
225 ``${n}`` to indicate the number of engines to be started. You can also uses
226 ``{n}`` to indicate the number of engines to be started. You can also use
226 expressions like ``${n/4}`` in the template to indicate the number of
227 expressions like ``{n/4}`` in the template to indicate the number of nodes.
227 nodes. There will always be a ${n} and ${cluster_dir} variable passed to the template.
228 There will always be ``{n}`` and ``{profile_dir}`` variables passed to the formatter.
228 These allow the batch system to know how many engines, and where the configuration
229 These allow the batch system to know how many engines, and where the configuration
229 files reside. The same is true for the batch queue, with the template variable ``$queue``.
230 files reside. The same is true for the batch queue, with the template variable
231 ``{queue}``.
230
232
231 3. Because ``$`` is a special character used by the template engine, you must
233 3. Any options to :command:`ipengine` can be given in the batch script
232 escape any ``$`` by using ``$$``. This is important when referring to
233 environment variables in the template, or in SGE, where the config lines start
234 with ``#$``, which will have to be ``#$$``.
235
236 4. Any options to :command:`ipengine` can be given in the batch script
237 template, or in :file:`ipengine_config.py`.
234 template, or in :file:`ipengine_config.py`.
238
235
239 5. Depending on the configuration of you system, you may have to set
236 4. Depending on the configuration of you system, you may have to set
240 environment variables in the script template.
237 environment variables in the script template.
241
238
242 The controller template should be similar, but simpler:
239 The controller template should be similar, but simpler:
@@ -247,12 +244,12 b' The controller template should be similar, but simpler:'
247 #PBS -j oe
244 #PBS -j oe
248 #PBS -l walltime=00:10:00
245 #PBS -l walltime=00:10:00
249 #PBS -l nodes=1:ppn=4
246 #PBS -l nodes=1:ppn=4
250 #PBS -q $queue
247 #PBS -q {queue}
251
248
252 cd $$PBS_O_WORKDIR
249 cd $PBS_O_WORKDIR
253 export PATH=$$HOME/usr/local/bin
250 export PATH=$HOME/usr/local/bin
254 export PYTHONPATH=$$HOME/usr/local/lib/python2.7/site-packages
251 export PYTHONPATH=$HOME/usr/local/lib/python2.7/site-packages
255 ipcontroller cluster_dir=${cluster_dir}
252 ipcontroller profile_dir={profile_dir}
256
253
257
254
258 Once you have created these scripts, save them with names like
255 Once you have created these scripts, save them with names like
@@ -268,14 +265,14 b' Once you have created these scripts, save them with names like'
268 Alternately, you can just define the templates as strings inside :file:`ipcluster_config`.
265 Alternately, you can just define the templates as strings inside :file:`ipcluster_config`.
269
266
270 Whether you are using your own templates or our defaults, the extra configurables available are
267 Whether you are using your own templates or our defaults, the extra configurables available are
271 the number of engines to launch (``$n``, and the batch system queue to which the jobs are to be
268 the number of engines to launch (``{n}``, and the batch system queue to which the jobs are to be
272 submitted (``$queue``)). These are configurables, and can be specified in
269 submitted (``{queue}``)). These are configurables, and can be specified in
273 :file:`ipcluster_config`:
270 :file:`ipcluster_config`:
274
271
275 .. sourcecode:: python
272 .. sourcecode:: python
276
273
277 c.PBSLauncher.queue = 'veryshort.q'
274 c.PBSLauncher.queue = 'veryshort.q'
278 c.PBSEngineSetLauncher.n = 64
275 c.IPClusterEnginesApp.n = 64
279
276
280 Note that assuming you are running PBS on a multi-node cluster, the Controller's default behavior
277 Note that assuming you are running PBS on a multi-node cluster, the Controller's default behavior
281 of listening only on localhost is likely too restrictive. In this case, also assuming the
278 of listening only on localhost is likely too restrictive. In this case, also assuming the
@@ -335,8 +332,8 b" The controller's remote location and configuration can be specified:"
335 # Set the arguments to be passed to ipcontroller
332 # Set the arguments to be passed to ipcontroller
336 # note that remotely launched ipcontroller will not get the contents of
333 # note that remotely launched ipcontroller will not get the contents of
337 # the local ipcontroller_config.py unless it resides on the *remote host*
334 # the local ipcontroller_config.py unless it resides on the *remote host*
338 # in the location specified by the `cluster_dir` argument.
335 # in the location specified by the `profile_dir` argument.
339 # c.SSHControllerLauncher.program_args = ['-r', '-ip', '0.0.0.0', '--cluster_dir', '/path/to/cd']
336 # c.SSHControllerLauncher.program_args = ['--reuse', 'ip=0.0.0.0', 'profile_dir=/path/to/cd']
340
337
341 .. note::
338 .. note::
342
339
@@ -352,7 +349,7 b' on that host.'
352
349
353 c.SSHEngineSetLauncher.engines = { 'host1.example.com' : 2,
350 c.SSHEngineSetLauncher.engines = { 'host1.example.com' : 2,
354 'host2.example.com' : 5,
351 'host2.example.com' : 5,
355 'host3.example.com' : (1, ['cluster_dir=/home/different/location']),
352 'host3.example.com' : (1, ['profile_dir=/home/different/location']),
356 'host4.example.com' : 8 }
353 'host4.example.com' : 8 }
357
354
358 * The `engines` dict, where the keys are the host we want to run engines on and
355 * The `engines` dict, where the keys are the host we want to run engines on and
@@ -365,7 +362,7 b' a single location:'
365
362
366 .. sourcecode:: python
363 .. sourcecode:: python
367
364
368 c.SSHEngineSetLauncher.engine_args = ['--cluster_dir', '/path/to/cluster_ssh']
365 c.SSHEngineSetLauncher.engine_args = ['profile_dir=/path/to/cluster_ssh']
369
366
370 Current limitations of the SSH mode of :command:`ipcluster` are:
367 Current limitations of the SSH mode of :command:`ipcluster` are:
371
368
General Comments 0
You need to be logged in to leave comments. Login now