##// END OF EJS Templates
Replace Itpl with str.format in parallel launcher....
Thomas Kluyver -
Show More
@@ -121,12 +121,12 b' c = get_config()'
121 # in the location specified by the --cluster_dir argument.
121 # in the location specified by the --cluster_dir argument.
122 # c.SSHControllerLauncher.program_args = ['-r', '-ip', '0.0.0.0', '--cluster_dir', '/path/to/cd']
122 # c.SSHControllerLauncher.program_args = ['-r', '-ip', '0.0.0.0', '--cluster_dir', '/path/to/cd']
123
123
124 # Set the default args passed to ipenginez for SSH launched engines
124 # Set the default args passed to ipengine for SSH launched engines
125 # c.SSHEngineSetLauncher.engine_args = ['--mpi', 'mpi4py']
125 # c.SSHEngineSetLauncher.engine_args = ['--mpi', 'mpi4py']
126
126
127 # SSH engines are launched as a dict of locations/n-engines.
127 # SSH engines are launched as a dict of locations/n-engines.
128 # if a value is a tuple instead of an int, it is assumed to be of the form
128 # if a value is a tuple instead of an int, it is assumed to be of the form
129 # (n, [args]), setting the arguments to passed to ipenginez on `host`.
129 # (n, [args]), setting the arguments to passed to ipengine on `host`.
130 # otherwise, c.SSHEngineSetLauncher.engine_args will be used as the default.
130 # otherwise, c.SSHEngineSetLauncher.engine_args will be used as the default.
131
131
132 # In this case, there will be 3 engines at my.example.com, and
132 # In this case, there will be 3 engines at my.example.com, and
@@ -162,13 +162,13 b' c = get_config()'
162
162
163 # The batch submission script used to start the controller. This is where
163 # The batch submission script used to start the controller. This is where
164 # environment variables would be setup, etc. This string is interpreted using
164 # environment variables would be setup, etc. This string is interpreted using
165 # the Itpl module in IPython.external. Basically, you can use ${n} for the
165 # Python's string formatting. Basically, you can use {queue} for the name
166 # number of engine and ${cluster_dir} for the cluster_dir.
166 # of the PBS queue, and {profile_dir} for the profile_dir.
167 # c.PBSControllerLauncher.batch_template = """
167 # c.PBSControllerLauncher.batch_template = """
168 # #PBS -N ipcontroller
168 # #PBS -N ipcontroller
169 # #PBS -q $queue
169 # #PBS -q {queue}
170 #
170 #
171 # ipcontrollerz --cluster-dir $cluster_dir
171 # ipcontroller profile_dir={profile_dir}
172 # """
172 # """
173
173
174 # You can also load this template from a file
174 # You can also load this template from a file
@@ -180,13 +180,14 b' c = get_config()'
180
180
181 # The batch submission script used to start the engines. This is where
181 # The batch submission script used to start the engines. This is where
182 # environment variables would be setup, etc. This string is interpreted using
182 # environment variables would be setup, etc. This string is interpreted using
183 # the Itpl module in IPython.external. Basically, you can use ${n} for the
183 # Python's string formatting. Basically, you can use {queue} for the name
184 # number of engine and ${cluster_dir} for the cluster_dir.
184 # of the PBS queue, and {profile_dir} for the profile_dir, and {n}
185 # for the number of engines.
185 # c.PBSEngineSetLauncher.batch_template = """
186 # c.PBSEngineSetLauncher.batch_template = """
186 # #PBS -N ipcontroller
187 # #PBS -N ipengine
187 # #PBS -l nprocs=$n
188 # #PBS -l nprocs={n}
188 #
189 #
189 # ipenginez --cluster-dir $cluster_dir$s
190 # ipengine profile_dir={profile_dir}
190 # """
191 # """
191
192
192 # You can also load this template from a file
193 # You can also load this template from a file
@@ -211,7 +212,7 b' c = get_config()'
211
212
212 # c.IPControllerTask.task_name = 'IPController'
213 # c.IPControllerTask.task_name = 'IPController'
213 # c.IPControllerTask.controller_cmd = [u'ipcontroller.exe']
214 # c.IPControllerTask.controller_cmd = [u'ipcontroller.exe']
214 # c.IPControllerTask.controller_args = ['--log-to-file', '--log-level', '40']
215 # c.IPControllerTask.controller_args = ['--log-to-file', 'log_level=40']
215 # c.IPControllerTask.environment_variables = {}
216 # c.IPControllerTask.environment_variables = {}
216
217
217 # c.WindowsHPCControllerLauncher.scheduler = 'HEADNODE'
218 # c.WindowsHPCControllerLauncher.scheduler = 'HEADNODE'
@@ -227,7 +228,7 b' c = get_config()'
227
228
228 # c.IPEngineTask.task_name = 'IPEngine'
229 # c.IPEngineTask.task_name = 'IPEngine'
229 # c.IPEngineTask.engine_cmd = [u'ipengine.exe']
230 # c.IPEngineTask.engine_cmd = [u'ipengine.exe']
230 # c.IPEngineTask.engine_args = ['--log-to-file', '--log-level', '40']
231 # c.IPEngineTask.engine_args = ['--log-to-file', 'log_level=40']
231 # c.IPEngineTask.environment_variables = {}
232 # c.IPEngineTask.environment_variables = {}
232
233
233 # c.WindowsHPCEngineSetLauncher.scheduler = 'HEADNODE'
234 # c.WindowsHPCEngineSetLauncher.scheduler = 'HEADNODE'
@@ -49,7 +49,6 b' except ImportError:'
49
49
50 from zmq.eventloop import ioloop
50 from zmq.eventloop import ioloop
51
51
52 from IPython.external import Itpl
53 # from IPython.config.configurable import Configurable
52 # from IPython.config.configurable import Configurable
54 from IPython.utils.traitlets import Any, Int, List, Unicode, Dict, Instance
53 from IPython.utils.traitlets import Any, Int, List, Unicode, Dict, Instance
55 from IPython.utils.path import get_ipython_module_path
54 from IPython.utils.path import get_ipython_module_path
@@ -805,8 +804,8 b' class BatchSystemLauncher(BaseLauncher):'
805
804
806 This class also has the notion of a batch script. The ``batch_template``
805 This class also has the notion of a batch script. The ``batch_template``
807 attribute can be set to a string that is a template for the batch script.
806 attribute can be set to a string that is a template for the batch script.
808 This template is instantiated using Itpl. Thus the template can use
807 This template is instantiated using string formatting. Thus the template can
809 ${n} fot the number of instances. Subclasses can add additional variables
808 use {n} fot the number of instances. Subclasses can add additional variables
810 to the template dict.
809 to the template dict.
811 """
810 """
812
811
@@ -866,7 +865,6 b' class BatchSystemLauncher(BaseLauncher):'
866 """Instantiate and write the batch script to the work_dir."""
865 """Instantiate and write the batch script to the work_dir."""
867 self.context['n'] = n
866 self.context['n'] = n
868 self.context['queue'] = self.queue
867 self.context['queue'] = self.queue
869 print self.context
870 # first priority is batch_template if set
868 # first priority is batch_template if set
871 if self.batch_template_file and not self.batch_template:
869 if self.batch_template_file and not self.batch_template:
872 # second priority is batch_template_file
870 # second priority is batch_template_file
@@ -890,7 +888,7 b' class BatchSystemLauncher(BaseLauncher):'
890 firstline, rest = self.batch_template.split('\n',1)
888 firstline, rest = self.batch_template.split('\n',1)
891 self.batch_template = u'\n'.join([firstline, self.queue_template, rest])
889 self.batch_template = u'\n'.join([firstline, self.queue_template, rest])
892
890
893 script_as_string = Itpl.itplns(self.batch_template, self.context)
891 script_as_string = self.batch_template.format(**self.context)
894 self.log.info('Writing instantiated batch script: %s' % self.batch_file)
892 self.log.info('Writing instantiated batch script: %s' % self.batch_file)
895
893
896 with open(self.batch_file, 'w') as f:
894 with open(self.batch_file, 'w') as f:
@@ -899,9 +897,8 b' class BatchSystemLauncher(BaseLauncher):'
899
897
900 def start(self, n, profile_dir):
898 def start(self, n, profile_dir):
901 """Start n copies of the process using a batch system."""
899 """Start n copies of the process using a batch system."""
902 # Here we save profile and profile_dir in the context so they
900 # Here we save profile_dir in the context so they
903 # can be used in the batch script template as ${profile} and
901 # can be used in the batch script template as {profile_dir}
904 # ${profile_dir}
905 self.context['profile_dir'] = profile_dir
902 self.context['profile_dir'] = profile_dir
906 self.profile_dir = unicode(profile_dir)
903 self.profile_dir = unicode(profile_dir)
907 self.write_batch_script(n)
904 self.write_batch_script(n)
@@ -929,9 +926,9 b' class PBSLauncher(BatchSystemLauncher):'
929
926
930 batch_file = Unicode(u'')
927 batch_file = Unicode(u'')
931 job_array_regexp = Unicode('#PBS\W+-t\W+[\w\d\-\$]+')
928 job_array_regexp = Unicode('#PBS\W+-t\W+[\w\d\-\$]+')
932 job_array_template = Unicode('#PBS -t 1-$n')
929 job_array_template = Unicode('#PBS -t 1-{n}')
933 queue_regexp = Unicode('#PBS\W+-q\W+\$?\w+')
930 queue_regexp = Unicode('#PBS\W+-q\W+\$?\w+')
934 queue_template = Unicode('#PBS -q $queue')
931 queue_template = Unicode('#PBS -q {queue}')
935
932
936
933
937 class PBSControllerLauncher(PBSLauncher):
934 class PBSControllerLauncher(PBSLauncher):
@@ -942,7 +939,7 b' class PBSControllerLauncher(PBSLauncher):'
942 default_template= Unicode("""#!/bin/sh
939 default_template= Unicode("""#!/bin/sh
943 #PBS -V
940 #PBS -V
944 #PBS -N ipcontroller
941 #PBS -N ipcontroller
945 %s --log-to-file profile_dir $profile_dir
942 %s --log-to-file profile_dir={profile_dir}
946 """%(' '.join(ipcontroller_cmd_argv)))
943 """%(' '.join(ipcontroller_cmd_argv)))
947
944
948 def start(self, profile_dir):
945 def start(self, profile_dir):
@@ -958,7 +955,7 b' class PBSEngineSetLauncher(PBSLauncher):'
958 default_template= Unicode(u"""#!/bin/sh
955 default_template= Unicode(u"""#!/bin/sh
959 #PBS -V
956 #PBS -V
960 #PBS -N ipengine
957 #PBS -N ipengine
961 %s profile_dir $profile_dir
958 %s profile_dir={profile_dir}
962 """%(' '.join(ipengine_cmd_argv)))
959 """%(' '.join(ipengine_cmd_argv)))
963
960
964 def start(self, n, profile_dir):
961 def start(self, n, profile_dir):
@@ -970,20 +967,20 b' class PBSEngineSetLauncher(PBSLauncher):'
970
967
971 class SGELauncher(PBSLauncher):
968 class SGELauncher(PBSLauncher):
972 """Sun GridEngine is a PBS clone with slightly different syntax"""
969 """Sun GridEngine is a PBS clone with slightly different syntax"""
973 job_array_regexp = Unicode('#\$\$\W+\-t')
970 job_array_regexp = Unicode('#\$\W+\-t')
974 job_array_template = Unicode('#$$ -t 1-$n')
971 job_array_template = Unicode('#$ -t 1-{n}')
975 queue_regexp = Unicode('#$$\W+-q\W+\$?\w+')
972 queue_regexp = Unicode('#\$\W+-q\W+\$?\w+')
976 queue_template = Unicode('#$$ -q $queue')
973 queue_template = Unicode('#$ -q $queue')
977
974
978 class SGEControllerLauncher(SGELauncher):
975 class SGEControllerLauncher(SGELauncher):
979 """Launch a controller using SGE."""
976 """Launch a controller using SGE."""
980
977
981 batch_file_name = Unicode(u'sge_controller', config=True,
978 batch_file_name = Unicode(u'sge_controller', config=True,
982 help="batch file name for the ipontroller job.")
979 help="batch file name for the ipontroller job.")
983 default_template= Unicode(u"""#$$ -V
980 default_template= Unicode(u"""#$ -V
984 #$$ -S /bin/sh
981 #$ -S /bin/sh
985 #$$ -N ipcontroller
982 #$ -N ipcontroller
986 %s --log-to-file profile_dir=$profile_dir
983 %s --log-to-file profile_dir={profile_dir}
987 """%(' '.join(ipcontroller_cmd_argv)))
984 """%(' '.join(ipcontroller_cmd_argv)))
988
985
989 def start(self, profile_dir):
986 def start(self, profile_dir):
@@ -995,10 +992,10 b' class SGEEngineSetLauncher(SGELauncher):'
995 """Launch Engines with SGE"""
992 """Launch Engines with SGE"""
996 batch_file_name = Unicode(u'sge_engines', config=True,
993 batch_file_name = Unicode(u'sge_engines', config=True,
997 help="batch file name for the engine(s) job.")
994 help="batch file name for the engine(s) job.")
998 default_template = Unicode("""#$$ -V
995 default_template = Unicode("""#$ -V
999 #$$ -S /bin/sh
996 #$ -S /bin/sh
1000 #$$ -N ipengine
997 #$ -N ipengine
1001 %s profile_dir=$profile_dir
998 %s profile_dir={profile_dir}
1002 """%(' '.join(ipengine_cmd_argv)))
999 """%(' '.join(ipengine_cmd_argv)))
1003
1000
1004 def start(self, n, profile_dir):
1001 def start(self, n, profile_dir):
@@ -1067,4 +1064,4 b' sge_launchers = ['
1067 SGEEngineSetLauncher,
1064 SGEEngineSetLauncher,
1068 ]
1065 ]
1069 all_launchers = local_launchers + mpi_launchers + ssh_launchers + winhpc_launchers\
1066 all_launchers = local_launchers + mpi_launchers + ssh_launchers + winhpc_launchers\
1070 + pbs_launchers + sge_launchers No newline at end of file
1067 + pbs_launchers + sge_launchers
General Comments 0
You need to be logged in to leave comments. Login now