Show More
@@ -121,12 +121,12 b' c = get_config()' | |||||
121 | # in the location specified by the --cluster_dir argument. |
|
121 | # in the location specified by the --cluster_dir argument. | |
122 | # c.SSHControllerLauncher.program_args = ['-r', '-ip', '0.0.0.0', '--cluster_dir', '/path/to/cd'] |
|
122 | # c.SSHControllerLauncher.program_args = ['-r', '-ip', '0.0.0.0', '--cluster_dir', '/path/to/cd'] | |
123 |
|
123 | |||
124 |
# Set the default args passed to ipengine |
|
124 | # Set the default args passed to ipengine for SSH launched engines | |
125 | # c.SSHEngineSetLauncher.engine_args = ['--mpi', 'mpi4py'] |
|
125 | # c.SSHEngineSetLauncher.engine_args = ['--mpi', 'mpi4py'] | |
126 |
|
126 | |||
127 | # SSH engines are launched as a dict of locations/n-engines. |
|
127 | # SSH engines are launched as a dict of locations/n-engines. | |
128 | # if a value is a tuple instead of an int, it is assumed to be of the form |
|
128 | # if a value is a tuple instead of an int, it is assumed to be of the form | |
129 |
# (n, [args]), setting the arguments to passed to ipengine |
|
129 | # (n, [args]), setting the arguments to passed to ipengine on `host`. | |
130 | # otherwise, c.SSHEngineSetLauncher.engine_args will be used as the default. |
|
130 | # otherwise, c.SSHEngineSetLauncher.engine_args will be used as the default. | |
131 |
|
131 | |||
132 | # In this case, there will be 3 engines at my.example.com, and |
|
132 | # In this case, there will be 3 engines at my.example.com, and | |
@@ -162,13 +162,13 b' c = get_config()' | |||||
162 |
|
162 | |||
163 | # The batch submission script used to start the controller. This is where |
|
163 | # The batch submission script used to start the controller. This is where | |
164 | # environment variables would be setup, etc. This string is interpreted using |
|
164 | # environment variables would be setup, etc. This string is interpreted using | |
165 |
# |
|
165 | # Python's string formatting. Basically, you can use {queue} for the name | |
166 |
# |
|
166 | # of the PBS queue, and {profile_dir} for the profile_dir. | |
167 | # c.PBSControllerLauncher.batch_template = """ |
|
167 | # c.PBSControllerLauncher.batch_template = """ | |
168 | # #PBS -N ipcontroller |
|
168 | # #PBS -N ipcontroller | |
169 |
# #PBS -q |
|
169 | # #PBS -q {queue} | |
170 | # |
|
170 | # | |
171 | # ipcontrollerz --cluster-dir $cluster_dir |
|
171 | # ipcontroller profile_dir={profile_dir} | |
172 | # """ |
|
172 | # """ | |
173 |
|
173 | |||
174 | # You can also load this template from a file |
|
174 | # You can also load this template from a file | |
@@ -180,13 +180,14 b' c = get_config()' | |||||
180 |
|
180 | |||
181 | # The batch submission script used to start the engines. This is where |
|
181 | # The batch submission script used to start the engines. This is where | |
182 | # environment variables would be setup, etc. This string is interpreted using |
|
182 | # environment variables would be setup, etc. This string is interpreted using | |
183 |
# |
|
183 | # Python's string formatting. Basically, you can use {queue} for the name | |
184 | # number of engine and ${cluster_dir} for the cluster_dir. |
|
184 | # of the PBS queue, and {profile_dir} for the profile_dir, and {n} | |
|
185 | # for the number of engines. | |||
185 | # c.PBSEngineSetLauncher.batch_template = """ |
|
186 | # c.PBSEngineSetLauncher.batch_template = """ | |
186 |
# #PBS -N ip |
|
187 | # #PBS -N ipengine | |
187 |
# #PBS -l nprocs= |
|
188 | # #PBS -l nprocs={n} | |
188 | # |
|
189 | # | |
189 | # ipenginez --cluster-dir $cluster_dir$s |
|
190 | # ipengine profile_dir={profile_dir} | |
190 | # """ |
|
191 | # """ | |
191 |
|
192 | |||
192 | # You can also load this template from a file |
|
193 | # You can also load this template from a file | |
@@ -211,7 +212,7 b' c = get_config()' | |||||
211 |
|
212 | |||
212 | # c.IPControllerTask.task_name = 'IPController' |
|
213 | # c.IPControllerTask.task_name = 'IPController' | |
213 | # c.IPControllerTask.controller_cmd = [u'ipcontroller.exe'] |
|
214 | # c.IPControllerTask.controller_cmd = [u'ipcontroller.exe'] | |
214 |
# c.IPControllerTask.controller_args = ['--log-to-file', ' |
|
215 | # c.IPControllerTask.controller_args = ['--log-to-file', 'log_level=40'] | |
215 | # c.IPControllerTask.environment_variables = {} |
|
216 | # c.IPControllerTask.environment_variables = {} | |
216 |
|
217 | |||
217 | # c.WindowsHPCControllerLauncher.scheduler = 'HEADNODE' |
|
218 | # c.WindowsHPCControllerLauncher.scheduler = 'HEADNODE' | |
@@ -227,7 +228,7 b' c = get_config()' | |||||
227 |
|
228 | |||
228 | # c.IPEngineTask.task_name = 'IPEngine' |
|
229 | # c.IPEngineTask.task_name = 'IPEngine' | |
229 | # c.IPEngineTask.engine_cmd = [u'ipengine.exe'] |
|
230 | # c.IPEngineTask.engine_cmd = [u'ipengine.exe'] | |
230 |
# c.IPEngineTask.engine_args = ['--log-to-file', ' |
|
231 | # c.IPEngineTask.engine_args = ['--log-to-file', 'log_level=40'] | |
231 | # c.IPEngineTask.environment_variables = {} |
|
232 | # c.IPEngineTask.environment_variables = {} | |
232 |
|
233 | |||
233 | # c.WindowsHPCEngineSetLauncher.scheduler = 'HEADNODE' |
|
234 | # c.WindowsHPCEngineSetLauncher.scheduler = 'HEADNODE' |
@@ -49,7 +49,6 b' except ImportError:' | |||||
49 |
|
49 | |||
50 | from zmq.eventloop import ioloop |
|
50 | from zmq.eventloop import ioloop | |
51 |
|
51 | |||
52 | from IPython.external import Itpl |
|
|||
53 | # from IPython.config.configurable import Configurable |
|
52 | # from IPython.config.configurable import Configurable | |
54 | from IPython.utils.traitlets import Any, Int, List, Unicode, Dict, Instance |
|
53 | from IPython.utils.traitlets import Any, Int, List, Unicode, Dict, Instance | |
55 | from IPython.utils.path import get_ipython_module_path |
|
54 | from IPython.utils.path import get_ipython_module_path | |
@@ -805,8 +804,8 b' class BatchSystemLauncher(BaseLauncher):' | |||||
805 |
|
804 | |||
806 | This class also has the notion of a batch script. The ``batch_template`` |
|
805 | This class also has the notion of a batch script. The ``batch_template`` | |
807 | attribute can be set to a string that is a template for the batch script. |
|
806 | attribute can be set to a string that is a template for the batch script. | |
808 |
This template is instantiated using |
|
807 | This template is instantiated using string formatting. Thus the template can | |
809 |
|
|
808 | use {n} fot the number of instances. Subclasses can add additional variables | |
810 | to the template dict. |
|
809 | to the template dict. | |
811 | """ |
|
810 | """ | |
812 |
|
811 | |||
@@ -866,7 +865,6 b' class BatchSystemLauncher(BaseLauncher):' | |||||
866 | """Instantiate and write the batch script to the work_dir.""" |
|
865 | """Instantiate and write the batch script to the work_dir.""" | |
867 | self.context['n'] = n |
|
866 | self.context['n'] = n | |
868 | self.context['queue'] = self.queue |
|
867 | self.context['queue'] = self.queue | |
869 | print self.context |
|
|||
870 | # first priority is batch_template if set |
|
868 | # first priority is batch_template if set | |
871 | if self.batch_template_file and not self.batch_template: |
|
869 | if self.batch_template_file and not self.batch_template: | |
872 | # second priority is batch_template_file |
|
870 | # second priority is batch_template_file | |
@@ -890,7 +888,7 b' class BatchSystemLauncher(BaseLauncher):' | |||||
890 | firstline, rest = self.batch_template.split('\n',1) |
|
888 | firstline, rest = self.batch_template.split('\n',1) | |
891 | self.batch_template = u'\n'.join([firstline, self.queue_template, rest]) |
|
889 | self.batch_template = u'\n'.join([firstline, self.queue_template, rest]) | |
892 |
|
890 | |||
893 |
script_as_string = |
|
891 | script_as_string = self.batch_template.format(**self.context) | |
894 | self.log.info('Writing instantiated batch script: %s' % self.batch_file) |
|
892 | self.log.info('Writing instantiated batch script: %s' % self.batch_file) | |
895 |
|
893 | |||
896 | with open(self.batch_file, 'w') as f: |
|
894 | with open(self.batch_file, 'w') as f: | |
@@ -899,9 +897,8 b' class BatchSystemLauncher(BaseLauncher):' | |||||
899 |
|
897 | |||
900 | def start(self, n, profile_dir): |
|
898 | def start(self, n, profile_dir): | |
901 | """Start n copies of the process using a batch system.""" |
|
899 | """Start n copies of the process using a batch system.""" | |
902 |
# Here we save profile |
|
900 | # Here we save profile_dir in the context so they | |
903 |
# can be used in the batch script template as |
|
901 | # can be used in the batch script template as {profile_dir} | |
904 | # ${profile_dir} |
|
|||
905 | self.context['profile_dir'] = profile_dir |
|
902 | self.context['profile_dir'] = profile_dir | |
906 | self.profile_dir = unicode(profile_dir) |
|
903 | self.profile_dir = unicode(profile_dir) | |
907 | self.write_batch_script(n) |
|
904 | self.write_batch_script(n) | |
@@ -929,9 +926,9 b' class PBSLauncher(BatchSystemLauncher):' | |||||
929 |
|
926 | |||
930 | batch_file = Unicode(u'') |
|
927 | batch_file = Unicode(u'') | |
931 | job_array_regexp = Unicode('#PBS\W+-t\W+[\w\d\-\$]+') |
|
928 | job_array_regexp = Unicode('#PBS\W+-t\W+[\w\d\-\$]+') | |
932 |
job_array_template = Unicode('#PBS -t 1- |
|
929 | job_array_template = Unicode('#PBS -t 1-{n}') | |
933 | queue_regexp = Unicode('#PBS\W+-q\W+\$?\w+') |
|
930 | queue_regexp = Unicode('#PBS\W+-q\W+\$?\w+') | |
934 |
queue_template = Unicode('#PBS -q |
|
931 | queue_template = Unicode('#PBS -q {queue}') | |
935 |
|
932 | |||
936 |
|
933 | |||
937 | class PBSControllerLauncher(PBSLauncher): |
|
934 | class PBSControllerLauncher(PBSLauncher): | |
@@ -942,7 +939,7 b' class PBSControllerLauncher(PBSLauncher):' | |||||
942 | default_template= Unicode("""#!/bin/sh |
|
939 | default_template= Unicode("""#!/bin/sh | |
943 | #PBS -V |
|
940 | #PBS -V | |
944 | #PBS -N ipcontroller |
|
941 | #PBS -N ipcontroller | |
945 |
%s --log-to-file profile_dir |
|
942 | %s --log-to-file profile_dir={profile_dir} | |
946 | """%(' '.join(ipcontroller_cmd_argv))) |
|
943 | """%(' '.join(ipcontroller_cmd_argv))) | |
947 |
|
944 | |||
948 | def start(self, profile_dir): |
|
945 | def start(self, profile_dir): | |
@@ -958,7 +955,7 b' class PBSEngineSetLauncher(PBSLauncher):' | |||||
958 | default_template= Unicode(u"""#!/bin/sh |
|
955 | default_template= Unicode(u"""#!/bin/sh | |
959 | #PBS -V |
|
956 | #PBS -V | |
960 | #PBS -N ipengine |
|
957 | #PBS -N ipengine | |
961 |
%s profile_dir |
|
958 | %s profile_dir={profile_dir} | |
962 | """%(' '.join(ipengine_cmd_argv))) |
|
959 | """%(' '.join(ipengine_cmd_argv))) | |
963 |
|
960 | |||
964 | def start(self, n, profile_dir): |
|
961 | def start(self, n, profile_dir): | |
@@ -970,20 +967,20 b' class PBSEngineSetLauncher(PBSLauncher):' | |||||
970 |
|
967 | |||
971 | class SGELauncher(PBSLauncher): |
|
968 | class SGELauncher(PBSLauncher): | |
972 | """Sun GridEngine is a PBS clone with slightly different syntax""" |
|
969 | """Sun GridEngine is a PBS clone with slightly different syntax""" | |
973 |
job_array_regexp = Unicode('#\$\ |
|
970 | job_array_regexp = Unicode('#\$\W+\-t') | |
974 |
job_array_template = Unicode('#$ |
|
971 | job_array_template = Unicode('#$ -t 1-{n}') | |
975 |
queue_regexp = Unicode('# |
|
972 | queue_regexp = Unicode('#\$\W+-q\W+\$?\w+') | |
976 |
queue_template = Unicode('#$ |
|
973 | queue_template = Unicode('#$ -q $queue') | |
977 |
|
974 | |||
978 | class SGEControllerLauncher(SGELauncher): |
|
975 | class SGEControllerLauncher(SGELauncher): | |
979 | """Launch a controller using SGE.""" |
|
976 | """Launch a controller using SGE.""" | |
980 |
|
977 | |||
981 | batch_file_name = Unicode(u'sge_controller', config=True, |
|
978 | batch_file_name = Unicode(u'sge_controller', config=True, | |
982 | help="batch file name for the ipontroller job.") |
|
979 | help="batch file name for the ipontroller job.") | |
983 |
default_template= Unicode(u"""#$ |
|
980 | default_template= Unicode(u"""#$ -V | |
984 |
#$ |
|
981 | #$ -S /bin/sh | |
985 |
#$ |
|
982 | #$ -N ipcontroller | |
986 |
%s --log-to-file profile_dir= |
|
983 | %s --log-to-file profile_dir={profile_dir} | |
987 | """%(' '.join(ipcontroller_cmd_argv))) |
|
984 | """%(' '.join(ipcontroller_cmd_argv))) | |
988 |
|
985 | |||
989 | def start(self, profile_dir): |
|
986 | def start(self, profile_dir): | |
@@ -995,10 +992,10 b' class SGEEngineSetLauncher(SGELauncher):' | |||||
995 | """Launch Engines with SGE""" |
|
992 | """Launch Engines with SGE""" | |
996 | batch_file_name = Unicode(u'sge_engines', config=True, |
|
993 | batch_file_name = Unicode(u'sge_engines', config=True, | |
997 | help="batch file name for the engine(s) job.") |
|
994 | help="batch file name for the engine(s) job.") | |
998 |
default_template = Unicode("""#$ |
|
995 | default_template = Unicode("""#$ -V | |
999 |
#$ |
|
996 | #$ -S /bin/sh | |
1000 |
#$ |
|
997 | #$ -N ipengine | |
1001 |
%s profile_dir= |
|
998 | %s profile_dir={profile_dir} | |
1002 | """%(' '.join(ipengine_cmd_argv))) |
|
999 | """%(' '.join(ipengine_cmd_argv))) | |
1003 |
|
1000 | |||
1004 | def start(self, n, profile_dir): |
|
1001 | def start(self, n, profile_dir): |
General Comments 0
You need to be logged in to leave comments.
Login now