Show More
@@ -60,7 +60,7 b' _description = """Start an IPython cluster for parallel computing.' | |||||
60 |
|
60 | |||
61 | An IPython cluster consists of 1 controller and 1 or more engines. |
|
61 | An IPython cluster consists of 1 controller and 1 or more engines. | |
62 | This command automates the startup of these processes using a wide range of |
|
62 | This command automates the startup of these processes using a wide range of | |
63 | startup methods (SSH, local processes, PBS, mpiexec, SGE, LSF, Condor, |
|
63 | startup methods (SSH, local processes, PBS, mpiexec, SGE, LSF, HTCondor, | |
64 | Windows HPC Server 2008). To start a cluster with 4 engines on your |
|
64 | Windows HPC Server 2008). To start a cluster with 4 engines on your | |
65 | local host simply do 'ipcluster start --n=4'. For more complex usage |
|
65 | local host simply do 'ipcluster start --n=4'. For more complex usage | |
66 | you will typically do 'ipython profile create mycluster --parallel', then edit |
|
66 | you will typically do 'ipython profile create mycluster --parallel', then edit | |
@@ -116,7 +116,7 b' def find_launcher_class(clsname, kind):' | |||||
116 | ========== |
|
116 | ========== | |
117 | clsname : str |
|
117 | clsname : str | |
118 | The full name of the launcher class, either with or without the |
|
118 | The full name of the launcher class, either with or without the | |
119 | module path, or an abbreviation (MPI, SSH, SGE, PBS, LSF, Condor |
|
119 | module path, or an abbreviation (MPI, SSH, SGE, PBS, LSF, HTCondor | |
120 | WindowsHPC). |
|
120 | WindowsHPC). | |
121 | kind : str |
|
121 | kind : str | |
122 | Either 'EngineSet' or 'Controller'. |
|
122 | Either 'EngineSet' or 'Controller'. | |
@@ -287,7 +287,7 b' class IPClusterEngines(BaseParallelApplication):' | |||||
287 | Note that SSH does *not* move the connection files |
|
287 | Note that SSH does *not* move the connection files | |
288 | around, so you will likely have to do this manually |
|
288 | around, so you will likely have to do this manually | |
289 | unless the machines are on a shared file system. |
|
289 | unless the machines are on a shared file system. | |
290 | Condor : use HTCondor to submit engines to a batch queue |
|
290 | HTCondor : use HTCondor to submit engines to a batch queue | |
291 | WindowsHPC : use Windows HPC |
|
291 | WindowsHPC : use Windows HPC | |
292 |
|
292 | |||
293 | If you are using one of IPython's builtin launchers, you can specify just the |
|
293 | If you are using one of IPython's builtin launchers, you can specify just the | |
@@ -489,7 +489,7 b' class IPClusterStart(IPClusterEngines):' | |||||
489 | PBS : use PBS (qsub) to submit the controller to a batch queue |
|
489 | PBS : use PBS (qsub) to submit the controller to a batch queue | |
490 | SGE : use SGE (qsub) to submit the controller to a batch queue |
|
490 | SGE : use SGE (qsub) to submit the controller to a batch queue | |
491 | LSF : use LSF (bsub) to submit the controller to a batch queue |
|
491 | LSF : use LSF (bsub) to submit the controller to a batch queue | |
492 | Condor: use HTCondor to submit the controller to a batch queue |
|
492 | HTCondor : use HTCondor to submit the controller to a batch queue | |
493 | SSH : use SSH to start the controller |
|
493 | SSH : use SSH to start the controller | |
494 | WindowsHPC : use Windows HPC |
|
494 | WindowsHPC : use Windows HPC | |
495 |
|
495 |
@@ -1285,10 +1285,10 b' class LSFEngineSetLauncher(LSFLauncher, BatchClusterAppMixin):' | |||||
1285 |
|
1285 | |||
1286 |
|
1286 | |||
1287 |
|
1287 | |||
1288 | class CondorLauncher(BatchSystemLauncher): |
|
1288 | class HTCondorLauncher(BatchSystemLauncher): | |
1289 | """A BatchSystemLauncher subclass for Condor. |
|
1289 | """A BatchSystemLauncher subclass for HTCondor. | |
1290 |
|
1290 | |||
1291 | Condor requires that we launch the ipengine/ipcontroller scripts rather |
|
1291 | HTCondor requires that we launch the ipengine/ipcontroller scripts rather | |
1292 | that the python instance but otherwise is very similar to PBS. This is because |
|
1292 | that the python instance but otherwise is very similar to PBS. This is because | |
1293 | HTCondor destroys sys.executable when launching remote processes - a launched |
|
1293 | HTCondor destroys sys.executable when launching remote processes - a launched | |
1294 | python process depends on sys.executable to effectively evaluate its |
|
1294 | python process depends on sys.executable to effectively evaluate its | |
@@ -1307,9 +1307,9 b' class CondorLauncher(BatchSystemLauncher):' | |||||
1307 | """ |
|
1307 | """ | |
1308 |
|
1308 | |||
1309 | submit_command = List(['condor_submit'], config=True, |
|
1309 | submit_command = List(['condor_submit'], config=True, | |
1310 | help="The Condor submit command ['condor_submit']") |
|
1310 | help="The HTCondor submit command ['condor_submit']") | |
1311 | delete_command = List(['condor_rm'], config=True, |
|
1311 | delete_command = List(['condor_rm'], config=True, | |
1312 | help="The Condor delete command ['condor_rm']") |
|
1312 | help="The HTCondor delete command ['condor_rm']") | |
1313 | job_id_regexp = CRegExp(r'(\d+)\.$', config=True, |
|
1313 | job_id_regexp = CRegExp(r'(\d+)\.$', config=True, | |
1314 | help="Regular expression for identifying the job ID [r'(\d+)\.$']") |
|
1314 | help="Regular expression for identifying the job ID [r'(\d+)\.$']") | |
1315 | job_id_regexp_group = Integer(1, config=True, |
|
1315 | job_id_regexp_group = Integer(1, config=True, | |
@@ -1324,21 +1324,21 b' class CondorLauncher(BatchSystemLauncher):' | |||||
1324 | """ |
|
1324 | """ | |
1325 | if not self.job_array_regexp.search(self.batch_template): |
|
1325 | if not self.job_array_regexp.search(self.batch_template): | |
1326 | self.log.debug("adding job array settings to batch script") |
|
1326 | self.log.debug("adding job array settings to batch script") | |
1327 | #Condor requires that the job array goes at the bottom of the script |
|
1327 | #HTCondor requires that the job array goes at the bottom of the script | |
1328 | self.batch_template = '\n'.join([self.batch_template, |
|
1328 | self.batch_template = '\n'.join([self.batch_template, | |
1329 | self.job_array_template]) |
|
1329 | self.job_array_template]) | |
1330 |
|
1330 | |||
1331 | def _insert_queue_in_script(self): |
|
1331 | def _insert_queue_in_script(self): | |
1332 | """AFAIK, Condor doesn't have a concept of multiple queues that can be |
|
1332 | """AFAIK, HTCondor doesn't have a concept of multiple queues that can be | |
1333 | specified in the script. |
|
1333 | specified in the script. | |
1334 | """ |
|
1334 | """ | |
1335 | pass |
|
1335 | pass | |
1336 |
|
1336 | |||
1337 |
|
1337 | |||
1338 | class CondorControllerLauncher(CondorLauncher, BatchClusterAppMixin): |
|
1338 | class HTCondorControllerLauncher(HTCondorLauncher, BatchClusterAppMixin): | |
1339 | """Launch a controller using Condor.""" |
|
1339 | """Launch a controller using HTCondor.""" | |
1340 |
|
1340 | |||
1341 | batch_file_name = Unicode(u'condor_controller', config=True, |
|
1341 | batch_file_name = Unicode(u'htcondor_controller', config=True, | |
1342 | help="batch file name for the controller job.") |
|
1342 | help="batch file name for the controller job.") | |
1343 | default_template = Unicode(r""" |
|
1343 | default_template = Unicode(r""" | |
1344 | universe = vanilla |
|
1344 | universe = vanilla | |
@@ -1350,12 +1350,12 b" arguments = --log-to-file '--profile-dir={profile_dir}' --cluster-id='{clu" | |||||
1350 |
|
1350 | |||
1351 | def start(self): |
|
1351 | def start(self): | |
1352 | """Start the controller by profile or profile_dir.""" |
|
1352 | """Start the controller by profile or profile_dir.""" | |
1353 | return super(CondorControllerLauncher, self).start(1) |
|
1353 | return super(HTCondorControllerLauncher, self).start(1) | |
1354 |
|
1354 | |||
1355 |
|
1355 | |||
1356 | class CondorEngineSetLauncher(CondorLauncher, BatchClusterAppMixin): |
|
1356 | class HTCondorEngineSetLauncher(HTCondorLauncher, BatchClusterAppMixin): | |
1357 | """Launch Engines using Condor""" |
|
1357 | """Launch Engines using HTCondor""" | |
1358 | batch_file_name = Unicode(u'condor_engines', config=True, |
|
1358 | batch_file_name = Unicode(u'htcondor_engines', config=True, | |
1359 | help="batch file name for the engine(s) job.") |
|
1359 | help="batch file name for the engine(s) job.") | |
1360 | default_template = Unicode(""" |
|
1360 | default_template = Unicode(""" | |
1361 | universe = vanilla |
|
1361 | universe = vanilla | |
@@ -1432,10 +1432,10 b' lsf_launchers = [' | |||||
1432 | LSFControllerLauncher, |
|
1432 | LSFControllerLauncher, | |
1433 | LSFEngineSetLauncher, |
|
1433 | LSFEngineSetLauncher, | |
1434 | ] |
|
1434 | ] | |
1435 | condor_launchers = [ |
|
1435 | htcondor_launchers = [ | |
1436 | CondorLauncher, |
|
1436 | HTCondorLauncher, | |
1437 | CondorControllerLauncher, |
|
1437 | HTCondorControllerLauncher, | |
1438 | CondorEngineSetLauncher, |
|
1438 | HTCondorEngineSetLauncher, | |
1439 | ] |
|
1439 | ] | |
1440 | all_launchers = local_launchers + mpi_launchers + ssh_launchers + winhpc_launchers\ |
|
1440 | all_launchers = local_launchers + mpi_launchers + ssh_launchers + winhpc_launchers\ | |
1441 | + pbs_launchers + sge_launchers + lsf_launchers + condor_launchers |
|
1441 | + pbs_launchers + sge_launchers + lsf_launchers + htcondor_launchers |
@@ -129,8 +129,8 b' class TestSGEControllerLauncher(BatchTest, ControllerLauncherTest, TestCase):' | |||||
129 | class TestLSFControllerLauncher(BatchTest, ControllerLauncherTest, TestCase): |
|
129 | class TestLSFControllerLauncher(BatchTest, ControllerLauncherTest, TestCase): | |
130 | launcher_class = launcher.LSFControllerLauncher |
|
130 | launcher_class = launcher.LSFControllerLauncher | |
131 |
|
131 | |||
132 | class TestCondorControllerLauncher(BatchTest, ControllerLauncherTest, TestCase): |
|
132 | class TestHTCondorControllerLauncher(BatchTest, ControllerLauncherTest, TestCase): | |
133 | launcher_class = launcher.CondorControllerLauncher |
|
133 | launcher_class = launcher.HTCondorControllerLauncher | |
134 |
|
134 | |||
135 | class TestSSHControllerLauncher(SSHTest, ControllerLauncherTest, TestCase): |
|
135 | class TestSSHControllerLauncher(SSHTest, ControllerLauncherTest, TestCase): | |
136 | launcher_class = launcher.SSHControllerLauncher |
|
136 | launcher_class = launcher.SSHControllerLauncher | |
@@ -158,8 +158,8 b' class TestSGEEngineSetLauncher(BatchTest, EngineSetLauncherTest, TestCase):' | |||||
158 | class TestLSFEngineSetLauncher(BatchTest, EngineSetLauncherTest, TestCase): |
|
158 | class TestLSFEngineSetLauncher(BatchTest, EngineSetLauncherTest, TestCase): | |
159 | launcher_class = launcher.LSFEngineSetLauncher |
|
159 | launcher_class = launcher.LSFEngineSetLauncher | |
160 |
|
160 | |||
161 | class TestCondorEngineSetLauncher(BatchTest, EngineSetLauncherTest, TestCase): |
|
161 | class TestHTCondorEngineSetLauncher(BatchTest, EngineSetLauncherTest, TestCase): | |
162 | launcher_class = launcher.CondorEngineSetLauncher |
|
162 | launcher_class = launcher.HTCondorEngineSetLauncher | |
163 |
|
163 | |||
164 | class TestSSHEngineSetLauncher(EngineSetLauncherTest, TestCase): |
|
164 | class TestSSHEngineSetLauncher(EngineSetLauncherTest, TestCase): | |
165 | launcher_class = launcher.SSHEngineSetLauncher |
|
165 | launcher_class = launcher.SSHEngineSetLauncher |
General Comments 0
You need to be logged in to leave comments.
Login now