Show More
@@ -60,7 +60,7 b' _description = """Start an IPython cluster for parallel computing.' | |||
|
60 | 60 | |
|
61 | 61 | An IPython cluster consists of 1 controller and 1 or more engines. |
|
62 | 62 | This command automates the startup of these processes using a wide range of |
|
63 | startup methods (SSH, local processes, PBS, mpiexec, SGE, LSF, Condor, | |
|
63 | startup methods (SSH, local processes, PBS, mpiexec, SGE, LSF, HTCondor, | |
|
64 | 64 | Windows HPC Server 2008). To start a cluster with 4 engines on your |
|
65 | 65 | local host simply do 'ipcluster start --n=4'. For more complex usage |
|
66 | 66 | you will typically do 'ipython profile create mycluster --parallel', then edit |
@@ -116,7 +116,7 b' def find_launcher_class(clsname, kind):' | |||
|
116 | 116 | ========== |
|
117 | 117 | clsname : str |
|
118 | 118 | The full name of the launcher class, either with or without the |
|
119 | module path, or an abbreviation (MPI, SSH, SGE, PBS, LSF, Condor | |
|
119 | module path, or an abbreviation (MPI, SSH, SGE, PBS, LSF, HTCondor | |
|
120 | 120 | WindowsHPC). |
|
121 | 121 | kind : str |
|
122 | 122 | Either 'EngineSet' or 'Controller'. |
@@ -287,7 +287,7 b' class IPClusterEngines(BaseParallelApplication):' | |||
|
287 | 287 | Note that SSH does *not* move the connection files |
|
288 | 288 | around, so you will likely have to do this manually |
|
289 | 289 | unless the machines are on a shared file system. |
|
290 | Condor : use HTCondor to submit engines to a batch queue | |
|
290 | HTCondor : use HTCondor to submit engines to a batch queue | |
|
291 | 291 | WindowsHPC : use Windows HPC |
|
292 | 292 | |
|
293 | 293 | If you are using one of IPython's builtin launchers, you can specify just the |
@@ -489,7 +489,7 b' class IPClusterStart(IPClusterEngines):' | |||
|
489 | 489 | PBS : use PBS (qsub) to submit the controller to a batch queue |
|
490 | 490 | SGE : use SGE (qsub) to submit the controller to a batch queue |
|
491 | 491 | LSF : use LSF (bsub) to submit the controller to a batch queue |
|
492 | Condor: use HTCondor to submit the controller to a batch queue | |
|
492 | HTCondor : use HTCondor to submit the controller to a batch queue | |
|
493 | 493 | SSH : use SSH to start the controller |
|
494 | 494 | WindowsHPC : use Windows HPC |
|
495 | 495 |
@@ -1285,10 +1285,10 b' class LSFEngineSetLauncher(LSFLauncher, BatchClusterAppMixin):' | |||
|
1285 | 1285 | |
|
1286 | 1286 | |
|
1287 | 1287 | |
|
1288 | class CondorLauncher(BatchSystemLauncher): | |
|
1289 | """A BatchSystemLauncher subclass for Condor. | |
|
1288 | class HTCondorLauncher(BatchSystemLauncher): | |
|
1289 | """A BatchSystemLauncher subclass for HTCondor. | |
|
1290 | 1290 | |
|
1291 | Condor requires that we launch the ipengine/ipcontroller scripts rather | |
|
1291 | HTCondor requires that we launch the ipengine/ipcontroller scripts rather | |
|
1292 | 1292 | that the python instance but otherwise is very similar to PBS. This is because |
|
1293 | 1293 | HTCondor destroys sys.executable when launching remote processes - a launched |
|
1294 | 1294 | python process depends on sys.executable to effectively evaluate its |
@@ -1307,9 +1307,9 b' class CondorLauncher(BatchSystemLauncher):' | |||
|
1307 | 1307 | """ |
|
1308 | 1308 | |
|
1309 | 1309 | submit_command = List(['condor_submit'], config=True, |
|
1310 | help="The Condor submit command ['condor_submit']") | |
|
1310 | help="The HTCondor submit command ['condor_submit']") | |
|
1311 | 1311 | delete_command = List(['condor_rm'], config=True, |
|
1312 | help="The Condor delete command ['condor_rm']") | |
|
1312 | help="The HTCondor delete command ['condor_rm']") | |
|
1313 | 1313 | job_id_regexp = CRegExp(r'(\d+)\.$', config=True, |
|
1314 | 1314 | help="Regular expression for identifying the job ID [r'(\d+)\.$']") |
|
1315 | 1315 | job_id_regexp_group = Integer(1, config=True, |
@@ -1324,21 +1324,21 b' class CondorLauncher(BatchSystemLauncher):' | |||
|
1324 | 1324 | """ |
|
1325 | 1325 | if not self.job_array_regexp.search(self.batch_template): |
|
1326 | 1326 | self.log.debug("adding job array settings to batch script") |
|
1327 | #Condor requires that the job array goes at the bottom of the script | |
|
1327 | #HTCondor requires that the job array goes at the bottom of the script | |
|
1328 | 1328 | self.batch_template = '\n'.join([self.batch_template, |
|
1329 | 1329 | self.job_array_template]) |
|
1330 | 1330 | |
|
1331 | 1331 | def _insert_queue_in_script(self): |
|
1332 | """AFAIK, Condor doesn't have a concept of multiple queues that can be | |
|
1332 | """AFAIK, HTCondor doesn't have a concept of multiple queues that can be | |
|
1333 | 1333 | specified in the script. |
|
1334 | 1334 | """ |
|
1335 | 1335 | pass |
|
1336 | 1336 | |
|
1337 | 1337 | |
|
1338 | class CondorControllerLauncher(CondorLauncher, BatchClusterAppMixin): | |
|
1339 | """Launch a controller using Condor.""" | |
|
1338 | class HTCondorControllerLauncher(HTCondorLauncher, BatchClusterAppMixin): | |
|
1339 | """Launch a controller using HTCondor.""" | |
|
1340 | 1340 | |
|
1341 | batch_file_name = Unicode(u'condor_controller', config=True, | |
|
1341 | batch_file_name = Unicode(u'htcondor_controller', config=True, | |
|
1342 | 1342 | help="batch file name for the controller job.") |
|
1343 | 1343 | default_template = Unicode(r""" |
|
1344 | 1344 | universe = vanilla |
@@ -1350,12 +1350,12 b" arguments = --log-to-file '--profile-dir={profile_dir}' --cluster-id='{clu" | |||
|
1350 | 1350 | |
|
1351 | 1351 | def start(self): |
|
1352 | 1352 | """Start the controller by profile or profile_dir.""" |
|
1353 | return super(CondorControllerLauncher, self).start(1) | |
|
1353 | return super(HTCondorControllerLauncher, self).start(1) | |
|
1354 | 1354 | |
|
1355 | 1355 | |
|
1356 | class CondorEngineSetLauncher(CondorLauncher, BatchClusterAppMixin): | |
|
1357 | """Launch Engines using Condor""" | |
|
1358 | batch_file_name = Unicode(u'condor_engines', config=True, | |
|
1356 | class HTCondorEngineSetLauncher(HTCondorLauncher, BatchClusterAppMixin): | |
|
1357 | """Launch Engines using HTCondor""" | |
|
1358 | batch_file_name = Unicode(u'htcondor_engines', config=True, | |
|
1359 | 1359 | help="batch file name for the engine(s) job.") |
|
1360 | 1360 | default_template = Unicode(""" |
|
1361 | 1361 | universe = vanilla |
@@ -1432,10 +1432,10 b' lsf_launchers = [' | |||
|
1432 | 1432 | LSFControllerLauncher, |
|
1433 | 1433 | LSFEngineSetLauncher, |
|
1434 | 1434 | ] |
|
1435 | condor_launchers = [ | |
|
1436 | CondorLauncher, | |
|
1437 | CondorControllerLauncher, | |
|
1438 | CondorEngineSetLauncher, | |
|
1435 | htcondor_launchers = [ | |
|
1436 | HTCondorLauncher, | |
|
1437 | HTCondorControllerLauncher, | |
|
1438 | HTCondorEngineSetLauncher, | |
|
1439 | 1439 | ] |
|
1440 | 1440 | all_launchers = local_launchers + mpi_launchers + ssh_launchers + winhpc_launchers\ |
|
1441 | + pbs_launchers + sge_launchers + lsf_launchers + condor_launchers | |
|
1441 | + pbs_launchers + sge_launchers + lsf_launchers + htcondor_launchers |
@@ -129,8 +129,8 b' class TestSGEControllerLauncher(BatchTest, ControllerLauncherTest, TestCase):' | |||
|
129 | 129 | class TestLSFControllerLauncher(BatchTest, ControllerLauncherTest, TestCase): |
|
130 | 130 | launcher_class = launcher.LSFControllerLauncher |
|
131 | 131 | |
|
132 | class TestCondorControllerLauncher(BatchTest, ControllerLauncherTest, TestCase): | |
|
133 | launcher_class = launcher.CondorControllerLauncher | |
|
132 | class TestHTCondorControllerLauncher(BatchTest, ControllerLauncherTest, TestCase): | |
|
133 | launcher_class = launcher.HTCondorControllerLauncher | |
|
134 | 134 | |
|
135 | 135 | class TestSSHControllerLauncher(SSHTest, ControllerLauncherTest, TestCase): |
|
136 | 136 | launcher_class = launcher.SSHControllerLauncher |
@@ -158,8 +158,8 b' class TestSGEEngineSetLauncher(BatchTest, EngineSetLauncherTest, TestCase):' | |||
|
158 | 158 | class TestLSFEngineSetLauncher(BatchTest, EngineSetLauncherTest, TestCase): |
|
159 | 159 | launcher_class = launcher.LSFEngineSetLauncher |
|
160 | 160 | |
|
161 | class TestCondorEngineSetLauncher(BatchTest, EngineSetLauncherTest, TestCase): | |
|
162 | launcher_class = launcher.CondorEngineSetLauncher | |
|
161 | class TestHTCondorEngineSetLauncher(BatchTest, EngineSetLauncherTest, TestCase): | |
|
162 | launcher_class = launcher.HTCondorEngineSetLauncher | |
|
163 | 163 | |
|
164 | 164 | class TestSSHEngineSetLauncher(EngineSetLauncherTest, TestCase): |
|
165 | 165 | launcher_class = launcher.SSHEngineSetLauncher |
General Comments 0
You need to be logged in to leave comments.
Login now