##// END OF EJS Templates
`ipython profile` prints profile help...
MinRK -
Show More
@@ -1,210 +1,220 b''
1 1 # encoding: utf-8
2 2 """
3 3 An application for managing IPython profiles.
4 4
5 5 To be invoked as the `ipython profile` subcommand.
6 6
7 7 Authors:
8 8
9 9 * Min RK
10 10
11 11 """
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Copyright (C) 2008-2011 The IPython Development Team
15 15 #
16 16 # Distributed under the terms of the BSD License. The full license is in
17 17 # the file COPYING, distributed as part of this software.
18 18 #-----------------------------------------------------------------------------
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Imports
22 22 #-----------------------------------------------------------------------------
23 23
24 24 import logging
25 25 import os
26 26
27 27 from IPython.config.application import Application, boolean_flag
28 28 from IPython.core.application import (
29 29 BaseIPythonApplication, base_flags, base_aliases
30 30 )
31 31 from IPython.core.profiledir import ProfileDir
32 32 from IPython.utils.path import get_ipython_dir
33 33 from IPython.utils.traitlets import Unicode, Bool, Dict
34 34
35 35 #-----------------------------------------------------------------------------
36 36 # Constants
37 37 #-----------------------------------------------------------------------------
38 38
39 39 create_help = """Create an ipcluster profile by name
40 40
41 41 Create an ipython profile directory by its name or
42 42 profile directory path. Profile directories contain
43 43 configuration, log and security related files and are named
44 44 using the convention 'profile_<name>'. By default they are
45 45 located in your ipython directory. Once created, you will
46 46 can edit the configuration files in the profile
47 47 directory to configure IPython. Most users will create a
48 48 cluster directory by profile name,
49 49 `ipython profile create myprofile`, which will put the directory
50 50 in `<ipython_dir>/profile_myprofile`.
51 51 """
52 52 list_help = """List available IPython profiles
53 53
54 54 List all available profiles, by profile location, that can
55 55 be found in the current working directly or in the ipython
56 56 directory. Profile directories are named using the convention
57 57 'profile_<profile>'.
58 58 """
59 59 profile_help = """Manage IPython profiles
60 60
61 61 Profile directories contain
62 62 configuration, log and security related files and are named
63 63 using the convention 'profile_<name>'. By default they are
64 64 located in your ipython directory. You can create profiles
65 65 with `ipython profile create <name>`, or see the profiles you
66 66 already have with `ipython profile list`
67 67
68 68 To get started configuring IPython, simply do:
69 69
70 70 $> ipython profile create
71 71
72 72 and IPython will create the default profile in <ipython_dir>/profile_default,
73 73 where you can edit ipython_config.py to start configuring IPython.
74 74
75 75 """
76 76
77 77 #-----------------------------------------------------------------------------
78 78 # Profile Application Class (for `ipython profile` subcommand)
79 79 #-----------------------------------------------------------------------------
80 80
81 81
82 82
83 83 class ProfileList(Application):
84 84 name = u'ipython-profile'
85 85 description = list_help
86 86
87 87 aliases = Dict(dict(
88 88 ipython_dir = 'ProfileList.ipython_dir',
89 89 log_level = 'Application.log_level',
90 90 ))
91 91 flags = Dict(dict(
92 92 debug = ({'Application' : {'log_level' : 0}},
93 93 "Set log_level to 0, maximizing log output."
94 94 )
95 95 ))
96 96 ipython_dir = Unicode(get_ipython_dir(), config=True,
97 97 help="""
98 98 The name of the IPython directory. This directory is used for logging
99 99 configuration (through profiles), history storage, etc. The default
100 100 is usually $HOME/.ipython. This options can also be specified through
101 101 the environment variable IPYTHON_DIR.
102 102 """
103 103 )
104 104
105 105 def list_profile_dirs(self):
106 106 # Find the search paths
107 107 paths = [os.getcwdu(), self.ipython_dir]
108 108
109 109 self.log.warn('Searching for IPython profiles in paths: %r' % paths)
110 110 for path in paths:
111 111 files = os.listdir(path)
112 112 for f in files:
113 113 full_path = os.path.join(path, f)
114 114 if os.path.isdir(full_path) and f.startswith('profile_'):
115 115 profile = f.split('_',1)[-1]
116 116 start_cmd = 'ipython profile=%s' % profile
117 117 print start_cmd + " ==> " + full_path
118 118
119 119 def start(self):
120 120 self.list_profile_dirs()
121 121
122 122
123 123 create_flags = {}
124 124 create_flags.update(base_flags)
125 125 create_flags.update(boolean_flag('reset', 'ProfileCreate.overwrite',
126 126 "reset config files to defaults", "leave existing config files"))
127 127 create_flags.update(boolean_flag('cluster', 'ProfileCreate.cluster',
128 128 "Include parallel computing config files",
129 129 "Don't include parallel computing config files"))
130 130
131 131 class ProfileCreate(BaseIPythonApplication):
132 132 name = u'ipython-profile'
133 133 description = create_help
134 134 auto_create = Bool(True, config=False)
135 135
136 136 def _copy_config_files_default(self):
137 137 return True
138 138
139 139 cluster = Bool(False, config=True,
140 140 help="whether to include parallel computing config files")
141 141 def _cluster_changed(self, name, old, new):
142 142 cluster_files = [ 'ipcontroller_config.py',
143 143 'ipengine_config.py',
144 144 'ipcluster_config.py'
145 145 ]
146 146 if new:
147 147 for cf in cluster_files:
148 148 self.config_files.append(cf)
149 149 else:
150 150 for cf in cluster_files:
151 151 if cf in self.config_files:
152 152 self.config_files.remove(cf)
153 153
154 154 def parse_command_line(self, argv):
155 155 super(ProfileCreate, self).parse_command_line(argv)
156 156 # accept positional arg as profile name
157 157 if self.extra_args:
158 158 self.profile = self.extra_args[0]
159 159
160 160 flags = Dict(create_flags)
161 161
162 162 aliases = Dict(dict(profile='BaseIPythonApplication.profile'))
163 163
164 164 classes = [ProfileDir]
165 165
166 166 def init_config_files(self):
167 167 super(ProfileCreate, self).init_config_files()
168 168 # use local imports, since these classes may import from here
169 169 from IPython.frontend.terminal.ipapp import TerminalIPythonApp
170 170 apps = [TerminalIPythonApp]
171 171 try:
172 172 from IPython.frontend.qt.console.qtconsoleapp import IPythonQtConsoleApp
173 173 except ImportError:
174 174 pass
175 175 else:
176 176 apps.append(IPythonQtConsoleApp)
177 177 if self.cluster:
178 178 from IPython.parallel.apps.ipcontrollerapp import IPControllerApp
179 179 from IPython.parallel.apps.ipengineapp import IPEngineApp
180 180 from IPython.parallel.apps.ipclusterapp import IPClusterStart
181 181 from IPython.parallel.apps.iploggerapp import IPLoggerApp
182 182 apps.extend([
183 183 IPControllerApp,
184 184 IPEngineApp,
185 185 IPClusterStart,
186 186 IPLoggerApp,
187 187 ])
188 188 for App in apps:
189 189 app = App()
190 190 app.config.update(self.config)
191 191 app.log = self.log
192 192 app.overwrite = self.overwrite
193 193 app.copy_config_files=True
194 194 app.profile = self.profile
195 195 app.init_profile_dir()
196 196 app.init_config_files()
197 197 print 'tic'
198 198
199 199 def stage_default_config_file(self):
200 200 pass
201 201
202 202 class ProfileApp(Application):
203 203 name = u'ipython-profile'
204 204 description = profile_help
205 205
206 206 subcommands = Dict(dict(
207 207 create = (ProfileCreate, "Create a new profile dir with default config files"),
208 208 list = (ProfileList, "List existing profiles")
209 209 ))
210 210
211 def start(self):
212 if self.subapp is None:
213 print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
214 print
215 self.print_description()
216 self.print_subcommands()
217 self.exit(1)
218 else:
219 return self.subapp.start()
220
@@ -1,445 +1,446 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 The ipcluster application.
5 5
6 6 Authors:
7 7
8 8 * Brian Granger
9 9 * MinRK
10 10
11 11 """
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Copyright (C) 2008-2011 The IPython Development Team
15 15 #
16 16 # Distributed under the terms of the BSD License. The full license is in
17 17 # the file COPYING, distributed as part of this software.
18 18 #-----------------------------------------------------------------------------
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Imports
22 22 #-----------------------------------------------------------------------------
23 23
24 24 import errno
25 25 import logging
26 26 import os
27 27 import re
28 28 import signal
29 29
30 30 from subprocess import check_call, CalledProcessError, PIPE
31 31 import zmq
32 32 from zmq.eventloop import ioloop
33 33
34 34 from IPython.config.application import Application, boolean_flag
35 35 from IPython.config.loader import Config
36 36 from IPython.core.application import BaseIPythonApplication
37 37 from IPython.core.profiledir import ProfileDir
38 38 from IPython.utils.daemonize import daemonize
39 39 from IPython.utils.importstring import import_item
40 40 from IPython.utils.traitlets import Int, Unicode, Bool, CFloat, Dict, List
41 41
42 42 from IPython.parallel.apps.baseapp import (
43 43 BaseParallelApplication,
44 44 PIDFileError,
45 45 base_flags, base_aliases
46 46 )
47 47
48 48
49 49 #-----------------------------------------------------------------------------
50 50 # Module level variables
51 51 #-----------------------------------------------------------------------------
52 52
53 53
54 54 default_config_file_name = u'ipcluster_config.py'
55 55
56 56
57 57 _description = """Start an IPython cluster for parallel computing.
58 58
59 59 An IPython cluster consists of 1 controller and 1 or more engines.
60 60 This command automates the startup of these processes using a wide
61 61 range of startup methods (SSH, local processes, PBS, mpiexec,
62 62 Windows HPC Server 2008). To start a cluster with 4 engines on your
63 63 local host simply do 'ipcluster start n=4'. For more complex usage
64 64 you will typically do 'ipcluster create profile=mycluster', then edit
65 65 configuration files, followed by 'ipcluster start profile=mycluster n=4'.
66 66 """
67 67
68 68
69 69 # Exit codes for ipcluster
70 70
71 71 # This will be the exit code if the ipcluster appears to be running because
72 72 # a .pid file exists
73 73 ALREADY_STARTED = 10
74 74
75 75
76 76 # This will be the exit code if ipcluster stop is run, but there is not .pid
77 77 # file to be found.
78 78 ALREADY_STOPPED = 11
79 79
80 80 # This will be the exit code if ipcluster engines is run, but there is not .pid
81 81 # file to be found.
82 82 NO_CLUSTER = 12
83 83
84 84
85 85 #-----------------------------------------------------------------------------
86 86 # Main application
87 87 #-----------------------------------------------------------------------------
88 88 start_help = """Start an IPython cluster for parallel computing
89 89
90 90 Start an ipython cluster by its profile name or cluster
91 91 directory. Cluster directories contain configuration, log and
92 92 security related files and are named using the convention
93 93 'profile_<name>' and should be creating using the 'start'
94 94 subcommand of 'ipcluster'. If your cluster directory is in
95 95 the cwd or the ipython directory, you can simply refer to it
96 96 using its profile name, 'ipcluster start n=4 profile=<profile>`,
97 97 otherwise use the 'profile_dir' option.
98 98 """
99 99 stop_help = """Stop a running IPython cluster
100 100
101 101 Stop a running ipython cluster by its profile name or cluster
102 102 directory. Cluster directories are named using the convention
103 103 'profile_<name>'. If your cluster directory is in
104 104 the cwd or the ipython directory, you can simply refer to it
105 105 using its profile name, 'ipcluster stop profile=<profile>`, otherwise
106 106 use the 'profile_dir' option.
107 107 """
108 108 engines_help = """Start engines connected to an existing IPython cluster
109 109
110 110 Start one or more engines to connect to an existing Cluster
111 111 by profile name or cluster directory.
112 112 Cluster directories contain configuration, log and
113 113 security related files and are named using the convention
114 114 'profile_<name>' and should be creating using the 'start'
115 115 subcommand of 'ipcluster'. If your cluster directory is in
116 116 the cwd or the ipython directory, you can simply refer to it
117 117 using its profile name, 'ipcluster engines n=4 profile=<profile>`,
118 118 otherwise use the 'profile_dir' option.
119 119 """
120 120 stop_aliases = dict(
121 121 signal='IPClusterStop.signal',
122 122 profile='BaseIPythonApplication.profile',
123 123 profile_dir='ProfileDir.location',
124 124 )
125 125
126 126 class IPClusterStop(BaseParallelApplication):
127 127 name = u'ipcluster'
128 128 description = stop_help
129 129 config_file_name = Unicode(default_config_file_name)
130 130
131 131 signal = Int(signal.SIGINT, config=True,
132 132 help="signal to use for stopping processes.")
133 133
134 134 aliases = Dict(stop_aliases)
135 135
136 136 def start(self):
137 137 """Start the app for the stop subcommand."""
138 138 try:
139 139 pid = self.get_pid_from_file()
140 140 except PIDFileError:
141 141 self.log.critical(
142 142 'Could not read pid file, cluster is probably not running.'
143 143 )
144 144 # Here I exit with a unusual exit status that other processes
145 145 # can watch for to learn how I existed.
146 146 self.remove_pid_file()
147 147 self.exit(ALREADY_STOPPED)
148 148
149 149 if not self.check_pid(pid):
150 150 self.log.critical(
151 151 'Cluster [pid=%r] is not running.' % pid
152 152 )
153 153 self.remove_pid_file()
154 154 # Here I exit with a unusual exit status that other processes
155 155 # can watch for to learn how I existed.
156 156 self.exit(ALREADY_STOPPED)
157 157
158 158 elif os.name=='posix':
159 159 sig = self.signal
160 160 self.log.info(
161 161 "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
162 162 )
163 163 try:
164 164 os.kill(pid, sig)
165 165 except OSError:
166 166 self.log.error("Stopping cluster failed, assuming already dead.",
167 167 exc_info=True)
168 168 self.remove_pid_file()
169 169 elif os.name=='nt':
170 170 try:
171 171 # kill the whole tree
172 172 p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE)
173 173 except (CalledProcessError, OSError):
174 174 self.log.error("Stopping cluster failed, assuming already dead.",
175 175 exc_info=True)
176 176 self.remove_pid_file()
177 177
178 178 engine_aliases = {}
179 179 engine_aliases.update(base_aliases)
180 180 engine_aliases.update(dict(
181 181 n='IPClusterEngines.n',
182 182 elauncher = 'IPClusterEngines.engine_launcher_class',
183 183 ))
184 184 class IPClusterEngines(BaseParallelApplication):
185 185
186 186 name = u'ipcluster'
187 187 description = engines_help
188 188 usage = None
189 189 config_file_name = Unicode(default_config_file_name)
190 190 default_log_level = logging.INFO
191 191 classes = List()
192 192 def _classes_default(self):
193 193 from IPython.parallel.apps import launcher
194 194 launchers = launcher.all_launchers
195 195 eslaunchers = [ l for l in launchers if 'EngineSet' in l.__name__]
196 196 return [ProfileDir]+eslaunchers
197 197
198 198 n = Int(2, config=True,
199 199 help="The number of engines to start.")
200 200
201 201 engine_launcher_class = Unicode('LocalEngineSetLauncher',
202 202 config=True,
203 203 help="The class for launching a set of Engines."
204 204 )
205 205 daemonize = Bool(False, config=True,
206 206 help='Daemonize the ipcluster program. This implies --log-to-file')
207 207
208 208 def _daemonize_changed(self, name, old, new):
209 209 if new:
210 210 self.log_to_file = True
211 211
212 212 aliases = Dict(engine_aliases)
213 213 # flags = Dict(flags)
214 214 _stopping = False
215 215
216 216 def initialize(self, argv=None):
217 217 super(IPClusterEngines, self).initialize(argv)
218 218 self.init_signal()
219 219 self.init_launchers()
220 220
221 221 def init_launchers(self):
222 222 self.engine_launcher = self.build_launcher(self.engine_launcher_class)
223 223 self.engine_launcher.on_stop(lambda r: self.loop.stop())
224 224
225 225 def init_signal(self):
226 226 # Setup signals
227 227 signal.signal(signal.SIGINT, self.sigint_handler)
228 228
229 229 def build_launcher(self, clsname):
230 230 """import and instantiate a Launcher based on importstring"""
231 231 if '.' not in clsname:
232 232 # not a module, presume it's the raw name in apps.launcher
233 233 clsname = 'IPython.parallel.apps.launcher.'+clsname
234 234 # print repr(clsname)
235 235 klass = import_item(clsname)
236 236
237 237 launcher = klass(
238 238 work_dir=self.profile_dir.location, config=self.config, log=self.log
239 239 )
240 240 return launcher
241 241
242 242 def start_engines(self):
243 243 self.log.info("Starting %i engines"%self.n)
244 244 self.engine_launcher.start(
245 245 self.n,
246 246 self.profile_dir.location
247 247 )
248 248
249 249 def stop_engines(self):
250 250 self.log.info("Stopping Engines...")
251 251 if self.engine_launcher.running:
252 252 d = self.engine_launcher.stop()
253 253 return d
254 254 else:
255 255 return None
256 256
257 257 def stop_launchers(self, r=None):
258 258 if not self._stopping:
259 259 self._stopping = True
260 260 self.log.error("IPython cluster: stopping")
261 261 self.stop_engines()
262 262 # Wait a few seconds to let things shut down.
263 263 dc = ioloop.DelayedCallback(self.loop.stop, 4000, self.loop)
264 264 dc.start()
265 265
266 266 def sigint_handler(self, signum, frame):
267 267 self.log.debug("SIGINT received, stopping launchers...")
268 268 self.stop_launchers()
269 269
270 270 def start_logging(self):
271 271 # Remove old log files of the controller and engine
272 272 if self.clean_logs:
273 273 log_dir = self.profile_dir.log_dir
274 274 for f in os.listdir(log_dir):
275 275 if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f):
276 276 os.remove(os.path.join(log_dir, f))
277 277 # This will remove old log files for ipcluster itself
278 278 # super(IPBaseParallelApplication, self).start_logging()
279 279
280 280 def start(self):
281 281 """Start the app for the engines subcommand."""
282 282 self.log.info("IPython cluster: started")
283 283 # First see if the cluster is already running
284 284
285 285 # Now log and daemonize
286 286 self.log.info(
287 287 'Starting engines with [daemon=%r]' % self.daemonize
288 288 )
289 289 # TODO: Get daemonize working on Windows or as a Windows Server.
290 290 if self.daemonize:
291 291 if os.name=='posix':
292 292 daemonize()
293 293
294 294 dc = ioloop.DelayedCallback(self.start_engines, 0, self.loop)
295 295 dc.start()
296 296 # Now write the new pid file AFTER our new forked pid is active.
297 297 # self.write_pid_file()
298 298 try:
299 299 self.loop.start()
300 300 except KeyboardInterrupt:
301 301 pass
302 302 except zmq.ZMQError as e:
303 303 if e.errno == errno.EINTR:
304 304 pass
305 305 else:
306 306 raise
307 307
308 308 start_aliases = {}
309 309 start_aliases.update(engine_aliases)
310 310 start_aliases.update(dict(
311 311 delay='IPClusterStart.delay',
312 312 clean_logs='IPClusterStart.clean_logs',
313 313 ))
314 314
315 315 class IPClusterStart(IPClusterEngines):
316 316
317 317 name = u'ipcluster'
318 318 description = start_help
319 319 default_log_level = logging.INFO
320 320 auto_create = Bool(True, config=True,
321 321 help="whether to create the profile_dir if it doesn't exist")
322 322 classes = List()
323 323 def _classes_default(self,):
324 324 from IPython.parallel.apps import launcher
325 325 return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers
326 326
327 327 clean_logs = Bool(True, config=True,
328 328 help="whether to cleanup old logs before starting")
329 329
330 330 delay = CFloat(1., config=True,
331 331 help="delay (in s) between starting the controller and the engines")
332 332
333 333 controller_launcher_class = Unicode('LocalControllerLauncher',
334 334 config=True,
335 335 help="The class for launching a Controller."
336 336 )
337 337 reset = Bool(False, config=True,
338 338 help="Whether to reset config files as part of '--create'."
339 339 )
340 340
341 341 # flags = Dict(flags)
342 342 aliases = Dict(start_aliases)
343 343
344 344 def init_launchers(self):
345 345 self.controller_launcher = self.build_launcher(self.controller_launcher_class)
346 346 self.engine_launcher = self.build_launcher(self.engine_launcher_class)
347 347 self.controller_launcher.on_stop(self.stop_launchers)
348 348
349 349 def start_controller(self):
350 350 self.controller_launcher.start(
351 351 self.profile_dir.location
352 352 )
353 353
354 354 def stop_controller(self):
355 355 # self.log.info("In stop_controller")
356 356 if self.controller_launcher and self.controller_launcher.running:
357 357 return self.controller_launcher.stop()
358 358
359 359 def stop_launchers(self, r=None):
360 360 if not self._stopping:
361 361 self.stop_controller()
362 362 super(IPClusterStart, self).stop_launchers()
363 363
364 364 def start(self):
365 365 """Start the app for the start subcommand."""
366 366 # First see if the cluster is already running
367 367 try:
368 368 pid = self.get_pid_from_file()
369 369 except PIDFileError:
370 370 pass
371 371 else:
372 372 if self.check_pid(pid):
373 373 self.log.critical(
374 374 'Cluster is already running with [pid=%s]. '
375 375 'use "ipcluster stop" to stop the cluster.' % pid
376 376 )
377 377 # Here I exit with a unusual exit status that other processes
378 378 # can watch for to learn how I existed.
379 379 self.exit(ALREADY_STARTED)
380 380 else:
381 381 self.remove_pid_file()
382 382
383 383
384 384 # Now log and daemonize
385 385 self.log.info(
386 386 'Starting ipcluster with [daemon=%r]' % self.daemonize
387 387 )
388 388 # TODO: Get daemonize working on Windows or as a Windows Server.
389 389 if self.daemonize:
390 390 if os.name=='posix':
391 391 daemonize()
392 392
393 393 dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop)
394 394 dc.start()
395 395 dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop)
396 396 dc.start()
397 397 # Now write the new pid file AFTER our new forked pid is active.
398 398 self.write_pid_file()
399 399 try:
400 400 self.loop.start()
401 401 except KeyboardInterrupt:
402 402 pass
403 403 except zmq.ZMQError as e:
404 404 if e.errno == errno.EINTR:
405 405 pass
406 406 else:
407 407 raise
408 408 finally:
409 409 self.remove_pid_file()
410 410
411 411 base='IPython.parallel.apps.ipclusterapp.IPCluster'
412 412
413 413 class IPClusterApp(Application):
414 414 name = u'ipcluster'
415 415 description = _description
416 416
417 417 subcommands = {
418 418 'start' : (base+'Start', start_help),
419 419 'stop' : (base+'Stop', stop_help),
420 420 'engines' : (base+'Engines', engines_help),
421 421 }
422 422
423 423 # no aliases or flags for parent App
424 424 aliases = Dict()
425 425 flags = Dict()
426 426
427 427 def start(self):
428 428 if self.subapp is None:
429 print "No subcommand specified! Must specify one of: %s"%(self.subcommands.keys())
429 print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
430 430 print
431 self.print_description()
431 432 self.print_subcommands()
432 433 self.exit(1)
433 434 else:
434 435 return self.subapp.start()
435 436
436 437 def launch_new_instance():
437 438 """Create and run the IPython cluster."""
438 app = IPBaseParallelApplication.instance()
439 app = IPClusterApp.instance()
439 440 app.initialize()
440 441 app.start()
441 442
442 443
443 444 if __name__ == '__main__':
444 445 launch_new_instance()
445 446
General Comments 0
You need to be logged in to leave comments. Login now