##// END OF EJS Templates
promote aliases and flags, to ensure they have priority over config files...
MinRK -
Show More
@@ -1,436 +1,485 b''
1 1 # encoding: utf-8
2 2 """
3 3 A base class for a configurable application.
4 4
5 5 Authors:
6 6
7 7 * Brian Granger
8 8 * Min RK
9 9 """
10 10
11 11 #-----------------------------------------------------------------------------
12 12 # Copyright (C) 2008-2011 The IPython Development Team
13 13 #
14 14 # Distributed under the terms of the BSD License. The full license is in
15 15 # the file COPYING, distributed as part of this software.
16 16 #-----------------------------------------------------------------------------
17 17
18 18 #-----------------------------------------------------------------------------
19 19 # Imports
20 20 #-----------------------------------------------------------------------------
21 21
22 22 import logging
23 23 import os
24 24 import re
25 25 import sys
26 26 from copy import deepcopy
27 from collections import defaultdict
27 28
28 29 from IPython.config.configurable import SingletonConfigurable
29 30 from IPython.config.loader import (
30 31 KVArgParseConfigLoader, PyFileConfigLoader, Config, ArgumentError, ConfigFileNotFound,
31 32 )
32 33
33 34 from IPython.utils.traitlets import (
34 35 Unicode, List, Int, Enum, Dict, Instance, TraitError
35 36 )
36 37 from IPython.utils.importstring import import_item
37 38 from IPython.utils.text import indent, wrap_paragraphs, dedent
38 39
39 40 #-----------------------------------------------------------------------------
40 41 # function for re-wrapping a helpstring
41 42 #-----------------------------------------------------------------------------
42 43
43 44 #-----------------------------------------------------------------------------
44 45 # Descriptions for the various sections
45 46 #-----------------------------------------------------------------------------
46 47
47 48 # merge flags&aliases into options
48 49 option_description = """
49 50 Arguments that take values are actually convenience aliases to full
50 51 Configurables, whose aliases are listed on the help line. For more information
51 52 on full configurables, see '--help-all'.
52 53 """.strip() # trim newlines of front and back
53 54
54 55 keyvalue_description = """
55 56 Parameters are set from command-line arguments of the form:
56 57 `--Class.trait=value`.
57 58 This line is evaluated in Python, so simple expressions are allowed, e.g.::
58 59 `--C.a='range(3)'` For setting C.a=[0,1,2].
59 60 """.strip() # trim newlines of front and back
60 61
61 62 subcommand_description = """
62 63 Subcommands are launched as `{app} cmd [args]`. For information on using
63 64 subcommand 'cmd', do: `{app} cmd -h`.
64 65 """.strip().format(app=os.path.basename(sys.argv[0]))
65 66 # get running program name
66 67
67 68 #-----------------------------------------------------------------------------
68 69 # Application class
69 70 #-----------------------------------------------------------------------------
70 71
71 72
72 73 class ApplicationError(Exception):
73 74 pass
74 75
75 76
76 77 class Application(SingletonConfigurable):
77 78 """A singleton application with full configuration support."""
78 79
79 80 # The name of the application, will usually match the name of the command
80 81 # line application
81 82 name = Unicode(u'application')
82 83
83 84 # The description of the application that is printed at the beginning
84 85 # of the help.
85 86 description = Unicode(u'This is an application.')
86 87 # default section descriptions
87 88 option_description = Unicode(option_description)
88 89 keyvalue_description = Unicode(keyvalue_description)
89 90 subcommand_description = Unicode(subcommand_description)
90 91
91 92 # The usage and example string that goes at the end of the help string.
92 93 examples = Unicode()
93 94
94 95 # A sequence of Configurable subclasses whose config=True attributes will
95 96 # be exposed at the command line.
96 97 classes = List([])
97 98
98 99 # The version string of this application.
99 100 version = Unicode(u'0.0')
100 101
101 102 # The log level for the application
102 103 log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
103 104 default_value=logging.WARN,
104 105 config=True,
105 106 help="Set the log level by value or name.")
106 107 def _log_level_changed(self, name, old, new):
107 108 """Adjust the log level when log_level is set."""
108 109 if isinstance(new, basestring):
109 110 new = getattr(logging, new)
110 111 self.log_level = new
111 112 self.log.setLevel(new)
112 113
113 114 # the alias map for configurables
114 115 aliases = Dict({'log-level' : 'Application.log_level'})
115 116
116 117 # flags for loading Configurables or store_const style flags
117 118 # flags are loaded from this dict by '--key' flags
118 119 # this must be a dict of two-tuples, the first element being the Config/dict
119 120 # and the second being the help string for the flag
120 121 flags = Dict()
121 122 def _flags_changed(self, name, old, new):
122 123 """ensure flags dict is valid"""
123 124 for key,value in new.iteritems():
124 125 assert len(value) == 2, "Bad flag: %r:%s"%(key,value)
125 126 assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value)
126 127 assert isinstance(value[1], basestring), "Bad flag: %r:%s"%(key,value)
127 128
128 129
129 130 # subcommands for launching other applications
130 131 # if this is not empty, this will be a parent Application
131 132 # this must be a dict of two-tuples,
132 133 # the first element being the application class/import string
133 134 # and the second being the help string for the subcommand
134 135 subcommands = Dict()
135 136 # parse_command_line will initialize a subapp, if requested
136 137 subapp = Instance('IPython.config.application.Application', allow_none=True)
137 138
138 139 # extra command-line arguments that don't set config values
139 140 extra_args = List(Unicode)
140 141
141 142
142 143 def __init__(self, **kwargs):
143 144 SingletonConfigurable.__init__(self, **kwargs)
144 145 # Ensure my class is in self.classes, so my attributes appear in command line
145 146 # options and config files.
146 147 if self.__class__ not in self.classes:
147 148 self.classes.insert(0, self.__class__)
148 149
149 150 self.init_logging()
150 151
151 152 def _config_changed(self, name, old, new):
152 153 SingletonConfigurable._config_changed(self, name, old, new)
153 154 self.log.debug('Config changed:')
154 155 self.log.debug(repr(new))
155 156
156 157 def init_logging(self):
157 158 """Start logging for this application.
158 159
159 160 The default is to log to stdout using a StreaHandler. The log level
160 161 starts at loggin.WARN, but this can be adjusted by setting the
161 162 ``log_level`` attribute.
162 163 """
163 164 self.log = logging.getLogger(self.__class__.__name__)
164 165 self.log.setLevel(self.log_level)
165 166 if sys.executable.endswith('pythonw.exe'):
166 167 # this should really go to a file, but file-logging is only
167 168 # hooked up in parallel applications
168 169 self._log_handler = logging.StreamHandler(open(os.devnull, 'w'))
169 170 else:
170 171 self._log_handler = logging.StreamHandler()
171 172 self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
172 173 self._log_handler.setFormatter(self._log_formatter)
173 174 self.log.addHandler(self._log_handler)
174 175
175 176 def initialize(self, argv=None):
176 177 """Do the basic steps to configure me.
177 178
178 179 Override in subclasses.
179 180 """
180 181 self.parse_command_line(argv)
181 182
182 183
183 184 def start(self):
184 185 """Start the app mainloop.
185 186
186 187 Override in subclasses.
187 188 """
188 189 if self.subapp is not None:
189 190 return self.subapp.start()
190 191
191 192 def print_alias_help(self):
192 193 """Print the alias part of the help."""
193 194 if not self.aliases:
194 195 return
195 196
196 197 lines = []
197 198 classdict = {}
198 199 for cls in self.classes:
199 200 # include all parents (up to, but excluding Configurable) in available names
200 201 for c in cls.mro()[:-3]:
201 202 classdict[c.__name__] = c
202 203
203 204 for alias, longname in self.aliases.iteritems():
204 205 classname, traitname = longname.split('.',1)
205 206 cls = classdict[classname]
206 207
207 208 trait = cls.class_traits(config=True)[traitname]
208 209 help = cls.class_get_trait_help(trait).splitlines()
209 210 # reformat first line
210 211 help[0] = help[0].replace(longname, alias) + ' (%s)'%longname
211 212 if len(alias) == 1:
212 213 help[0] = help[0].replace('--%s='%alias, '-%s '%alias)
213 214 lines.extend(help)
214 215 # lines.append('')
215 216 print os.linesep.join(lines)
216 217
217 218 def print_flag_help(self):
218 219 """Print the flag part of the help."""
219 220 if not self.flags:
220 221 return
221 222
222 223 lines = []
223 224 for m, (cfg,help) in self.flags.iteritems():
224 225 prefix = '--' if len(m) > 1 else '-'
225 226 lines.append(prefix+m)
226 227 lines.append(indent(dedent(help.strip())))
227 228 # lines.append('')
228 229 print os.linesep.join(lines)
229 230
230 231 def print_options(self):
231 232 if not self.flags and not self.aliases:
232 233 return
233 234 lines = ['Options']
234 235 lines.append('-'*len(lines[0]))
235 236 lines.append('')
236 237 for p in wrap_paragraphs(self.option_description):
237 238 lines.append(p)
238 239 lines.append('')
239 240 print os.linesep.join(lines)
240 241 self.print_flag_help()
241 242 self.print_alias_help()
242 243 print
243 244
244 245 def print_subcommands(self):
245 246 """Print the subcommand part of the help."""
246 247 if not self.subcommands:
247 248 return
248 249
249 250 lines = ["Subcommands"]
250 251 lines.append('-'*len(lines[0]))
251 252 lines.append('')
252 253 for p in wrap_paragraphs(self.subcommand_description):
253 254 lines.append(p)
254 255 lines.append('')
255 256 for subc, (cls, help) in self.subcommands.iteritems():
256 257 lines.append(subc)
257 258 if help:
258 259 lines.append(indent(dedent(help.strip())))
259 260 lines.append('')
260 261 print os.linesep.join(lines)
261 262
262 263 def print_help(self, classes=False):
263 264 """Print the help for each Configurable class in self.classes.
264 265
265 266 If classes=False (the default), only flags and aliases are printed.
266 267 """
267 268 self.print_subcommands()
268 269 self.print_options()
269 270
270 271 if classes:
271 272 if self.classes:
272 273 print "Class parameters"
273 274 print "----------------"
274 275 print
275 276 for p in wrap_paragraphs(self.keyvalue_description):
276 277 print p
277 278 print
278 279
279 280 for cls in self.classes:
280 281 cls.class_print_help()
281 282 print
282 283 else:
283 284 print "To see all available configurables, use `--help-all`"
284 285 print
285 286
286 287 def print_description(self):
287 288 """Print the application description."""
288 289 for p in wrap_paragraphs(self.description):
289 290 print p
290 291 print
291 292
292 293 def print_examples(self):
293 294 """Print usage and examples.
294 295
295 296 This usage string goes at the end of the command line help string
296 297 and should contain examples of the application's usage.
297 298 """
298 299 if self.examples:
299 300 print "Examples"
300 301 print "--------"
301 302 print
302 303 print indent(dedent(self.examples.strip()))
303 304 print
304 305
305 306 def print_version(self):
306 307 """Print the version string."""
307 308 print self.version
308 309
309 310 def update_config(self, config):
310 311 """Fire the traits events when the config is updated."""
311 312 # Save a copy of the current config.
312 313 newconfig = deepcopy(self.config)
313 314 # Merge the new config into the current one.
314 315 newconfig._merge(config)
315 316 # Save the combined config as self.config, which triggers the traits
316 317 # events.
317 318 self.config = newconfig
318 319
319 320 def initialize_subcommand(self, subc, argv=None):
320 321 """Initialize a subcommand with argv."""
321 322 subapp,help = self.subcommands.get(subc)
322 323
323 324 if isinstance(subapp, basestring):
324 325 subapp = import_item(subapp)
325 326
326 327 # clear existing instances
327 328 self.__class__.clear_instance()
328 329 # instantiate
329 330 self.subapp = subapp.instance()
330 331 # and initialize subapp
331 332 self.subapp.initialize(argv)
332 333
334 def flatten_flags(self):
335 """flatten flags and aliases, so cl-args override as expected.
336
337 This prevents issues such as an alias pointing to InteractiveShell,
338 but a config file setting the same trait in TerminalInteraciveShell
339 getting inappropriate priority over the command-line arg.
340
341 Only aliases with exactly one descendent in the class list
342 will be promoted.
343
344 """
345 # build a tree of classes in our list that inherit from a particular
346 # it will be a dict by parent classname of classes in our list
347 # that are descendents
348 mro_tree = defaultdict(list)
349 for cls in self.classes:
350 clsname = cls.__name__
351 for parent in cls.mro()[1:-3]:
352 # exclude cls itself and Configurable,HasTraits,object
353 mro_tree[parent.__name__].append(clsname)
354 # flatten aliases, which have the form:
355 # { 'alias' : 'Class.trait' }
356 aliases = {}
357 for alias, cls_trait in self.aliases.iteritems():
358 cls,trait = cls_trait.split('.',1)
359 children = mro_tree[cls]
360 if len(children) == 1:
361 # exactly one descendent, promote alias
362 cls = children[0]
363 aliases[alias] = '.'.join([cls,trait])
364
365 # flatten flags, which are of the form:
366 # { 'key' : ({'Cls' : {'trait' : value}}, 'help')}
367 flags = {}
368 for key, (flagdict, help) in self.flags.iteritems():
369 newflag = {}
370 for cls, subdict in flagdict.iteritems():
371 children = mro_tree[cls]
372 # exactly one descendent, promote flag section
373 if len(children) == 1:
374 cls = children[0]
375 newflag[cls] = subdict
376 flags[key] = (newflag, help)
377 return flags, aliases
378
333 379 def parse_command_line(self, argv=None):
334 380 """Parse the command line arguments."""
335 381 argv = sys.argv[1:] if argv is None else argv
336 382
337 383 if self.subcommands and len(argv) > 0:
338 384 # we have subcommands, and one may have been specified
339 385 subc, subargv = argv[0], argv[1:]
340 386 if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands:
341 387 # it's a subcommand, and *not* a flag or class parameter
342 388 return self.initialize_subcommand(subc, subargv)
343 389
344 390 if '-h' in argv or '--help' in argv or '--help-all' in argv:
345 391 self.print_description()
346 392 self.print_help('--help-all' in argv)
347 393 self.print_examples()
348 394 self.exit(0)
349 395
350 396 if '--version' in argv:
351 397 self.print_version()
352 398 self.exit(0)
353 399
354 loader = KVArgParseConfigLoader(argv=argv, aliases=self.aliases,
355 flags=self.flags)
400 # flatten flags&aliases, so cl-args get appropriate priority:
401 flags,aliases = self.flatten_flags()
402
403 loader = KVArgParseConfigLoader(argv=argv, aliases=aliases,
404 flags=flags)
356 405 try:
357 406 config = loader.load_config()
358 407 self.update_config(config)
359 408 except (TraitError, ArgumentError) as e:
360 409 self.print_description()
361 410 self.print_help()
362 411 self.print_examples()
363 412 self.log.fatal(str(e))
364 413 self.exit(1)
365 414 # store unparsed args in extra_args
366 415 self.extra_args = loader.extra_args
367 416
368 417 def load_config_file(self, filename, path=None):
369 418 """Load a .py based config file by filename and path."""
370 419 loader = PyFileConfigLoader(filename, path=path)
371 420 try:
372 421 config = loader.load_config()
373 422 except ConfigFileNotFound:
374 423 # problem finding the file, raise
375 424 raise
376 425 except Exception:
377 426 # try to get the full filename, but it will be empty in the
378 427 # unlikely event that the error raised before filefind finished
379 428 filename = loader.full_filename or filename
380 429 # problem while running the file
381 430 self.log.error("Exception while loading config file %s",
382 431 filename, exc_info=True)
383 432 else:
384 433 self.log.debug("Loaded config file: %s", loader.full_filename)
385 434 self.update_config(config)
386 435
387 436 def generate_config_file(self):
388 437 """generate default config file from Configurables"""
389 438 lines = ["# Configuration file for %s."%self.name]
390 439 lines.append('')
391 440 lines.append('c = get_config()')
392 441 lines.append('')
393 442 for cls in self.classes:
394 443 lines.append(cls.class_config_section())
395 444 return '\n'.join(lines)
396 445
397 446 def exit(self, exit_status=0):
398 447 self.log.debug("Exiting application: %s" % self.name)
399 448 sys.exit(exit_status)
400 449
401 450 #-----------------------------------------------------------------------------
402 451 # utility functions, for convenience
403 452 #-----------------------------------------------------------------------------
404 453
405 454 def boolean_flag(name, configurable, set_help='', unset_help=''):
406 455 """Helper for building basic --trait, --no-trait flags.
407 456
408 457 Parameters
409 458 ----------
410 459
411 460 name : str
412 461 The name of the flag.
413 462 configurable : str
414 463 The 'Class.trait' string of the trait to be set/unset with the flag
415 464 set_help : unicode
416 465 help string for --name flag
417 466 unset_help : unicode
418 467 help string for --no-name flag
419 468
420 469 Returns
421 470 -------
422 471
423 472 cfg : dict
424 473 A dict with two keys: 'name', and 'no-name', for setting and unsetting
425 474 the trait, respectively.
426 475 """
427 476 # default helpstrings
428 477 set_help = set_help or "set %s=True"%configurable
429 478 unset_help = unset_help or "set %s=False"%configurable
430 479
431 480 cls,trait = configurable.split('.')
432 481
433 482 setter = {cls : {trait : True}}
434 483 unsetter = {cls : {trait : False}}
435 484 return {name : (setter, set_help), 'no-'+name : (unsetter, unset_help)}
436 485
@@ -1,146 +1,175 b''
1 1 """
2 2 Tests for IPython.config.application.Application
3 3
4 4 Authors:
5 5
6 6 * Brian Granger
7 7 """
8 8
9 9 #-----------------------------------------------------------------------------
10 10 # Copyright (C) 2008-2011 The IPython Development Team
11 11 #
12 12 # Distributed under the terms of the BSD License. The full license is in
13 13 # the file COPYING, distributed as part of this software.
14 14 #-----------------------------------------------------------------------------
15 15
16 16 #-----------------------------------------------------------------------------
17 17 # Imports
18 18 #-----------------------------------------------------------------------------
19 19
20 import logging
20 21 from unittest import TestCase
21 22
22 23 from IPython.config.configurable import Configurable
24 from IPython.config.loader import Config
23 25
24 26 from IPython.config.application import (
25 27 Application
26 28 )
27 29
28 30 from IPython.utils.traitlets import (
29 31 Bool, Unicode, Int, Float, List, Dict
30 32 )
31 33
32 34 #-----------------------------------------------------------------------------
33 35 # Code
34 36 #-----------------------------------------------------------------------------
35 37
36 38 class Foo(Configurable):
37 39
38 40 i = Int(0, config=True, help="The integer i.")
39 41 j = Int(1, config=True, help="The integer j.")
40 42 name = Unicode(u'Brian', config=True, help="First name.")
41 43
42 44
43 45 class Bar(Configurable):
44 46
45 47 b = Int(0, config=True, help="The integer b.")
46 48 enabled = Bool(True, config=True, help="Enable bar.")
47 49
48 50
49 51 class MyApp(Application):
50 52
51 53 name = Unicode(u'myapp')
52 54 running = Bool(False, config=True,
53 55 help="Is the app running?")
54 56 classes = List([Bar, Foo])
55 57 config_file = Unicode(u'', config=True,
56 58 help="Load this config file")
57 59
58 60 aliases = Dict({
59 61 'i' : 'Foo.i',
60 62 'j' : 'Foo.j',
61 63 'name' : 'Foo.name',
62 64 'enabled' : 'Bar.enabled',
63 'log-level' : 'MyApp.log_level',
65 'log-level' : 'Application.log_level',
64 66 })
65 67
66 68 flags = Dict(dict(enable=({'Bar': {'enabled' : True}}, "Set Bar.enabled to True"),
67 disable=({'Bar': {'enabled' : False}}, "Set Bar.enabled to False")))
69 disable=({'Bar': {'enabled' : False}}, "Set Bar.enabled to False"),
70 crit=({'Application' : {'log_level' : logging.CRITICAL}},
71 "set level=CRITICAL"),
72 ))
68 73
69 74 def init_foo(self):
70 75 self.foo = Foo(config=self.config)
71 76
72 77 def init_bar(self):
73 78 self.bar = Bar(config=self.config)
74 79
75 80
76 81 class TestApplication(TestCase):
77 82
78 83 def test_basic(self):
79 84 app = MyApp()
80 85 self.assertEquals(app.name, u'myapp')
81 86 self.assertEquals(app.running, False)
82 87 self.assertEquals(app.classes, [MyApp,Bar,Foo])
83 88 self.assertEquals(app.config_file, u'')
84 89
85 90 def test_config(self):
86 91 app = MyApp()
87 92 app.parse_command_line(["--i=10","--Foo.j=10","--enabled=False","--log-level=50"])
88 93 config = app.config
89 94 self.assertEquals(config.Foo.i, 10)
90 95 self.assertEquals(config.Foo.j, 10)
91 96 self.assertEquals(config.Bar.enabled, False)
92 97 self.assertEquals(config.MyApp.log_level,50)
93 98
94 99 def test_config_propagation(self):
95 100 app = MyApp()
96 101 app.parse_command_line(["--i=10","--Foo.j=10","--enabled=False","--log-level=50"])
97 102 app.init_foo()
98 103 app.init_bar()
99 104 self.assertEquals(app.foo.i, 10)
100 105 self.assertEquals(app.foo.j, 10)
101 106 self.assertEquals(app.bar.enabled, False)
102 107
103 108 def test_flags(self):
104 109 app = MyApp()
105 110 app.parse_command_line(["--disable"])
106 111 app.init_bar()
107 112 self.assertEquals(app.bar.enabled, False)
108 113 app.parse_command_line(["--enable"])
109 114 app.init_bar()
110 115 self.assertEquals(app.bar.enabled, True)
111 116
112 117 def test_aliases(self):
113 118 app = MyApp()
114 119 app.parse_command_line(["--i=5", "--j=10"])
115 120 app.init_foo()
116 121 self.assertEquals(app.foo.i, 5)
117 122 app.init_foo()
118 123 self.assertEquals(app.foo.j, 10)
119 124
120 125 def test_flag_clobber(self):
121 126 """test that setting flags doesn't clobber existing settings"""
122 127 app = MyApp()
123 128 app.parse_command_line(["--Bar.b=5", "--disable"])
124 129 app.init_bar()
125 130 self.assertEquals(app.bar.enabled, False)
126 131 self.assertEquals(app.bar.b, 5)
127 132 app.parse_command_line(["--enable", "--Bar.b=10"])
128 133 app.init_bar()
129 134 self.assertEquals(app.bar.enabled, True)
130 135 self.assertEquals(app.bar.b, 10)
131 136
137 def test_flatten_flags(self):
138 cfg = Config()
139 cfg.MyApp.log_level = logging.WARN
140 app = MyApp()
141 app.update_config(cfg)
142 self.assertEquals(app.log_level, logging.WARN)
143 self.assertEquals(app.config.MyApp.log_level, logging.WARN)
144 app.initialize(["--crit"])
145 self.assertEquals(app.log_level, logging.CRITICAL)
146 # this would be app.config.Application.log_level if it failed:
147 self.assertEquals(app.config.MyApp.log_level, logging.CRITICAL)
148
149 def test_flatten_aliases(self):
150 cfg = Config()
151 cfg.MyApp.log_level = logging.WARN
152 app = MyApp()
153 app.update_config(cfg)
154 self.assertEquals(app.log_level, logging.WARN)
155 self.assertEquals(app.config.MyApp.log_level, logging.WARN)
156 app.initialize(["--log-level", "CRITICAL"])
157 self.assertEquals(app.log_level, logging.CRITICAL)
158 # this would be app.config.Application.log_level if it failed:
159 self.assertEquals(app.config.MyApp.log_level, "CRITICAL")
160
132 161 def test_extra_args(self):
133 162 app = MyApp()
134 163 app.parse_command_line(["--Bar.b=5", 'extra', "--disable", 'args'])
135 164 app.init_bar()
136 165 self.assertEquals(app.bar.enabled, False)
137 166 self.assertEquals(app.bar.b, 5)
138 167 self.assertEquals(app.extra_args, ['extra', 'args'])
139 168 app = MyApp()
140 169 app.parse_command_line(["--Bar.b=5", '--', 'extra', "--disable", 'args'])
141 170 app.init_bar()
142 171 self.assertEquals(app.bar.enabled, True)
143 172 self.assertEquals(app.bar.b, 5)
144 173 self.assertEquals(app.extra_args, ['extra', '--disable', 'args'])
145 174
146 175
@@ -1,528 +1,522 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 The ipcluster application.
5 5
6 6 Authors:
7 7
8 8 * Brian Granger
9 9 * MinRK
10 10
11 11 """
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Copyright (C) 2008-2011 The IPython Development Team
15 15 #
16 16 # Distributed under the terms of the BSD License. The full license is in
17 17 # the file COPYING, distributed as part of this software.
18 18 #-----------------------------------------------------------------------------
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Imports
22 22 #-----------------------------------------------------------------------------
23 23
24 24 import errno
25 25 import logging
26 26 import os
27 27 import re
28 28 import signal
29 29
30 30 from subprocess import check_call, CalledProcessError, PIPE
31 31 import zmq
32 32 from zmq.eventloop import ioloop
33 33
34 34 from IPython.config.application import Application, boolean_flag
35 35 from IPython.config.loader import Config
36 36 from IPython.core.application import BaseIPythonApplication
37 37 from IPython.core.profiledir import ProfileDir
38 38 from IPython.utils.daemonize import daemonize
39 39 from IPython.utils.importstring import import_item
40 40 from IPython.utils.sysinfo import num_cpus
41 41 from IPython.utils.traitlets import (Int, Unicode, Bool, CFloat, Dict, List,
42 42 DottedObjectName)
43 43
44 44 from IPython.parallel.apps.baseapp import (
45 45 BaseParallelApplication,
46 46 PIDFileError,
47 47 base_flags, base_aliases
48 48 )
49 49
50 50
51 51 #-----------------------------------------------------------------------------
52 52 # Module level variables
53 53 #-----------------------------------------------------------------------------
54 54
55 55
56 56 default_config_file_name = u'ipcluster_config.py'
57 57
58 58
59 59 _description = """Start an IPython cluster for parallel computing.
60 60
61 61 An IPython cluster consists of 1 controller and 1 or more engines.
62 62 This command automates the startup of these processes using a wide
63 63 range of startup methods (SSH, local processes, PBS, mpiexec,
64 64 Windows HPC Server 2008). To start a cluster with 4 engines on your
65 65 local host simply do 'ipcluster start --n=4'. For more complex usage
66 66 you will typically do 'ipython profile create mycluster --parallel', then edit
67 67 configuration files, followed by 'ipcluster start --profile=mycluster --n=4'.
68 68 """
69 69
70 70 _main_examples = """
71 71 ipcluster start --n=4 # start a 4 node cluster on localhost
72 72 ipcluster start -h # show the help string for the start subcmd
73 73
74 74 ipcluster stop -h # show the help string for the stop subcmd
75 75 ipcluster engines -h # show the help string for the engines subcmd
76 76 """
77 77
78 78 _start_examples = """
79 79 ipython profile create mycluster --parallel # create mycluster profile
80 80 ipcluster start --profile=mycluster --n=4 # start mycluster with 4 nodes
81 81 """
82 82
83 83 _stop_examples = """
84 84 ipcluster stop --profile=mycluster # stop a running cluster by profile name
85 85 """
86 86
87 87 _engines_examples = """
88 88 ipcluster engines --profile=mycluster --n=4 # start 4 engines only
89 89 """
90 90
91 91
92 92 # Exit codes for ipcluster
93 93
94 94 # This will be the exit code if the ipcluster appears to be running because
95 95 # a .pid file exists
96 96 ALREADY_STARTED = 10
97 97
98 98
99 99 # This will be the exit code if ipcluster stop is run, but there is not .pid
100 100 # file to be found.
101 101 ALREADY_STOPPED = 11
102 102
103 103 # This will be the exit code if ipcluster engines is run, but there is not .pid
104 104 # file to be found.
105 105 NO_CLUSTER = 12
106 106
107 107
108 108 #-----------------------------------------------------------------------------
109 109 # Main application
110 110 #-----------------------------------------------------------------------------
111 111 start_help = """Start an IPython cluster for parallel computing
112 112
113 113 Start an ipython cluster by its profile name or cluster
114 114 directory. Cluster directories contain configuration, log and
115 115 security related files and are named using the convention
116 116 'profile_<name>' and should be creating using the 'start'
117 117 subcommand of 'ipcluster'. If your cluster directory is in
118 118 the cwd or the ipython directory, you can simply refer to it
119 119 using its profile name, 'ipcluster start --n=4 --profile=<profile>`,
120 120 otherwise use the 'profile-dir' option.
121 121 """
122 122 stop_help = """Stop a running IPython cluster
123 123
124 124 Stop a running ipython cluster by its profile name or cluster
125 125 directory. Cluster directories are named using the convention
126 126 'profile_<name>'. If your cluster directory is in
127 127 the cwd or the ipython directory, you can simply refer to it
128 128 using its profile name, 'ipcluster stop --profile=<profile>`, otherwise
129 129 use the '--profile-dir' option.
130 130 """
131 131 engines_help = """Start engines connected to an existing IPython cluster
132 132
133 133 Start one or more engines to connect to an existing Cluster
134 134 by profile name or cluster directory.
135 135 Cluster directories contain configuration, log and
136 136 security related files and are named using the convention
137 137 'profile_<name>' and should be creating using the 'start'
138 138 subcommand of 'ipcluster'. If your cluster directory is in
139 139 the cwd or the ipython directory, you can simply refer to it
140 140 using its profile name, 'ipcluster engines --n=4 --profile=<profile>`,
141 141 otherwise use the 'profile-dir' option.
142 142 """
143 143 stop_aliases = dict(
144 144 signal='IPClusterStop.signal',
145 145 )
146 146 stop_aliases.update(base_aliases)
147 147
148 148 class IPClusterStop(BaseParallelApplication):
149 149 name = u'ipcluster'
150 150 description = stop_help
151 151 examples = _stop_examples
152 152 config_file_name = Unicode(default_config_file_name)
153 153
154 154 signal = Int(signal.SIGINT, config=True,
155 155 help="signal to use for stopping processes.")
156 156
157 157 aliases = Dict(stop_aliases)
158 158
159 159 def start(self):
160 160 """Start the app for the stop subcommand."""
161 161 try:
162 162 pid = self.get_pid_from_file()
163 163 except PIDFileError:
164 164 self.log.critical(
165 165 'Could not read pid file, cluster is probably not running.'
166 166 )
167 167 # Here I exit with a unusual exit status that other processes
168 168 # can watch for to learn how I existed.
169 169 self.remove_pid_file()
170 170 self.exit(ALREADY_STOPPED)
171 171
172 172 if not self.check_pid(pid):
173 173 self.log.critical(
174 174 'Cluster [pid=%r] is not running.' % pid
175 175 )
176 176 self.remove_pid_file()
177 177 # Here I exit with a unusual exit status that other processes
178 178 # can watch for to learn how I existed.
179 179 self.exit(ALREADY_STOPPED)
180 180
181 181 elif os.name=='posix':
182 182 sig = self.signal
183 183 self.log.info(
184 184 "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
185 185 )
186 186 try:
187 187 os.kill(pid, sig)
188 188 except OSError:
189 189 self.log.error("Stopping cluster failed, assuming already dead.",
190 190 exc_info=True)
191 191 self.remove_pid_file()
192 192 elif os.name=='nt':
193 193 try:
194 194 # kill the whole tree
195 195 p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE)
196 196 except (CalledProcessError, OSError):
197 197 self.log.error("Stopping cluster failed, assuming already dead.",
198 198 exc_info=True)
199 199 self.remove_pid_file()
200 200
201 201 engine_aliases = {}
202 202 engine_aliases.update(base_aliases)
203 203 engine_aliases.update(dict(
204 204 n='IPClusterEngines.n',
205 205 engines = 'IPClusterEngines.engine_launcher_class',
206 206 daemonize = 'IPClusterEngines.daemonize',
207 207 ))
208 208 engine_flags = {}
209 209 engine_flags.update(base_flags)
210 210
211 211 engine_flags.update(dict(
212 212 daemonize=(
213 213 {'IPClusterEngines' : {'daemonize' : True}},
214 214 """run the cluster into the background (not available on Windows)""",
215 215 )
216 216 ))
217 217 class IPClusterEngines(BaseParallelApplication):
218 218
219 219 name = u'ipcluster'
220 220 description = engines_help
221 221 examples = _engines_examples
222 222 usage = None
223 223 config_file_name = Unicode(default_config_file_name)
224 224 default_log_level = logging.INFO
225 225 classes = List()
226 226 def _classes_default(self):
227 227 from IPython.parallel.apps import launcher
228 228 launchers = launcher.all_launchers
229 229 eslaunchers = [ l for l in launchers if 'EngineSet' in l.__name__]
230 230 return [ProfileDir]+eslaunchers
231 231
232 232 n = Int(num_cpus(), config=True,
233 233 help="""The number of engines to start. The default is to use one for each
234 234 CPU on your machine""")
235 235
236 236 engine_launcher_class = DottedObjectName('LocalEngineSetLauncher',
237 237 config=True,
238 238 help="""The class for launching a set of Engines. Change this value
239 239 to use various batch systems to launch your engines, such as PBS,SGE,MPIExec,etc.
240 240 Each launcher class has its own set of configuration options, for making sure
241 241 it will work in your environment.
242 242
243 243 You can also write your own launcher, and specify it's absolute import path,
244 244 as in 'mymodule.launcher.FTLEnginesLauncher`.
245 245
246 246 Examples include:
247 247
248 248 LocalEngineSetLauncher : start engines locally as subprocesses [default]
249 249 MPIExecEngineSetLauncher : use mpiexec to launch in an MPI environment
250 250 PBSEngineSetLauncher : use PBS (qsub) to submit engines to a batch queue
251 251 SGEEngineSetLauncher : use SGE (qsub) to submit engines to a batch queue
252 252 SSHEngineSetLauncher : use SSH to start the controller
253 253 Note that SSH does *not* move the connection files
254 254 around, so you will likely have to do this manually
255 255 unless the machines are on a shared file system.
256 256 WindowsHPCEngineSetLauncher : use Windows HPC
257 257 """
258 258 )
259 259 daemonize = Bool(False, config=True,
260 260 help="""Daemonize the ipcluster program. This implies --log-to-file.
261 261 Not available on Windows.
262 262 """)
263 263
264 264 def _daemonize_changed(self, name, old, new):
265 265 if new:
266 266 self.log_to_file = True
267 267
268 268 aliases = Dict(engine_aliases)
269 269 flags = Dict(engine_flags)
270 270 _stopping = False
271 271
272 272 def initialize(self, argv=None):
273 273 super(IPClusterEngines, self).initialize(argv)
274 274 self.init_signal()
275 275 self.init_launchers()
276 276
277 277 def init_launchers(self):
278 278 self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet')
279 279 self.engine_launcher.on_stop(lambda r: self.loop.stop())
280 280
281 281 def init_signal(self):
282 282 # Setup signals
283 283 signal.signal(signal.SIGINT, self.sigint_handler)
284 284
285 285 def build_launcher(self, clsname, kind=None):
286 286 """import and instantiate a Launcher based on importstring"""
287 287 if '.' not in clsname:
288 288 # not a module, presume it's the raw name in apps.launcher
289 289 if kind and kind not in clsname:
290 290 # doesn't match necessary full class name, assume it's
291 291 # just 'PBS' or 'MPIExec' prefix:
292 292 clsname = clsname + kind + 'Launcher'
293 293 clsname = 'IPython.parallel.apps.launcher.'+clsname
294 294 try:
295 295 klass = import_item(clsname)
296 296 except (ImportError, KeyError):
297 297 self.log.fatal("Could not import launcher class: %r"%clsname)
298 298 self.exit(1)
299 299
300 300 launcher = klass(
301 301 work_dir=u'.', config=self.config, log=self.log,
302 302 profile_dir=self.profile_dir.location, cluster_id=self.cluster_id,
303 303 )
304 304 return launcher
305 305
306 306 def start_engines(self):
307 307 self.log.info("Starting %i engines"%self.n)
308 308 self.engine_launcher.start(self.n)
309 309
310 310 def stop_engines(self):
311 311 self.log.info("Stopping Engines...")
312 312 if self.engine_launcher.running:
313 313 d = self.engine_launcher.stop()
314 314 return d
315 315 else:
316 316 return None
317 317
318 318 def stop_launchers(self, r=None):
319 319 if not self._stopping:
320 320 self._stopping = True
321 321 self.log.error("IPython cluster: stopping")
322 322 self.stop_engines()
323 323 # Wait a few seconds to let things shut down.
324 324 dc = ioloop.DelayedCallback(self.loop.stop, 4000, self.loop)
325 325 dc.start()
326 326
327 327 def sigint_handler(self, signum, frame):
328 328 self.log.debug("SIGINT received, stopping launchers...")
329 329 self.stop_launchers()
330 330
331 331 def start_logging(self):
332 332 # Remove old log files of the controller and engine
333 333 if self.clean_logs:
334 334 log_dir = self.profile_dir.log_dir
335 335 for f in os.listdir(log_dir):
336 336 if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f):
337 337 os.remove(os.path.join(log_dir, f))
338 338 # This will remove old log files for ipcluster itself
339 339 # super(IPBaseParallelApplication, self).start_logging()
340 340
341 341 def start(self):
342 342 """Start the app for the engines subcommand."""
343 343 self.log.info("IPython cluster: started")
344 344 # First see if the cluster is already running
345 345
346 346 # Now log and daemonize
347 347 self.log.info(
348 348 'Starting engines with [daemon=%r]' % self.daemonize
349 349 )
350 350 # TODO: Get daemonize working on Windows or as a Windows Server.
351 351 if self.daemonize:
352 352 if os.name=='posix':
353 353 daemonize()
354 354
355 355 dc = ioloop.DelayedCallback(self.start_engines, 0, self.loop)
356 356 dc.start()
357 357 # Now write the new pid file AFTER our new forked pid is active.
358 358 # self.write_pid_file()
359 359 try:
360 360 self.loop.start()
361 361 except KeyboardInterrupt:
362 362 pass
363 363 except zmq.ZMQError as e:
364 364 if e.errno == errno.EINTR:
365 365 pass
366 366 else:
367 367 raise
368 368
369 369 start_aliases = {}
370 370 start_aliases.update(engine_aliases)
371 371 start_aliases.update(dict(
372 372 delay='IPClusterStart.delay',
373 373 controller = 'IPClusterStart.controller_launcher_class',
374 374 ))
375 375 start_aliases['clean-logs'] = 'IPClusterStart.clean_logs'
376 376
377 # set inherited Start keys directly, to ensure command-line args get higher priority
378 # than config file options.
379 for key,value in start_aliases.items():
380 if value.startswith('IPClusterEngines'):
381 start_aliases[key] = value.replace('IPClusterEngines', 'IPClusterStart')
382
383 377 class IPClusterStart(IPClusterEngines):
384 378
385 379 name = u'ipcluster'
386 380 description = start_help
387 381 examples = _start_examples
388 382 default_log_level = logging.INFO
389 383 auto_create = Bool(True, config=True,
390 384 help="whether to create the profile_dir if it doesn't exist")
391 385 classes = List()
392 386 def _classes_default(self,):
393 387 from IPython.parallel.apps import launcher
394 388 return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers
395 389
396 390 clean_logs = Bool(True, config=True,
397 391 help="whether to cleanup old logs before starting")
398 392
399 393 delay = CFloat(1., config=True,
400 394 help="delay (in s) between starting the controller and the engines")
401 395
402 396 controller_launcher_class = DottedObjectName('LocalControllerLauncher',
403 397 config=True,
404 398 helep="""The class for launching a Controller. Change this value if you want
405 399 your controller to also be launched by a batch system, such as PBS,SGE,MPIExec,etc.
406 400
407 401 Each launcher class has its own set of configuration options, for making sure
408 402 it will work in your environment.
409 403
410 404 Examples include:
411 405
412 406 LocalControllerLauncher : start engines locally as subprocesses
413 407 MPIExecControllerLauncher : use mpiexec to launch engines in an MPI universe
414 408 PBSControllerLauncher : use PBS (qsub) to submit engines to a batch queue
415 409 SGEControllerLauncher : use SGE (qsub) to submit engines to a batch queue
416 410 SSHControllerLauncher : use SSH to start the controller
417 411 WindowsHPCControllerLauncher : use Windows HPC
418 412 """
419 413 )
420 414 reset = Bool(False, config=True,
421 415 help="Whether to reset config files as part of '--create'."
422 416 )
423 417
424 418 # flags = Dict(flags)
425 419 aliases = Dict(start_aliases)
426 420
427 421 def init_launchers(self):
428 422 self.controller_launcher = self.build_launcher(self.controller_launcher_class, 'Controller')
429 423 self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet')
430 424 self.controller_launcher.on_stop(self.stop_launchers)
431 425
432 426 def start_controller(self):
433 427 self.controller_launcher.start()
434 428
435 429 def stop_controller(self):
436 430 # self.log.info("In stop_controller")
437 431 if self.controller_launcher and self.controller_launcher.running:
438 432 return self.controller_launcher.stop()
439 433
440 434 def stop_launchers(self, r=None):
441 435 if not self._stopping:
442 436 self.stop_controller()
443 437 super(IPClusterStart, self).stop_launchers()
444 438
445 439 def start(self):
446 440 """Start the app for the start subcommand."""
447 441 # First see if the cluster is already running
448 442 try:
449 443 pid = self.get_pid_from_file()
450 444 except PIDFileError:
451 445 pass
452 446 else:
453 447 if self.check_pid(pid):
454 448 self.log.critical(
455 449 'Cluster is already running with [pid=%s]. '
456 450 'use "ipcluster stop" to stop the cluster.' % pid
457 451 )
458 452 # Here I exit with a unusual exit status that other processes
459 453 # can watch for to learn how I existed.
460 454 self.exit(ALREADY_STARTED)
461 455 else:
462 456 self.remove_pid_file()
463 457
464 458
465 459 # Now log and daemonize
466 460 self.log.info(
467 461 'Starting ipcluster with [daemon=%r]' % self.daemonize
468 462 )
469 463 # TODO: Get daemonize working on Windows or as a Windows Server.
470 464 if self.daemonize:
471 465 if os.name=='posix':
472 466 daemonize()
473 467
474 468 dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop)
475 469 dc.start()
476 470 dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop)
477 471 dc.start()
478 472 # Now write the new pid file AFTER our new forked pid is active.
479 473 self.write_pid_file()
480 474 try:
481 475 self.loop.start()
482 476 except KeyboardInterrupt:
483 477 pass
484 478 except zmq.ZMQError as e:
485 479 if e.errno == errno.EINTR:
486 480 pass
487 481 else:
488 482 raise
489 483 finally:
490 484 self.remove_pid_file()
491 485
492 486 base='IPython.parallel.apps.ipclusterapp.IPCluster'
493 487
494 488 class IPClusterApp(Application):
495 489 name = u'ipcluster'
496 490 description = _description
497 491 examples = _main_examples
498 492
499 493 subcommands = {
500 494 'start' : (base+'Start', start_help),
501 495 'stop' : (base+'Stop', stop_help),
502 496 'engines' : (base+'Engines', engines_help),
503 497 }
504 498
505 499 # no aliases or flags for parent App
506 500 aliases = Dict()
507 501 flags = Dict()
508 502
509 503 def start(self):
510 504 if self.subapp is None:
511 505 print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
512 506 print
513 507 self.print_description()
514 508 self.print_subcommands()
515 509 self.exit(1)
516 510 else:
517 511 return self.subapp.start()
518 512
519 513 def launch_new_instance():
520 514 """Create and run the IPython cluster."""
521 515 app = IPClusterApp.instance()
522 516 app.initialize()
523 517 app.start()
524 518
525 519
526 520 if __name__ == '__main__':
527 521 launch_new_instance()
528 522
General Comments 0
You need to be logged in to leave comments. Login now