##// END OF EJS Templates
aliases match flag pattern ('-' as wordsep, not '_')...
MinRK -
Show More
@@ -1,404 +1,404 b''
1 1 # encoding: utf-8
2 2 """
3 3 A base class for a configurable application.
4 4
5 5 Authors:
6 6
7 7 * Brian Granger
8 8 * Min RK
9 9 """
10 10
11 11 #-----------------------------------------------------------------------------
12 12 # Copyright (C) 2008-2011 The IPython Development Team
13 13 #
14 14 # Distributed under the terms of the BSD License. The full license is in
15 15 # the file COPYING, distributed as part of this software.
16 16 #-----------------------------------------------------------------------------
17 17
18 18 #-----------------------------------------------------------------------------
19 19 # Imports
20 20 #-----------------------------------------------------------------------------
21 21
22 22 import logging
23 23 import os
24 24 import re
25 25 import sys
26 26 from copy import deepcopy
27 27
28 28 from IPython.config.configurable import SingletonConfigurable
29 29 from IPython.config.loader import (
30 30 KeyValueConfigLoader, PyFileConfigLoader, Config, ArgumentError
31 31 )
32 32
33 33 from IPython.utils.traitlets import (
34 34 Unicode, List, Int, Enum, Dict, Instance, TraitError
35 35 )
36 36 from IPython.utils.importstring import import_item
37 37 from IPython.utils.text import indent, wrap_paragraphs, dedent
38 38
39 39 #-----------------------------------------------------------------------------
40 40 # function for re-wrapping a helpstring
41 41 #-----------------------------------------------------------------------------
42 42
43 43 #-----------------------------------------------------------------------------
44 44 # Descriptions for the various sections
45 45 #-----------------------------------------------------------------------------
46 46
47 47 # merge flags&aliases into options
48 48 option_description = """
49 49 IPython command-line arguments are passed as '--<flag>', or '--<name>=<value>'.
50 50
51 51 Arguments that take values are actually aliases to full Configurables, whose
52 52 aliases are listed on the help line. For more information on full
53 53 configurables, see '--help-all'.
54 54 """.strip() # trim newlines of front and back
55 55
56 56 keyvalue_description = """
57 57 Parameters are set from command-line arguments of the form:
58 58 `--Class.trait=value`.
59 59 This line is evaluated in Python, so simple expressions are allowed, e.g.::
60 60 `--C.a='range(3)'` For setting C.a=[0,1,2].
61 61 """.strip() # trim newlines of front and back
62 62
63 63 subcommand_description = """
64 64 Subcommands are launched as `{app} cmd [args]`. For information on using
65 65 subcommand 'cmd', do: `{app} cmd -h`.
66 66 """.strip().format(app=os.path.basename(sys.argv[0]))
67 67 # get running program name
68 68
69 69 #-----------------------------------------------------------------------------
70 70 # Application class
71 71 #-----------------------------------------------------------------------------
72 72
73 73
74 74 class ApplicationError(Exception):
75 75 pass
76 76
77 77
78 78 class Application(SingletonConfigurable):
79 79 """A singleton application with full configuration support."""
80 80
81 81 # The name of the application, will usually match the name of the command
82 82 # line application
83 83 name = Unicode(u'application')
84 84
85 85 # The description of the application that is printed at the beginning
86 86 # of the help.
87 87 description = Unicode(u'This is an application.')
88 88 # default section descriptions
89 89 option_description = Unicode(option_description)
90 90 keyvalue_description = Unicode(keyvalue_description)
91 91 subcommand_description = Unicode(subcommand_description)
92 92
93 93
94 94 # A sequence of Configurable subclasses whose config=True attributes will
95 95 # be exposed at the command line.
96 96 classes = List([])
97 97
98 98 # The version string of this application.
99 99 version = Unicode(u'0.0')
100 100
101 101 # The log level for the application
102 102 log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
103 103 default_value=logging.WARN,
104 104 config=True,
105 105 help="Set the log level by value or name.")
106 106 def _log_level_changed(self, name, old, new):
107 107 """Adjust the log level when log_level is set."""
108 108 if isinstance(new, basestring):
109 109 new = getattr(logging, new)
110 110 self.log_level = new
111 111 self.log.setLevel(new)
112 112
113 113 # the alias map for configurables
114 aliases = Dict(dict(log_level='Application.log_level'))
114 aliases = Dict({'log-level' : 'Application.log_level'})
115 115
116 116 # flags for loading Configurables or store_const style flags
117 117 # flags are loaded from this dict by '--key' flags
118 118 # this must be a dict of two-tuples, the first element being the Config/dict
119 119 # and the second being the help string for the flag
120 120 flags = Dict()
121 121 def _flags_changed(self, name, old, new):
122 122 """ensure flags dict is valid"""
123 123 for key,value in new.iteritems():
124 124 assert len(value) == 2, "Bad flag: %r:%s"%(key,value)
125 125 assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value)
126 126 assert isinstance(value[1], basestring), "Bad flag: %r:%s"%(key,value)
127 127
128 128
129 129 # subcommands for launching other applications
130 130 # if this is not empty, this will be a parent Application
131 131 # this must be a dict of two-tuples,
132 132 # the first element being the application class/import string
133 133 # and the second being the help string for the subcommand
134 134 subcommands = Dict()
135 135 # parse_command_line will initialize a subapp, if requested
136 136 subapp = Instance('IPython.config.application.Application', allow_none=True)
137 137
138 138 # extra command-line arguments that don't set config values
139 139 extra_args = List(Unicode)
140 140
141 141
142 142 def __init__(self, **kwargs):
143 143 SingletonConfigurable.__init__(self, **kwargs)
144 144 # Add my class to self.classes so my attributes appear in command line
145 145 # options.
146 146 self.classes.insert(0, self.__class__)
147 147
148 148 self.init_logging()
149 149
150 150 def _config_changed(self, name, old, new):
151 151 SingletonConfigurable._config_changed(self, name, old, new)
152 152 self.log.debug('Config changed:')
153 153 self.log.debug(repr(new))
154 154
155 155 def init_logging(self):
156 156 """Start logging for this application.
157 157
158 158 The default is to log to stdout using a StreaHandler. The log level
159 159 starts at loggin.WARN, but this can be adjusted by setting the
160 160 ``log_level`` attribute.
161 161 """
162 162 self.log = logging.getLogger(self.__class__.__name__)
163 163 self.log.setLevel(self.log_level)
164 164 if sys.executable.endswith('pythonw.exe'):
165 165 # this should really go to a file, but file-logging is only
166 166 # hooked up in parallel applications
167 167 self._log_handler = logging.StreamHandler(open(os.devnull, 'w'))
168 168 else:
169 169 self._log_handler = logging.StreamHandler()
170 170 self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
171 171 self._log_handler.setFormatter(self._log_formatter)
172 172 self.log.addHandler(self._log_handler)
173 173
174 174 def initialize(self, argv=None):
175 175 """Do the basic steps to configure me.
176 176
177 177 Override in subclasses.
178 178 """
179 179 self.parse_command_line(argv)
180 180
181 181
182 182 def start(self):
183 183 """Start the app mainloop.
184 184
185 185 Override in subclasses.
186 186 """
187 187 if self.subapp is not None:
188 188 return self.subapp.start()
189 189
190 190 def print_alias_help(self):
191 191 """Print the alias part of the help."""
192 192 if not self.aliases:
193 193 return
194 194
195 195 lines = []
196 196 classdict = {}
197 197 for cls in self.classes:
198 198 # include all parents (up to, but excluding Configurable) in available names
199 199 for c in cls.mro()[:-3]:
200 200 classdict[c.__name__] = c
201 201
202 202 for alias, longname in self.aliases.iteritems():
203 203 classname, traitname = longname.split('.',1)
204 204 cls = classdict[classname]
205 205
206 206 trait = cls.class_traits(config=True)[traitname]
207 207 help = cls.class_get_trait_help(trait).splitlines()
208 208 # reformat first line
209 209 help[0] = help[0].replace(longname, alias) + ' (%s)'%longname
210 210 lines.extend(help)
211 211 # lines.append('')
212 212 print os.linesep.join(lines)
213 213
214 214 def print_flag_help(self):
215 215 """Print the flag part of the help."""
216 216 if not self.flags:
217 217 return
218 218
219 219 lines = []
220 220 for m, (cfg,help) in self.flags.iteritems():
221 221 lines.append('--'+m)
222 222 lines.append(indent(dedent(help.strip())))
223 223 # lines.append('')
224 224 print os.linesep.join(lines)
225 225
226 226 def print_options(self):
227 227 if not self.flags and not self.aliases:
228 228 return
229 229 lines = ['Options']
230 230 lines.append('-'*len(lines[0]))
231 231 lines.append('')
232 232 for p in wrap_paragraphs(self.option_description):
233 233 lines.append(p)
234 234 lines.append('')
235 235 print os.linesep.join(lines)
236 236 self.print_flag_help()
237 237 self.print_alias_help()
238 238 print
239 239
240 240 def print_subcommands(self):
241 241 """Print the subcommand part of the help."""
242 242 if not self.subcommands:
243 243 return
244 244
245 245 lines = ["Subcommands"]
246 246 lines.append('-'*len(lines[0]))
247 247 lines.append('')
248 248 for p in wrap_paragraphs(self.subcommand_description):
249 249 lines.append(p)
250 250 lines.append('')
251 251 for subc, (cls,help) in self.subcommands.iteritems():
252 252 lines.append("%s : %s"%(subc, cls))
253 253 if help:
254 254 lines.append(indent(dedent(help.strip())))
255 255 lines.append('')
256 256 print os.linesep.join(lines)
257 257
258 258 def print_help(self, classes=False):
259 259 """Print the help for each Configurable class in self.classes.
260 260
261 261 If classes=False (the default), only flags and aliases are printed.
262 262 """
263 263 self.print_subcommands()
264 264 self.print_options()
265 265
266 266 if classes:
267 267 if self.classes:
268 268 print "Class parameters"
269 269 print "----------------"
270 270 print
271 271 for p in wrap_paragraphs(self.keyvalue_description):
272 272 print p
273 273 print
274 274
275 275 for cls in self.classes:
276 276 cls.class_print_help()
277 277 print
278 278 else:
279 279 print "To see all available configurables, use `--help-all`"
280 280 print
281 281
282 282 def print_description(self):
283 283 """Print the application description."""
284 284 for p in wrap_paragraphs(self.description):
285 285 print p
286 286 print
287 287
288 288 def print_version(self):
289 289 """Print the version string."""
290 290 print self.version
291 291
292 292 def update_config(self, config):
293 293 """Fire the traits events when the config is updated."""
294 294 # Save a copy of the current config.
295 295 newconfig = deepcopy(self.config)
296 296 # Merge the new config into the current one.
297 297 newconfig._merge(config)
298 298 # Save the combined config as self.config, which triggers the traits
299 299 # events.
300 300 self.config = newconfig
301 301
302 302 def initialize_subcommand(self, subc, argv=None):
303 303 """Initialize a subcommand with argv."""
304 304 subapp,help = self.subcommands.get(subc)
305 305
306 306 if isinstance(subapp, basestring):
307 307 subapp = import_item(subapp)
308 308
309 309 # clear existing instances
310 310 self.__class__.clear_instance()
311 311 # instantiate
312 312 self.subapp = subapp.instance()
313 313 # and initialize subapp
314 314 self.subapp.initialize(argv)
315 315
316 316 def parse_command_line(self, argv=None):
317 317 """Parse the command line arguments."""
318 318 argv = sys.argv[1:] if argv is None else argv
319 319
320 320 if self.subcommands and len(argv) > 0:
321 321 # we have subcommands, and one may have been specified
322 322 subc, subargv = argv[0], argv[1:]
323 323 if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands:
324 324 # it's a subcommand, and *not* a flag or class parameter
325 325 return self.initialize_subcommand(subc, subargv)
326 326
327 327 if '-h' in argv or '--help' in argv or '--help-all' in argv:
328 328 self.print_description()
329 329 self.print_help('--help-all' in argv)
330 330 self.exit(0)
331 331
332 332 if '--version' in argv:
333 333 self.print_version()
334 334 self.exit(0)
335 335
336 336 loader = KeyValueConfigLoader(argv=argv, aliases=self.aliases,
337 337 flags=self.flags)
338 338 try:
339 339 config = loader.load_config()
340 340 self.update_config(config)
341 341 except (TraitError, ArgumentError) as e:
342 342 self.print_description()
343 343 self.print_help()
344 344 self.log.fatal(str(e))
345 345 self.exit(1)
346 346 # store unparsed args in extra_args
347 347 self.extra_args = loader.extra_args
348 348
349 349 def load_config_file(self, filename, path=None):
350 350 """Load a .py based config file by filename and path."""
351 351 loader = PyFileConfigLoader(filename, path=path)
352 352 config = loader.load_config()
353 353 self.update_config(config)
354 354
355 355 def generate_config_file(self):
356 356 """generate default config file from Configurables"""
357 357 lines = ["# Configuration file for %s."%self.name]
358 358 lines.append('')
359 359 lines.append('c = get_config()')
360 360 lines.append('')
361 361 for cls in self.classes:
362 362 lines.append(cls.class_config_section())
363 363 return '\n'.join(lines)
364 364
365 365 def exit(self, exit_status=0):
366 366 self.log.debug("Exiting application: %s" % self.name)
367 367 sys.exit(exit_status)
368 368
369 369 #-----------------------------------------------------------------------------
370 370 # utility functions, for convenience
371 371 #-----------------------------------------------------------------------------
372 372
373 373 def boolean_flag(name, configurable, set_help='', unset_help=''):
374 374 """Helper for building basic --trait, --no-trait flags.
375 375
376 376 Parameters
377 377 ----------
378 378
379 379 name : str
380 380 The name of the flag.
381 381 configurable : str
382 382 The 'Class.trait' string of the trait to be set/unset with the flag
383 383 set_help : unicode
384 384 help string for --name flag
385 385 unset_help : unicode
386 386 help string for --no-name flag
387 387
388 388 Returns
389 389 -------
390 390
391 391 cfg : dict
392 392 A dict with two keys: 'name', and 'no-name', for setting and unsetting
393 393 the trait, respectively.
394 394 """
395 395 # default helpstrings
396 396 set_help = set_help or "set %s=True"%configurable
397 397 unset_help = unset_help or "set %s=False"%configurable
398 398
399 399 cls,trait = configurable.split('.')
400 400
401 401 setter = {cls : {trait : True}}
402 402 unsetter = {cls : {trait : False}}
403 403 return {name : (setter, set_help), 'no-'+name : (unsetter, unset_help)}
404 404
@@ -1,569 +1,588 b''
1 1 """A simple configuration system.
2 2
3 3 Authors
4 4 -------
5 5 * Brian Granger
6 6 * Fernando Perez
7 7 * Min RK
8 8 """
9 9
10 10 #-----------------------------------------------------------------------------
11 11 # Copyright (C) 2008-2011 The IPython Development Team
12 12 #
13 13 # Distributed under the terms of the BSD License. The full license is in
14 14 # the file COPYING, distributed as part of this software.
15 15 #-----------------------------------------------------------------------------
16 16
17 17 #-----------------------------------------------------------------------------
18 18 # Imports
19 19 #-----------------------------------------------------------------------------
20 20
21 21 import __builtin__
22 22 import re
23 23 import sys
24 24
25 25 from IPython.external import argparse
26 26 from IPython.utils.path import filefind, get_ipython_dir
27 from IPython.utils import warn
27 28
28 29 #-----------------------------------------------------------------------------
29 30 # Exceptions
30 31 #-----------------------------------------------------------------------------
31 32
32 33
33 34 class ConfigError(Exception):
34 35 pass
35 36
36 37
37 38 class ConfigLoaderError(ConfigError):
38 39 pass
39 40
40 41 class ArgumentError(ConfigLoaderError):
41 42 pass
42 43
43 44 #-----------------------------------------------------------------------------
44 45 # Argparse fix
45 46 #-----------------------------------------------------------------------------
46 47
47 48 # Unfortunately argparse by default prints help messages to stderr instead of
48 49 # stdout. This makes it annoying to capture long help screens at the command
49 50 # line, since one must know how to pipe stderr, which many users don't know how
50 51 # to do. So we override the print_help method with one that defaults to
51 52 # stdout and use our class instead.
52 53
53 54 class ArgumentParser(argparse.ArgumentParser):
54 55 """Simple argparse subclass that prints help to stdout by default."""
55 56
56 57 def print_help(self, file=None):
57 58 if file is None:
58 59 file = sys.stdout
59 60 return super(ArgumentParser, self).print_help(file)
60 61
61 62 print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
62 63
63 64 #-----------------------------------------------------------------------------
64 65 # Config class for holding config information
65 66 #-----------------------------------------------------------------------------
66 67
67 68
68 69 class Config(dict):
69 70 """An attribute based dict that can do smart merges."""
70 71
71 72 def __init__(self, *args, **kwds):
72 73 dict.__init__(self, *args, **kwds)
73 74 # This sets self.__dict__ = self, but it has to be done this way
74 75 # because we are also overriding __setattr__.
75 76 dict.__setattr__(self, '__dict__', self)
76 77
77 78 def _merge(self, other):
78 79 to_update = {}
79 80 for k, v in other.iteritems():
80 81 if not self.has_key(k):
81 82 to_update[k] = v
82 83 else: # I have this key
83 84 if isinstance(v, Config):
84 85 # Recursively merge common sub Configs
85 86 self[k]._merge(v)
86 87 else:
87 88 # Plain updates for non-Configs
88 89 to_update[k] = v
89 90
90 91 self.update(to_update)
91 92
92 93 def _is_section_key(self, key):
93 94 if key[0].upper()==key[0] and not key.startswith('_'):
94 95 return True
95 96 else:
96 97 return False
97 98
98 99 def __contains__(self, key):
99 100 if self._is_section_key(key):
100 101 return True
101 102 else:
102 103 return super(Config, self).__contains__(key)
103 104 # .has_key is deprecated for dictionaries.
104 105 has_key = __contains__
105 106
106 107 def _has_section(self, key):
107 108 if self._is_section_key(key):
108 109 if super(Config, self).__contains__(key):
109 110 return True
110 111 return False
111 112
112 113 def copy(self):
113 114 return type(self)(dict.copy(self))
114 115
115 116 def __copy__(self):
116 117 return self.copy()
117 118
118 119 def __deepcopy__(self, memo):
119 120 import copy
120 121 return type(self)(copy.deepcopy(self.items()))
121 122
122 123 def __getitem__(self, key):
123 124 # We cannot use directly self._is_section_key, because it triggers
124 125 # infinite recursion on top of PyPy. Instead, we manually fish the
125 126 # bound method.
126 127 is_section_key = self.__class__._is_section_key.__get__(self)
127 128
128 129 # Because we use this for an exec namespace, we need to delegate
129 130 # the lookup of names in __builtin__ to itself. This means
130 131 # that you can't have section or attribute names that are
131 132 # builtins.
132 133 try:
133 134 return getattr(__builtin__, key)
134 135 except AttributeError:
135 136 pass
136 137 if is_section_key(key):
137 138 try:
138 139 return dict.__getitem__(self, key)
139 140 except KeyError:
140 141 c = Config()
141 142 dict.__setitem__(self, key, c)
142 143 return c
143 144 else:
144 145 return dict.__getitem__(self, key)
145 146
146 147 def __setitem__(self, key, value):
147 148 # Don't allow names in __builtin__ to be modified.
148 149 if hasattr(__builtin__, key):
149 150 raise ConfigError('Config variable names cannot have the same name '
150 151 'as a Python builtin: %s' % key)
151 152 if self._is_section_key(key):
152 153 if not isinstance(value, Config):
153 154 raise ValueError('values whose keys begin with an uppercase '
154 155 'char must be Config instances: %r, %r' % (key, value))
155 156 else:
156 157 dict.__setitem__(self, key, value)
157 158
158 159 def __getattr__(self, key):
159 160 try:
160 161 return self.__getitem__(key)
161 162 except KeyError, e:
162 163 raise AttributeError(e)
163 164
164 165 def __setattr__(self, key, value):
165 166 try:
166 167 self.__setitem__(key, value)
167 168 except KeyError, e:
168 169 raise AttributeError(e)
169 170
170 171 def __delattr__(self, key):
171 172 try:
172 173 dict.__delitem__(self, key)
173 174 except KeyError, e:
174 175 raise AttributeError(e)
175 176
176 177
177 178 #-----------------------------------------------------------------------------
178 179 # Config loading classes
179 180 #-----------------------------------------------------------------------------
180 181
181 182
182 183 class ConfigLoader(object):
183 184 """A object for loading configurations from just about anywhere.
184 185
185 186 The resulting configuration is packaged as a :class:`Struct`.
186 187
187 188 Notes
188 189 -----
189 190 A :class:`ConfigLoader` does one thing: load a config from a source
190 191 (file, command line arguments) and returns the data as a :class:`Struct`.
191 192 There are lots of things that :class:`ConfigLoader` does not do. It does
192 193 not implement complex logic for finding config files. It does not handle
193 194 default values or merge multiple configs. These things need to be
194 195 handled elsewhere.
195 196 """
196 197
197 198 def __init__(self):
198 199 """A base class for config loaders.
199 200
200 201 Examples
201 202 --------
202 203
203 204 >>> cl = ConfigLoader()
204 205 >>> config = cl.load_config()
205 206 >>> config
206 207 {}
207 208 """
208 209 self.clear()
209 210
210 211 def clear(self):
211 212 self.config = Config()
212 213
213 214 def load_config(self):
214 215 """Load a config from somewhere, return a :class:`Config` instance.
215 216
216 217 Usually, this will cause self.config to be set and then returned.
217 218 However, in most cases, :meth:`ConfigLoader.clear` should be called
218 219 to erase any previous state.
219 220 """
220 221 self.clear()
221 222 return self.config
222 223
223 224
224 225 class FileConfigLoader(ConfigLoader):
225 226 """A base class for file based configurations.
226 227
227 228 As we add more file based config loaders, the common logic should go
228 229 here.
229 230 """
230 231 pass
231 232
232 233
233 234 class PyFileConfigLoader(FileConfigLoader):
234 235 """A config loader for pure python files.
235 236
236 237 This calls execfile on a plain python file and looks for attributes
237 238 that are all caps. These attribute are added to the config Struct.
238 239 """
239 240
240 241 def __init__(self, filename, path=None):
241 242 """Build a config loader for a filename and path.
242 243
243 244 Parameters
244 245 ----------
245 246 filename : str
246 247 The file name of the config file.
247 248 path : str, list, tuple
248 249 The path to search for the config file on, or a sequence of
249 250 paths to try in order.
250 251 """
251 252 super(PyFileConfigLoader, self).__init__()
252 253 self.filename = filename
253 254 self.path = path
254 255 self.full_filename = ''
255 256 self.data = None
256 257
257 258 def load_config(self):
258 259 """Load the config from a file and return it as a Struct."""
259 260 self.clear()
260 261 self._find_file()
261 262 self._read_file_as_dict()
262 263 self._convert_to_config()
263 264 return self.config
264 265
265 266 def _find_file(self):
266 267 """Try to find the file by searching the paths."""
267 268 self.full_filename = filefind(self.filename, self.path)
268 269
269 270 def _read_file_as_dict(self):
270 271 """Load the config file into self.config, with recursive loading."""
271 272 # This closure is made available in the namespace that is used
272 273 # to exec the config file. It allows users to call
273 274 # load_subconfig('myconfig.py') to load config files recursively.
274 275 # It needs to be a closure because it has references to self.path
275 276 # and self.config. The sub-config is loaded with the same path
276 277 # as the parent, but it uses an empty config which is then merged
277 278 # with the parents.
278 279
279 280 # If a profile is specified, the config file will be loaded
280 281 # from that profile
281 282
282 283 def load_subconfig(fname, profile=None):
283 284 # import here to prevent circular imports
284 285 from IPython.core.profiledir import ProfileDir, ProfileDirError
285 286 if profile is not None:
286 287 try:
287 288 profile_dir = ProfileDir.find_profile_dir_by_name(
288 289 get_ipython_dir(),
289 290 profile,
290 291 )
291 292 except ProfileDirError:
292 293 return
293 294 path = profile_dir.location
294 295 else:
295 296 path = self.path
296 297 loader = PyFileConfigLoader(fname, path)
297 298 try:
298 299 sub_config = loader.load_config()
299 300 except IOError:
300 301 # Pass silently if the sub config is not there. This happens
301 302 # when a user s using a profile, but not the default config.
302 303 pass
303 304 else:
304 305 self.config._merge(sub_config)
305 306
306 307 # Again, this needs to be a closure and should be used in config
307 308 # files to get the config being loaded.
308 309 def get_config():
309 310 return self.config
310 311
311 312 namespace = dict(load_subconfig=load_subconfig, get_config=get_config)
312 313 fs_encoding = sys.getfilesystemencoding() or 'ascii'
313 314 conf_filename = self.full_filename.encode(fs_encoding)
314 315 execfile(conf_filename, namespace)
315 316
316 317 def _convert_to_config(self):
317 318 if self.data is None:
318 319 ConfigLoaderError('self.data does not exist')
319 320
320 321
321 322 class CommandLineConfigLoader(ConfigLoader):
322 323 """A config loader for command line arguments.
323 324
324 325 As we add more command line based loaders, the common logic should go
325 326 here.
326 327 """
327 328
328 kv_pattern = re.compile(r'\-\-[A-Za-z]\w*(\.\w+)*\=.*')
329 # raw --identifier=value pattern
330 # but *also* accept '-' as wordsep, for aliases
331 # accepts: --foo=a
332 # --Class.trait=value
333 # --alias-name=value
334 # rejects: -foo=value
335 # --foo
336 # --Class.trait
337 kv_pattern = re.compile(r'\-\-[A-Za-z][\w\-]*(\.[\w\-]+)*\=.*')
338
339 # just flags, no assignments, with two *or one* leading '-'
340 # accepts: --foo
341 # -foo-bar-again
342 # rejects: --anything=anything
343 # --two.word
344
329 345 flag_pattern = re.compile(r'\-\-?\w+[\-\w]*$')
330 346
331 347 class KeyValueConfigLoader(CommandLineConfigLoader):
332 348 """A config loader that loads key value pairs from the command line.
333 349
334 350 This allows command line options to be gives in the following form::
335 351
336 352 ipython --profile="foo" --InteractiveShell.autocall=False
337 353 """
338 354
339 355 def __init__(self, argv=None, aliases=None, flags=None):
340 356 """Create a key value pair config loader.
341 357
342 358 Parameters
343 359 ----------
344 360 argv : list
345 361 A list that has the form of sys.argv[1:] which has unicode
346 362 elements of the form u"key=value". If this is None (default),
347 363 then sys.argv[1:] will be used.
348 364 aliases : dict
349 365 A dict of aliases for configurable traits.
350 366 Keys are the short aliases, Values are the resolved trait.
351 367 Of the form: `{'alias' : 'Configurable.trait'}`
352 368 flags : dict
353 369 A dict of flags, keyed by str name. Vaues can be Config objects,
354 370 dicts, or "key=value" strings. If Config or dict, when the flag
355 371 is triggered, The flag is loaded as `self.config.update(m)`.
356 372
357 373 Returns
358 374 -------
359 375 config : Config
360 376 The resulting Config object.
361 377
362 378 Examples
363 379 --------
364 380
365 381 >>> from IPython.config.loader import KeyValueConfigLoader
366 382 >>> cl = KeyValueConfigLoader()
367 >>> cl.load_config(["--foo='bar'","--A.name='brian'","--B.number=0"])
368 {'A': {'name': 'brian'}, 'B': {'number': 0}, 'foo': 'bar'}
383 >>> cl.load_config(["--A.name='brian'","--B.number=0"])
384 {'A': {'name': 'brian'}, 'B': {'number': 0}}
369 385 """
370 386 self.clear()
371 387 if argv is None:
372 388 argv = sys.argv[1:]
373 389 self.argv = argv
374 390 self.aliases = aliases or {}
375 391 self.flags = flags or {}
376 392
377 393
378 394 def clear(self):
379 395 super(KeyValueConfigLoader, self).clear()
380 396 self.extra_args = []
381 397
382 398
383 399 def _decode_argv(self, argv, enc=None):
384 400 """decode argv if bytes, using stin.encoding, falling back on default enc"""
385 401 uargv = []
386 402 if enc is None:
387 403 enc = sys.stdin.encoding or sys.getdefaultencoding()
388 404 for arg in argv:
389 405 if not isinstance(arg, unicode):
390 406 # only decode if not already decoded
391 407 arg = arg.decode(enc)
392 408 uargv.append(arg)
393 409 return uargv
394 410
395 411
396 412 def load_config(self, argv=None, aliases=None, flags=None):
397 413 """Parse the configuration and generate the Config object.
398 414
399 415 After loading, any arguments that are not key-value or
400 416 flags will be stored in self.extra_args - a list of
401 417 unparsed command-line arguments. This is used for
402 418 arguments such as input files or subcommands.
403 419
404 420 Parameters
405 421 ----------
406 422 argv : list, optional
407 423 A list that has the form of sys.argv[1:] which has unicode
408 424 elements of the form u"key=value". If this is None (default),
409 425 then self.argv will be used.
410 426 aliases : dict
411 427 A dict of aliases for configurable traits.
412 428 Keys are the short aliases, Values are the resolved trait.
413 429 Of the form: `{'alias' : 'Configurable.trait'}`
414 430 flags : dict
415 431 A dict of flags, keyed by str name. Values can be Config objects
416 432 or dicts. When the flag is triggered, The config is loaded as
417 433 `self.config.update(cfg)`.
418 434 """
419 435 from IPython.config.configurable import Configurable
420 436
421 437 self.clear()
422 438 if argv is None:
423 439 argv = self.argv
424 440 if aliases is None:
425 441 aliases = self.aliases
426 442 if flags is None:
427 443 flags = self.flags
428 444
429 445 # ensure argv is a list of unicode strings:
430 446 uargv = self._decode_argv(argv)
431 447 for idx,raw in enumerate(uargv):
432 448 # strip leading '-'
433 449 item = raw.lstrip('-')
434 450
435 451 if raw == '--':
436 452 # don't parse arguments after '--'
437 453 # this is useful for relaying arguments to scripts, e.g.
438 454 # ipython -i foo.py --pylab=qt -- args after '--' go-to-foo.py
439 455 self.extra_args.extend(uargv[idx+1:])
440 456 break
441 457
442 458 if kv_pattern.match(raw):
443 459 lhs,rhs = item.split('=',1)
444 460 # Substitute longnames for aliases.
445 461 if lhs in aliases:
446 462 lhs = aliases[lhs]
463 if '.' not in lhs:
464 # probably a mistyped alias, but not technically illegal
465 warn.warn("Unrecognized alias: '%s', it will probably have no effect."%lhs)
447 466 exec_str = 'self.config.' + lhs + '=' + rhs
448 467 try:
449 468 # Try to see if regular Python syntax will work. This
450 469 # won't handle strings as the quote marks are removed
451 470 # by the system shell.
452 471 exec exec_str in locals(), globals()
453 472 except (NameError, SyntaxError):
454 473 # This case happens if the rhs is a string but without
455 474 # the quote marks. Use repr, to get quote marks, and
456 475 # 'u' prefix and see if
457 476 # it succeeds. If it still fails, we let it raise.
458 477 exec_str = u'self.config.' + lhs + '=' + repr(rhs)
459 478 exec exec_str in locals(), globals()
460 479 elif flag_pattern.match(raw):
461 480 if item in flags:
462 481 cfg,help = flags[item]
463 482 if isinstance(cfg, (dict, Config)):
464 483 # don't clobber whole config sections, update
465 484 # each section from config:
466 485 for sec,c in cfg.iteritems():
467 486 self.config[sec].update(c)
468 487 else:
469 488 raise ValueError("Invalid flag: '%s'"%raw)
470 489 else:
471 490 raise ArgumentError("Unrecognized flag: '%s'"%raw)
472 491 elif raw.startswith('-'):
473 492 kv = '--'+item
474 493 if kv_pattern.match(kv):
475 494 raise ArgumentError("Invalid argument: '%s', did you mean '%s'?"%(raw, kv))
476 495 else:
477 496 raise ArgumentError("Invalid argument: '%s'"%raw)
478 497 else:
479 498 # keep all args that aren't valid in a list,
480 499 # in case our parent knows what to do with them.
481 500 self.extra_args.append(item)
482 501 return self.config
483 502
484 503 class ArgParseConfigLoader(CommandLineConfigLoader):
485 504 """A loader that uses the argparse module to load from the command line."""
486 505
487 506 def __init__(self, argv=None, *parser_args, **parser_kw):
488 507 """Create a config loader for use with argparse.
489 508
490 509 Parameters
491 510 ----------
492 511
493 512 argv : optional, list
494 513 If given, used to read command-line arguments from, otherwise
495 514 sys.argv[1:] is used.
496 515
497 516 parser_args : tuple
498 517 A tuple of positional arguments that will be passed to the
499 518 constructor of :class:`argparse.ArgumentParser`.
500 519
501 520 parser_kw : dict
502 521 A tuple of keyword arguments that will be passed to the
503 522 constructor of :class:`argparse.ArgumentParser`.
504 523
505 524 Returns
506 525 -------
507 526 config : Config
508 527 The resulting Config object.
509 528 """
510 529 super(CommandLineConfigLoader, self).__init__()
511 530 if argv == None:
512 531 argv = sys.argv[1:]
513 532 self.argv = argv
514 533 self.parser_args = parser_args
515 534 self.version = parser_kw.pop("version", None)
516 535 kwargs = dict(argument_default=argparse.SUPPRESS)
517 536 kwargs.update(parser_kw)
518 537 self.parser_kw = kwargs
519 538
520 539 def load_config(self, argv=None):
521 540 """Parse command line arguments and return as a Config object.
522 541
523 542 Parameters
524 543 ----------
525 544
526 545 args : optional, list
527 546 If given, a list with the structure of sys.argv[1:] to parse
528 547 arguments from. If not given, the instance's self.argv attribute
529 548 (given at construction time) is used."""
530 549 self.clear()
531 550 if argv is None:
532 551 argv = self.argv
533 552 self._create_parser()
534 553 self._parse_args(argv)
535 554 self._convert_to_config()
536 555 return self.config
537 556
538 557 def get_extra_args(self):
539 558 if hasattr(self, 'extra_args'):
540 559 return self.extra_args
541 560 else:
542 561 return []
543 562
544 563 def _create_parser(self):
545 564 self.parser = ArgumentParser(*self.parser_args, **self.parser_kw)
546 565 self._add_arguments()
547 566
548 567 def _add_arguments(self):
549 568 raise NotImplementedError("subclasses must implement _add_arguments")
550 569
551 570 def _parse_args(self, args):
552 571 """self.parser->self.parsed_data"""
553 572 # decode sys.argv to support unicode command-line options
554 573 uargs = []
555 574 for a in args:
556 575 if isinstance(a, str):
557 576 # don't decode if we already got unicode
558 577 a = a.decode(sys.stdin.encoding or
559 578 sys.getdefaultencoding())
560 579 uargs.append(a)
561 580 self.parsed_data, self.extra_args = self.parser.parse_known_args(uargs)
562 581
563 582 def _convert_to_config(self):
564 583 """self.parsed_data->self.config"""
565 584 for k, v in vars(self.parsed_data).iteritems():
566 585 exec_str = 'self.config.' + k + '= v'
567 586 exec exec_str in locals(), globals()
568 587
569 588
@@ -1,141 +1,146 b''
1 1 """
2 2 Tests for IPython.config.application.Application
3 3
4 4 Authors:
5 5
6 6 * Brian Granger
7 7 """
8 8
9 9 #-----------------------------------------------------------------------------
10 10 # Copyright (C) 2008-2011 The IPython Development Team
11 11 #
12 12 # Distributed under the terms of the BSD License. The full license is in
13 13 # the file COPYING, distributed as part of this software.
14 14 #-----------------------------------------------------------------------------
15 15
16 16 #-----------------------------------------------------------------------------
17 17 # Imports
18 18 #-----------------------------------------------------------------------------
19 19
20 20 from unittest import TestCase
21 21
22 22 from IPython.config.configurable import Configurable
23 23
24 24 from IPython.config.application import (
25 25 Application
26 26 )
27 27
28 28 from IPython.utils.traitlets import (
29 29 Bool, Unicode, Int, Float, List, Dict
30 30 )
31 31
32 32 #-----------------------------------------------------------------------------
33 33 # Code
34 34 #-----------------------------------------------------------------------------
35 35
36 36 class Foo(Configurable):
37 37
38 38 i = Int(0, config=True, help="The integer i.")
39 39 j = Int(1, config=True, help="The integer j.")
40 40 name = Unicode(u'Brian', config=True, help="First name.")
41 41
42 42
43 43 class Bar(Configurable):
44 44
45 45 b = Int(0, config=True, help="The integer b.")
46 46 enabled = Bool(True, config=True, help="Enable bar.")
47 47
48 48
49 49 class MyApp(Application):
50 50
51 51 name = Unicode(u'myapp')
52 52 running = Bool(False, config=True,
53 53 help="Is the app running?")
54 54 classes = List([Bar, Foo])
55 55 config_file = Unicode(u'', config=True,
56 56 help="Load this config file")
57 57
58 aliases = Dict(dict(i='Foo.i',j='Foo.j',name='Foo.name',
59 enabled='Bar.enabled', log_level='MyApp.log_level'))
58 aliases = Dict({
59 'i' : 'Foo.i',
60 'j' : 'Foo.j',
61 'name' : 'Foo.name',
62 'enabled' : 'Bar.enabled',
63 'log-level' : 'MyApp.log_level',
64 })
60 65
61 66 flags = Dict(dict(enable=({'Bar': {'enabled' : True}}, "Set Bar.enabled to True"),
62 67 disable=({'Bar': {'enabled' : False}}, "Set Bar.enabled to False")))
63 68
64 69 def init_foo(self):
65 70 self.foo = Foo(config=self.config)
66 71
67 72 def init_bar(self):
68 73 self.bar = Bar(config=self.config)
69 74
70 75
71 76 class TestApplication(TestCase):
72 77
73 78 def test_basic(self):
74 79 app = MyApp()
75 80 self.assertEquals(app.name, u'myapp')
76 81 self.assertEquals(app.running, False)
77 82 self.assertEquals(app.classes, [MyApp,Bar,Foo])
78 83 self.assertEquals(app.config_file, u'')
79 84
80 85 def test_config(self):
81 86 app = MyApp()
82 app.parse_command_line(["--i=10","--Foo.j=10","--enabled=False","--log_level=50"])
87 app.parse_command_line(["--i=10","--Foo.j=10","--enabled=False","--log-level=50"])
83 88 config = app.config
84 89 self.assertEquals(config.Foo.i, 10)
85 90 self.assertEquals(config.Foo.j, 10)
86 91 self.assertEquals(config.Bar.enabled, False)
87 92 self.assertEquals(config.MyApp.log_level,50)
88 93
89 94 def test_config_propagation(self):
90 95 app = MyApp()
91 app.parse_command_line(["--i=10","--Foo.j=10","--enabled=False","--log_level=50"])
96 app.parse_command_line(["--i=10","--Foo.j=10","--enabled=False","--log-level=50"])
92 97 app.init_foo()
93 98 app.init_bar()
94 99 self.assertEquals(app.foo.i, 10)
95 100 self.assertEquals(app.foo.j, 10)
96 101 self.assertEquals(app.bar.enabled, False)
97 102
98 103 def test_flags(self):
99 104 app = MyApp()
100 105 app.parse_command_line(["--disable"])
101 106 app.init_bar()
102 107 self.assertEquals(app.bar.enabled, False)
103 108 app.parse_command_line(["--enable"])
104 109 app.init_bar()
105 110 self.assertEquals(app.bar.enabled, True)
106 111
107 112 def test_aliases(self):
108 113 app = MyApp()
109 114 app.parse_command_line(["--i=5", "--j=10"])
110 115 app.init_foo()
111 116 self.assertEquals(app.foo.i, 5)
112 117 app.init_foo()
113 118 self.assertEquals(app.foo.j, 10)
114 119
115 120 def test_flag_clobber(self):
116 121 """test that setting flags doesn't clobber existing settings"""
117 122 app = MyApp()
118 123 app.parse_command_line(["--Bar.b=5", "--disable"])
119 124 app.init_bar()
120 125 self.assertEquals(app.bar.enabled, False)
121 126 self.assertEquals(app.bar.b, 5)
122 127 app.parse_command_line(["--enable", "--Bar.b=10"])
123 128 app.init_bar()
124 129 self.assertEquals(app.bar.enabled, True)
125 130 self.assertEquals(app.bar.b, 10)
126 131
127 132 def test_extra_args(self):
128 133 app = MyApp()
129 134 app.parse_command_line(["--Bar.b=5", 'extra', "--disable", 'args'])
130 135 app.init_bar()
131 136 self.assertEquals(app.bar.enabled, False)
132 137 self.assertEquals(app.bar.b, 5)
133 138 self.assertEquals(app.extra_args, ['extra', 'args'])
134 139 app = MyApp()
135 140 app.parse_command_line(["--Bar.b=5", '--', 'extra', "--disable", 'args'])
136 141 app.init_bar()
137 142 self.assertEquals(app.bar.enabled, True)
138 143 self.assertEquals(app.bar.b, 5)
139 144 self.assertEquals(app.extra_args, ['extra', '--disable', 'args'])
140 145
141 146
@@ -1,219 +1,226 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 Tests for IPython.config.loader
5 5
6 6 Authors:
7 7
8 8 * Brian Granger
9 9 * Fernando Perez (design help)
10 10 """
11 11
12 12 #-----------------------------------------------------------------------------
13 13 # Copyright (C) 2008-2009 The IPython Development Team
14 14 #
15 15 # Distributed under the terms of the BSD License. The full license is in
16 16 # the file COPYING, distributed as part of this software.
17 17 #-----------------------------------------------------------------------------
18 18
19 19 #-----------------------------------------------------------------------------
20 20 # Imports
21 21 #-----------------------------------------------------------------------------
22 22
23 23 import os
24 24 import sys
25 25 from tempfile import mkstemp
26 26 from unittest import TestCase
27 27
28 28 from nose import SkipTest
29 29
30 from IPython.testing.tools import mute_warn
31
30 32 from IPython.utils.traitlets import Int, Unicode
31 33 from IPython.config.configurable import Configurable
32 34 from IPython.config.loader import (
33 35 Config,
34 36 PyFileConfigLoader,
35 37 KeyValueConfigLoader,
36 38 ArgParseConfigLoader,
37 39 ConfigError
38 40 )
39 41
40 42 #-----------------------------------------------------------------------------
41 43 # Actual tests
42 44 #-----------------------------------------------------------------------------
43 45
44 46
45 47 pyfile = """
46 48 c = get_config()
47 49 c.a=10
48 50 c.b=20
49 51 c.Foo.Bar.value=10
50 52 c.Foo.Bam.value=range(10)
51 53 c.D.C.value='hi there'
52 54 """
53 55
54 56 class TestPyFileCL(TestCase):
55 57
56 58 def test_basic(self):
57 59 fd, fname = mkstemp('.py')
58 60 f = os.fdopen(fd, 'w')
59 61 f.write(pyfile)
60 62 f.close()
61 63 # Unlink the file
62 64 cl = PyFileConfigLoader(fname)
63 65 config = cl.load_config()
64 66 self.assertEquals(config.a, 10)
65 67 self.assertEquals(config.b, 20)
66 68 self.assertEquals(config.Foo.Bar.value, 10)
67 69 self.assertEquals(config.Foo.Bam.value, range(10))
68 70 self.assertEquals(config.D.C.value, 'hi there')
69 71
70 72 class MyLoader1(ArgParseConfigLoader):
71 73 def _add_arguments(self):
72 74 p = self.parser
73 75 p.add_argument('-f', '--foo', dest='Global.foo', type=str)
74 76 p.add_argument('-b', dest='MyClass.bar', type=int)
75 77 p.add_argument('-n', dest='n', action='store_true')
76 78 p.add_argument('Global.bam', type=str)
77 79
78 80 class MyLoader2(ArgParseConfigLoader):
79 81 def _add_arguments(self):
80 82 subparsers = self.parser.add_subparsers(dest='subparser_name')
81 83 subparser1 = subparsers.add_parser('1')
82 84 subparser1.add_argument('-x',dest='Global.x')
83 85 subparser2 = subparsers.add_parser('2')
84 86 subparser2.add_argument('y')
85 87
86 88 class TestArgParseCL(TestCase):
87 89
88 90 def test_basic(self):
89 91 cl = MyLoader1()
90 92 config = cl.load_config('-f hi -b 10 -n wow'.split())
91 93 self.assertEquals(config.Global.foo, 'hi')
92 94 self.assertEquals(config.MyClass.bar, 10)
93 95 self.assertEquals(config.n, True)
94 96 self.assertEquals(config.Global.bam, 'wow')
95 97 config = cl.load_config(['wow'])
96 98 self.assertEquals(config.keys(), ['Global'])
97 99 self.assertEquals(config.Global.keys(), ['bam'])
98 100 self.assertEquals(config.Global.bam, 'wow')
99 101
100 102 def test_add_arguments(self):
101 103 cl = MyLoader2()
102 104 config = cl.load_config('2 frobble'.split())
103 105 self.assertEquals(config.subparser_name, '2')
104 106 self.assertEquals(config.y, 'frobble')
105 107 config = cl.load_config('1 -x frobble'.split())
106 108 self.assertEquals(config.subparser_name, '1')
107 109 self.assertEquals(config.Global.x, 'frobble')
108 110
109 111 def test_argv(self):
110 112 cl = MyLoader1(argv='-f hi -b 10 -n wow'.split())
111 113 config = cl.load_config()
112 114 self.assertEquals(config.Global.foo, 'hi')
113 115 self.assertEquals(config.MyClass.bar, 10)
114 116 self.assertEquals(config.n, True)
115 117 self.assertEquals(config.Global.bam, 'wow')
116 118
117 119
118 120 class TestKeyValueCL(TestCase):
119 121
120 122 def test_basic(self):
121 123 cl = KeyValueConfigLoader()
122 124 argv = ['--'+s.strip('c.') for s in pyfile.split('\n')[2:-1]]
125 with mute_warn():
123 126 config = cl.load_config(argv)
124 127 self.assertEquals(config.a, 10)
125 128 self.assertEquals(config.b, 20)
126 129 self.assertEquals(config.Foo.Bar.value, 10)
127 130 self.assertEquals(config.Foo.Bam.value, range(10))
128 131 self.assertEquals(config.D.C.value, 'hi there')
129 132
130 133 def test_extra_args(self):
131 134 cl = KeyValueConfigLoader()
135 with mute_warn():
132 136 config = cl.load_config(['--a=5', 'b', '--c=10', 'd'])
133 137 self.assertEquals(cl.extra_args, ['b', 'd'])
134 138 self.assertEquals(config.a, 5)
135 139 self.assertEquals(config.c, 10)
140 with mute_warn():
136 141 config = cl.load_config(['--', '--a=5', '--c=10'])
137 142 self.assertEquals(cl.extra_args, ['--a=5', '--c=10'])
138 143
139 144 def test_unicode_args(self):
140 145 cl = KeyValueConfigLoader()
141 146 argv = [u'--a=épsîlön']
147 with mute_warn():
142 148 config = cl.load_config(argv)
143 149 self.assertEquals(config.a, u'épsîlön')
144 150
145 151 def test_unicode_bytes_args(self):
146 152 uarg = u'--a=é'
147 153 try:
148 154 barg = uarg.encode(sys.stdin.encoding)
149 155 except (TypeError, UnicodeEncodeError):
150 156 raise SkipTest("sys.stdin.encoding can't handle 'é'")
151 157
152 158 cl = KeyValueConfigLoader()
159 with mute_warn():
153 160 config = cl.load_config([barg])
154 161 self.assertEquals(config.a, u'é')
155 162
156 163
157 164 class TestConfig(TestCase):
158 165
159 166 def test_setget(self):
160 167 c = Config()
161 168 c.a = 10
162 169 self.assertEquals(c.a, 10)
163 170 self.assertEquals(c.has_key('b'), False)
164 171
165 172 def test_auto_section(self):
166 173 c = Config()
167 174 self.assertEquals(c.has_key('A'), True)
168 175 self.assertEquals(c._has_section('A'), False)
169 176 A = c.A
170 177 A.foo = 'hi there'
171 178 self.assertEquals(c._has_section('A'), True)
172 179 self.assertEquals(c.A.foo, 'hi there')
173 180 del c.A
174 181 self.assertEquals(len(c.A.keys()),0)
175 182
176 183 def test_merge_doesnt_exist(self):
177 184 c1 = Config()
178 185 c2 = Config()
179 186 c2.bar = 10
180 187 c2.Foo.bar = 10
181 188 c1._merge(c2)
182 189 self.assertEquals(c1.Foo.bar, 10)
183 190 self.assertEquals(c1.bar, 10)
184 191 c2.Bar.bar = 10
185 192 c1._merge(c2)
186 193 self.assertEquals(c1.Bar.bar, 10)
187 194
188 195 def test_merge_exists(self):
189 196 c1 = Config()
190 197 c2 = Config()
191 198 c1.Foo.bar = 10
192 199 c1.Foo.bam = 30
193 200 c2.Foo.bar = 20
194 201 c2.Foo.wow = 40
195 202 c1._merge(c2)
196 203 self.assertEquals(c1.Foo.bam, 30)
197 204 self.assertEquals(c1.Foo.bar, 20)
198 205 self.assertEquals(c1.Foo.wow, 40)
199 206 c2.Foo.Bam.bam = 10
200 207 c1._merge(c2)
201 208 self.assertEquals(c1.Foo.Bam.bam, 10)
202 209
203 210 def test_deepcopy(self):
204 211 c1 = Config()
205 212 c1.Foo.bar = 10
206 213 c1.Foo.bam = 30
207 214 c1.a = 'asdf'
208 215 c1.b = range(10)
209 216 import copy
210 217 c2 = copy.deepcopy(c1)
211 218 self.assertEquals(c1, c2)
212 219 self.assert_(c1 is not c2)
213 220 self.assert_(c1.Foo is not c2.Foo)
214 221
215 222 def test_builtin(self):
216 223 c1 = Config()
217 224 exec 'foo = True' in c1
218 225 self.assertEquals(c1.foo, True)
219 226 self.assertRaises(ConfigError, setattr, c1, 'ValueError', 10)
@@ -1,307 +1,307 b''
1 1 # encoding: utf-8
2 2 """
3 3 An application for IPython.
4 4
5 5 All top-level applications should use the classes in this module for
6 6 handling configuration and creating componenets.
7 7
8 8 The job of an :class:`Application` is to create the master configuration
9 9 object and then create the configurable objects, passing the config to them.
10 10
11 11 Authors:
12 12
13 13 * Brian Granger
14 14 * Fernando Perez
15 15 * Min RK
16 16
17 17 """
18 18
19 19 #-----------------------------------------------------------------------------
20 20 # Copyright (C) 2008-2011 The IPython Development Team
21 21 #
22 22 # Distributed under the terms of the BSD License. The full license is in
23 23 # the file COPYING, distributed as part of this software.
24 24 #-----------------------------------------------------------------------------
25 25
26 26 #-----------------------------------------------------------------------------
27 27 # Imports
28 28 #-----------------------------------------------------------------------------
29 29
30 30 import glob
31 31 import logging
32 32 import os
33 33 import shutil
34 34 import sys
35 35
36 36 from IPython.config.application import Application
37 37 from IPython.config.configurable import Configurable
38 38 from IPython.config.loader import Config
39 39 from IPython.core import release, crashhandler
40 40 from IPython.core.profiledir import ProfileDir, ProfileDirError
41 41 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
42 42 from IPython.utils.traitlets import List, Unicode, Type, Bool, Dict
43 43
44 44 #-----------------------------------------------------------------------------
45 45 # Classes and functions
46 46 #-----------------------------------------------------------------------------
47 47
48 48
49 49 #-----------------------------------------------------------------------------
50 50 # Base Application Class
51 51 #-----------------------------------------------------------------------------
52 52
53 53 # aliases and flags
54 54
55 base_aliases = dict(
56 profile='BaseIPythonApplication.profile',
57 ipython_dir='BaseIPythonApplication.ipython_dir',
58 log_level='Application.log_level',
59 )
55 base_aliases = {
56 'profile' : 'BaseIPythonApplication.profile',
57 'ipython-dir' : 'BaseIPythonApplication.ipython_dir',
58 'log-level' : 'Application.log_level',
59 }
60 60
61 61 base_flags = dict(
62 62 debug = ({'Application' : {'log_level' : logging.DEBUG}},
63 63 "set log level to logging.DEBUG (maximize logging output)"),
64 64 quiet = ({'Application' : {'log_level' : logging.CRITICAL}},
65 65 "set log level to logging.CRITICAL (minimize logging output)"),
66 66 init = ({'BaseIPythonApplication' : {
67 67 'copy_config_files' : True,
68 68 'auto_create' : True}
69 69 }, "Initialize profile with default config files")
70 70 )
71 71
72 72
73 73 class BaseIPythonApplication(Application):
74 74
75 75 name = Unicode(u'ipython')
76 76 description = Unicode(u'IPython: an enhanced interactive Python shell.')
77 77 version = Unicode(release.version)
78 78
79 79 aliases = Dict(base_aliases)
80 80 flags = Dict(base_flags)
81 81 classes = List([ProfileDir])
82 82
83 83 # Track whether the config_file has changed,
84 84 # because some logic happens only if we aren't using the default.
85 85 config_file_specified = Bool(False)
86 86
87 87 config_file_name = Unicode(u'ipython_config.py')
88 88 def _config_file_name_default(self):
89 89 return self.name.replace('-','_') + u'_config.py'
90 90 def _config_file_name_changed(self, name, old, new):
91 91 if new != old:
92 92 self.config_file_specified = True
93 93
94 94 # The directory that contains IPython's builtin profiles.
95 95 builtin_profile_dir = Unicode(
96 96 os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
97 97 )
98 98
99 99 config_file_paths = List(Unicode)
100 100 def _config_file_paths_default(self):
101 101 return [os.getcwdu()]
102 102
103 103 profile = Unicode(u'default', config=True,
104 104 help="""The IPython profile to use."""
105 105 )
106 106 def _profile_changed(self, name, old, new):
107 107 self.builtin_profile_dir = os.path.join(
108 108 get_ipython_package_dir(), u'config', u'profile', new
109 109 )
110 110
111 111 ipython_dir = Unicode(get_ipython_dir(), config=True,
112 112 help="""
113 113 The name of the IPython directory. This directory is used for logging
114 114 configuration (through profiles), history storage, etc. The default
115 115 is usually $HOME/.ipython. This options can also be specified through
116 116 the environment variable IPYTHON_DIR.
117 117 """
118 118 )
119 119
120 120 overwrite = Bool(False, config=True,
121 121 help="""Whether to overwrite existing config files when copying""")
122 122 auto_create = Bool(False, config=True,
123 123 help="""Whether to create profile dir if it doesn't exist""")
124 124
125 125 config_files = List(Unicode)
126 126 def _config_files_default(self):
127 127 return [u'ipython_config.py']
128 128
129 129 copy_config_files = Bool(False, config=True,
130 130 help="""Whether to install the default config files into the profile dir.
131 131 If a new profile is being created, and IPython contains config files for that
132 132 profile, then they will be staged into the new directory. Otherwise,
133 133 default config files will be automatically generated.
134 134 """)
135 135
136 136 # The class to use as the crash handler.
137 137 crash_handler_class = Type(crashhandler.CrashHandler)
138 138
139 139 def __init__(self, **kwargs):
140 140 super(BaseIPythonApplication, self).__init__(**kwargs)
141 141 # ensure even default IPYTHON_DIR exists
142 142 if not os.path.exists(self.ipython_dir):
143 143 self._ipython_dir_changed('ipython_dir', self.ipython_dir, self.ipython_dir)
144 144
145 145 #-------------------------------------------------------------------------
146 146 # Various stages of Application creation
147 147 #-------------------------------------------------------------------------
148 148
149 149 def init_crash_handler(self):
150 150 """Create a crash handler, typically setting sys.excepthook to it."""
151 151 self.crash_handler = self.crash_handler_class(self)
152 152 sys.excepthook = self.crash_handler
153 153
154 154 def _ipython_dir_changed(self, name, old, new):
155 155 if old in sys.path:
156 156 sys.path.remove(old)
157 157 sys.path.append(os.path.abspath(new))
158 158 if not os.path.isdir(new):
159 159 os.makedirs(new, mode=0777)
160 160 readme = os.path.join(new, 'README')
161 161 if not os.path.exists(readme):
162 162 path = os.path.join(get_ipython_package_dir(), u'config', u'profile')
163 163 shutil.copy(os.path.join(path, 'README'), readme)
164 164 self.log.debug("IPYTHON_DIR set to: %s" % new)
165 165
166 166 def load_config_file(self, suppress_errors=True):
167 167 """Load the config file.
168 168
169 169 By default, errors in loading config are handled, and a warning
170 170 printed on screen. For testing, the suppress_errors option is set
171 171 to False, so errors will make tests fail.
172 172 """
173 173 base_config = 'ipython_config.py'
174 174 self.log.debug("Attempting to load config file: %s" %
175 175 base_config)
176 176 try:
177 177 Application.load_config_file(
178 178 self,
179 179 base_config,
180 180 path=self.config_file_paths
181 181 )
182 182 except IOError:
183 183 # ignore errors loading parent
184 184 pass
185 185 if self.config_file_name == base_config:
186 186 # don't load secondary config
187 187 return
188 188 self.log.debug("Attempting to load config file: %s" %
189 189 self.config_file_name)
190 190 try:
191 191 Application.load_config_file(
192 192 self,
193 193 self.config_file_name,
194 194 path=self.config_file_paths
195 195 )
196 196 except IOError:
197 197 # Only warn if the default config file was NOT being used.
198 198 if self.config_file_specified:
199 199 self.log.warn("Config file not found, skipping: %s" %
200 200 self.config_file_name)
201 201 except:
202 202 # For testing purposes.
203 203 if not suppress_errors:
204 204 raise
205 205 self.log.warn("Error loading config file: %s" %
206 206 self.config_file_name, exc_info=True)
207 207
208 208 def init_profile_dir(self):
209 209 """initialize the profile dir"""
210 210 try:
211 211 # location explicitly specified:
212 212 location = self.config.ProfileDir.location
213 213 except AttributeError:
214 214 # location not specified, find by profile name
215 215 try:
216 216 p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
217 217 except ProfileDirError:
218 218 # not found, maybe create it (always create default profile)
219 219 if self.auto_create or self.profile=='default':
220 220 try:
221 221 p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
222 222 except ProfileDirError:
223 223 self.log.fatal("Could not create profile: %r"%self.profile)
224 224 self.exit(1)
225 225 else:
226 226 self.log.info("Created profile dir: %r"%p.location)
227 227 else:
228 228 self.log.fatal("Profile %r not found."%self.profile)
229 229 self.exit(1)
230 230 else:
231 231 self.log.info("Using existing profile dir: %r"%p.location)
232 232 else:
233 233 # location is fully specified
234 234 try:
235 235 p = ProfileDir.find_profile_dir(location, self.config)
236 236 except ProfileDirError:
237 237 # not found, maybe create it
238 238 if self.auto_create:
239 239 try:
240 240 p = ProfileDir.create_profile_dir(location, self.config)
241 241 except ProfileDirError:
242 242 self.log.fatal("Could not create profile directory: %r"%location)
243 243 self.exit(1)
244 244 else:
245 245 self.log.info("Creating new profile dir: %r"%location)
246 246 else:
247 247 self.log.fatal("Profile directory %r not found."%location)
248 248 self.exit(1)
249 249 else:
250 250 self.log.info("Using existing profile dir: %r"%location)
251 251
252 252 self.profile_dir = p
253 253 self.config_file_paths.append(p.location)
254 254
255 255 def init_config_files(self):
256 256 """[optionally] copy default config files into profile dir."""
257 257 # copy config files
258 258 path = self.builtin_profile_dir
259 259 if self.copy_config_files:
260 260 src = self.profile
261 261
262 262 cfg = self.config_file_name
263 263 if path and os.path.exists(os.path.join(path, cfg)):
264 264 self.log.warn("Staging %r from %s into %r [overwrite=%s]"%(
265 265 cfg, src, self.profile_dir.location, self.overwrite)
266 266 )
267 267 self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite)
268 268 else:
269 269 self.stage_default_config_file()
270 270 else:
271 271 # Still stage *bundled* config files, but not generated ones
272 272 # This is necessary for `ipython profile=sympy` to load the profile
273 273 # on the first go
274 274 files = glob.glob(os.path.join(path, '*.py'))
275 275 for fullpath in files:
276 276 cfg = os.path.basename(fullpath)
277 277 if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False):
278 278 # file was copied
279 279 self.log.warn("Staging bundled %s from %s into %r"%(
280 280 cfg, self.profile, self.profile_dir.location)
281 281 )
282 282
283 283
284 284 def stage_default_config_file(self):
285 285 """auto generate default config file, and stage it into the profile."""
286 286 s = self.generate_config_file()
287 287 fname = os.path.join(self.profile_dir.location, self.config_file_name)
288 288 if self.overwrite or not os.path.exists(fname):
289 289 self.log.warn("Generating default config file: %r"%(fname))
290 290 with open(fname, 'w') as f:
291 291 f.write(s)
292 292
293 293
294 294 def initialize(self, argv=None):
295 295 # don't hook up crash handler before parsing command-line
296 296 self.parse_command_line(argv)
297 297 self.init_crash_handler()
298 298 if self.subapp is not None:
299 299 # stop here if subapp is taking over
300 300 return
301 301 cl_config = self.config
302 302 self.init_profile_dir()
303 303 self.init_config_files()
304 304 self.load_config_file()
305 305 # enforce cl-opts override configfile opts:
306 306 self.update_config(cl_config)
307 307
@@ -1,220 +1,220 b''
1 1 # encoding: utf-8
2 2 """
3 3 An application for managing IPython profiles.
4 4
5 5 To be invoked as the `ipython profile` subcommand.
6 6
7 7 Authors:
8 8
9 9 * Min RK
10 10
11 11 """
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Copyright (C) 2008-2011 The IPython Development Team
15 15 #
16 16 # Distributed under the terms of the BSD License. The full license is in
17 17 # the file COPYING, distributed as part of this software.
18 18 #-----------------------------------------------------------------------------
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Imports
22 22 #-----------------------------------------------------------------------------
23 23
24 24 import logging
25 25 import os
26 26
27 27 from IPython.config.application import Application, boolean_flag
28 28 from IPython.core.application import (
29 29 BaseIPythonApplication, base_flags, base_aliases
30 30 )
31 31 from IPython.core.profiledir import ProfileDir
32 32 from IPython.utils.path import get_ipython_dir
33 33 from IPython.utils.traitlets import Unicode, Bool, Dict
34 34
35 35 #-----------------------------------------------------------------------------
36 36 # Constants
37 37 #-----------------------------------------------------------------------------
38 38
39 39 create_help = """Create an IPython profile by name
40 40
41 41 Create an ipython profile directory by its name or
42 42 profile directory path. Profile directories contain
43 43 configuration, log and security related files and are named
44 44 using the convention 'profile_<name>'. By default they are
45 45 located in your ipython directory. Once created, you will
46 46 can edit the configuration files in the profile
47 47 directory to configure IPython. Most users will create a
48 48 profile directory by name,
49 49 `ipython profile create myprofile`, which will put the directory
50 50 in `<ipython_dir>/profile_myprofile`.
51 51 """
52 52 list_help = """List available IPython profiles
53 53
54 54 List all available profiles, by profile location, that can
55 55 be found in the current working directly or in the ipython
56 56 directory. Profile directories are named using the convention
57 57 'profile_<profile>'.
58 58 """
59 59 profile_help = """Manage IPython profiles
60 60
61 61 Profile directories contain
62 62 configuration, log and security related files and are named
63 63 using the convention 'profile_<name>'. By default they are
64 64 located in your ipython directory. You can create profiles
65 65 with `ipython profile create <name>`, or see the profiles you
66 66 already have with `ipython profile list`
67 67
68 68 To get started configuring IPython, simply do:
69 69
70 70 $> ipython profile create
71 71
72 72 and IPython will create the default profile in <ipython_dir>/profile_default,
73 73 where you can edit ipython_config.py to start configuring IPython.
74 74
75 75 """
76 76
77 77 #-----------------------------------------------------------------------------
78 78 # Profile Application Class (for `ipython profile` subcommand)
79 79 #-----------------------------------------------------------------------------
80 80
81 81
82 82
83 83 class ProfileList(Application):
84 84 name = u'ipython-profile'
85 85 description = list_help
86 86
87 aliases = Dict(dict(
88 ipython_dir = 'ProfileList.ipython_dir',
89 log_level = 'Application.log_level',
90 ))
87 aliases = Dict({
88 'ipython-dir' : 'ProfileList.ipython_dir',
89 'log-level' : 'Application.log_level',
90 })
91 91 flags = Dict(dict(
92 92 debug = ({'Application' : {'log_level' : 0}},
93 "Set log_level to 0, maximizing log output."
93 "Set Application.log_level to 0, maximizing log output."
94 94 )
95 95 ))
96 96 ipython_dir = Unicode(get_ipython_dir(), config=True,
97 97 help="""
98 98 The name of the IPython directory. This directory is used for logging
99 99 configuration (through profiles), history storage, etc. The default
100 100 is usually $HOME/.ipython. This options can also be specified through
101 101 the environment variable IPYTHON_DIR.
102 102 """
103 103 )
104 104
105 105 def list_profile_dirs(self):
106 106 # Find the search paths
107 107 paths = [os.getcwdu(), self.ipython_dir]
108 108
109 109 self.log.warn('Searching for IPython profiles in paths: %r' % paths)
110 110 for path in paths:
111 111 files = os.listdir(path)
112 112 for f in files:
113 113 full_path = os.path.join(path, f)
114 114 if os.path.isdir(full_path) and f.startswith('profile_'):
115 115 profile = f.split('_',1)[-1]
116 116 start_cmd = 'ipython profile=%s' % profile
117 117 print start_cmd + " ==> " + full_path
118 118
119 119 def start(self):
120 120 self.list_profile_dirs()
121 121
122 122
123 123 create_flags = {}
124 124 create_flags.update(base_flags)
125 125 create_flags.update(boolean_flag('reset', 'ProfileCreate.overwrite',
126 126 "reset config files to defaults", "leave existing config files"))
127 127 create_flags.update(boolean_flag('parallel', 'ProfileCreate.parallel',
128 128 "Include parallel computing config files",
129 129 "Don't include parallel computing config files"))
130 130
131 131 class ProfileCreate(BaseIPythonApplication):
132 132 name = u'ipython-profile'
133 133 description = create_help
134 134 auto_create = Bool(True, config=False)
135 135
136 136 def _copy_config_files_default(self):
137 137 return True
138 138
139 139 parallel = Bool(False, config=True,
140 140 help="whether to include parallel computing config files")
141 141 def _parallel_changed(self, name, old, new):
142 142 parallel_files = [ 'ipcontroller_config.py',
143 143 'ipengine_config.py',
144 144 'ipcluster_config.py'
145 145 ]
146 146 if new:
147 147 for cf in parallel_files:
148 148 self.config_files.append(cf)
149 149 else:
150 150 for cf in parallel_files:
151 151 if cf in self.config_files:
152 152 self.config_files.remove(cf)
153 153
154 154 def parse_command_line(self, argv):
155 155 super(ProfileCreate, self).parse_command_line(argv)
156 156 # accept positional arg as profile name
157 157 if self.extra_args:
158 158 self.profile = self.extra_args[0]
159 159
160 160 flags = Dict(create_flags)
161 161
162 162 classes = [ProfileDir]
163 163
164 164 def init_config_files(self):
165 165 super(ProfileCreate, self).init_config_files()
166 166 # use local imports, since these classes may import from here
167 167 from IPython.frontend.terminal.ipapp import TerminalIPythonApp
168 168 apps = [TerminalIPythonApp]
169 169 try:
170 170 from IPython.frontend.qt.console.qtconsoleapp import IPythonQtConsoleApp
171 171 except Exception:
172 172 # this should be ImportError, but under weird circumstances
173 173 # this might be an AttributeError, or possibly others
174 174 # in any case, nothing should cause the profile creation to crash.
175 175 pass
176 176 else:
177 177 apps.append(IPythonQtConsoleApp)
178 178 if self.parallel:
179 179 from IPython.parallel.apps.ipcontrollerapp import IPControllerApp
180 180 from IPython.parallel.apps.ipengineapp import IPEngineApp
181 181 from IPython.parallel.apps.ipclusterapp import IPClusterStart
182 182 from IPython.parallel.apps.iploggerapp import IPLoggerApp
183 183 apps.extend([
184 184 IPControllerApp,
185 185 IPEngineApp,
186 186 IPClusterStart,
187 187 IPLoggerApp,
188 188 ])
189 189 for App in apps:
190 190 app = App()
191 191 app.config.update(self.config)
192 192 app.log = self.log
193 193 app.overwrite = self.overwrite
194 194 app.copy_config_files=True
195 195 app.profile = self.profile
196 196 app.init_profile_dir()
197 197 app.init_config_files()
198 198
199 199 def stage_default_config_file(self):
200 200 pass
201 201
202 202 class ProfileApp(Application):
203 203 name = u'ipython-profile'
204 204 description = profile_help
205 205
206 206 subcommands = Dict(dict(
207 207 create = (ProfileCreate, "Create a new profile dir with default config files"),
208 208 list = (ProfileList, "List existing profiles")
209 209 ))
210 210
211 211 def start(self):
212 212 if self.subapp is None:
213 213 print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
214 214 print
215 215 self.print_description()
216 216 self.print_subcommands()
217 217 self.exit(1)
218 218 else:
219 219 return self.subapp.start()
220 220
@@ -1,253 +1,253 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 A mixin for :class:`~IPython.core.application.Application` classes that
5 5 launch InteractiveShell instances, load extensions, etc.
6 6
7 7 Authors
8 8 -------
9 9
10 10 * Min Ragan-Kelley
11 11 """
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Copyright (C) 2008-2011 The IPython Development Team
15 15 #
16 16 # Distributed under the terms of the BSD License. The full license is in
17 17 # the file COPYING, distributed as part of this software.
18 18 #-----------------------------------------------------------------------------
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Imports
22 22 #-----------------------------------------------------------------------------
23 23
24 24 from __future__ import absolute_import
25 25
26 26 import os
27 27 import sys
28 28
29 29 from IPython.config.application import boolean_flag
30 30 from IPython.config.configurable import Configurable
31 31 from IPython.config.loader import Config
32 32 from IPython.utils.path import filefind
33 33 from IPython.utils.traitlets import Unicode, Instance, List
34 34
35 35 #-----------------------------------------------------------------------------
36 36 # Aliases and Flags
37 37 #-----------------------------------------------------------------------------
38 38
39 39 shell_flags = {}
40 40
41 41 addflag = lambda *args: shell_flags.update(boolean_flag(*args))
42 42 addflag('autoindent', 'InteractiveShell.autoindent',
43 43 'Turn on autoindenting.', 'Turn off autoindenting.'
44 44 )
45 45 addflag('automagic', 'InteractiveShell.automagic',
46 46 """Turn on the auto calling of magic commands. Type %%magic at the
47 47 IPython prompt for more information.""",
48 48 'Turn off the auto calling of magic commands.'
49 49 )
50 50 addflag('pdb', 'InteractiveShell.pdb',
51 51 "Enable auto calling the pdb debugger after every exception.",
52 52 "Disable auto calling the pdb debugger after every exception."
53 53 )
54 54 addflag('pprint', 'PlainTextFormatter.pprint',
55 55 "Enable auto pretty printing of results.",
56 56 "Disable auto auto pretty printing of results."
57 57 )
58 58 addflag('color-info', 'InteractiveShell.color_info',
59 59 """IPython can display information about objects via a set of func-
60 60 tions, and optionally can use colors for this, syntax highlighting
61 61 source code and various other elements. However, because this
62 62 information is passed through a pager (like 'less') and many pagers get
63 63 confused with color codes, this option is off by default. You can test
64 64 it and turn it on permanently in your ipython_config.py file if it
65 65 works for you. Test it and turn it on permanently if it works with
66 66 your system. The magic function %%color_info allows you to toggle this
67 67 interactively for testing.""",
68 68 "Disable using colors for info related things."
69 69 )
70 70 addflag('deep-reload', 'InteractiveShell.deep_reload',
71 71 """Enable deep (recursive) reloading by default. IPython can use the
72 72 deep_reload module which reloads changes in modules recursively (it
73 73 replaces the reload() function, so you don't need to change anything to
74 74 use it). deep_reload() forces a full reload of modules whose code may
75 75 have changed, which the default reload() function does not. When
76 76 deep_reload is off, IPython will use the normal reload(), but
77 77 deep_reload will still be available as dreload(). This feature is off
78 78 by default [which means that you have both normal reload() and
79 79 dreload()].""",
80 80 "Disable deep (recursive) reloading by default."
81 81 )
82 82 nosep_config = Config()
83 83 nosep_config.InteractiveShell.separate_in = ''
84 84 nosep_config.InteractiveShell.separate_out = ''
85 85 nosep_config.InteractiveShell.separate_out2 = ''
86 86
87 87 shell_flags['nosep']=(nosep_config, "Eliminate all spacing between prompts.")
88 88
89 89
90 90 # it's possible we don't want short aliases for *all* of these:
91 91 shell_aliases = dict(
92 92 autocall='InteractiveShell.autocall',
93 cache_size='InteractiveShell.cache_size',
94 93 colors='InteractiveShell.colors',
95 94 logfile='InteractiveShell.logfile',
96 95 logappend='InteractiveShell.logappend',
97 96 c='InteractiveShellApp.code_to_run',
98 97 ext='InteractiveShellApp.extra_extension',
99 98 )
99 shell_aliases['cache-size'] = 'InteractiveShell.cache_size'
100 100
101 101 #-----------------------------------------------------------------------------
102 102 # Main classes and functions
103 103 #-----------------------------------------------------------------------------
104 104
105 105 class InteractiveShellApp(Configurable):
106 106 """A Mixin for applications that start InteractiveShell instances.
107 107
108 108 Provides configurables for loading extensions and executing files
109 109 as part of configuring a Shell environment.
110 110
111 111 Provides init_extensions() and init_code() methods, to be called
112 112 after init_shell(), which must be implemented by subclasses.
113 113 """
114 114 extensions = List(Unicode, config=True,
115 115 help="A list of dotted module names of IPython extensions to load."
116 116 )
117 117 extra_extension = Unicode('', config=True,
118 118 help="dotted module name of an IPython extension to load."
119 119 )
120 120 def _extra_extension_changed(self, name, old, new):
121 121 if new:
122 122 # add to self.extensions
123 123 self.extensions.append(new)
124 124
125 125 exec_files = List(Unicode, config=True,
126 126 help="""List of files to run at IPython startup."""
127 127 )
128 128 file_to_run = Unicode('', config=True,
129 129 help="""A file to be run""")
130 130
131 131 exec_lines = List(Unicode, config=True,
132 132 help="""lines of code to run at IPython startup."""
133 133 )
134 134 code_to_run = Unicode('', config=True,
135 135 help="Execute the given command string."
136 136 )
137 137 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
138 138
139 139 def init_shell(self):
140 140 raise NotImplementedError("Override in subclasses")
141 141
142 142 def init_extensions(self):
143 143 """Load all IPython extensions in IPythonApp.extensions.
144 144
145 145 This uses the :meth:`ExtensionManager.load_extensions` to load all
146 146 the extensions listed in ``self.extensions``.
147 147 """
148 148 if not self.extensions:
149 149 return
150 150 try:
151 151 self.log.debug("Loading IPython extensions...")
152 152 extensions = self.extensions
153 153 for ext in extensions:
154 154 try:
155 155 self.log.info("Loading IPython extension: %s" % ext)
156 156 self.shell.extension_manager.load_extension(ext)
157 157 except:
158 158 self.log.warn("Error in loading extension: %s" % ext)
159 159 self.shell.showtraceback()
160 160 except:
161 161 self.log.warn("Unknown error in loading extensions:")
162 162 self.shell.showtraceback()
163 163
164 164 def init_code(self):
165 165 """run the pre-flight code, specified via exec_lines"""
166 166 self._run_exec_lines()
167 167 self._run_exec_files()
168 168 self._run_cmd_line_code()
169 169
170 170 def _run_exec_lines(self):
171 171 """Run lines of code in IPythonApp.exec_lines in the user's namespace."""
172 172 if not self.exec_lines:
173 173 return
174 174 try:
175 175 self.log.debug("Running code from IPythonApp.exec_lines...")
176 176 for line in self.exec_lines:
177 177 try:
178 178 self.log.info("Running code in user namespace: %s" %
179 179 line)
180 180 self.shell.run_cell(line, store_history=False)
181 181 except:
182 182 self.log.warn("Error in executing line in user "
183 183 "namespace: %s" % line)
184 184 self.shell.showtraceback()
185 185 except:
186 186 self.log.warn("Unknown error in handling IPythonApp.exec_lines:")
187 187 self.shell.showtraceback()
188 188
189 189 def _exec_file(self, fname):
190 190 try:
191 191 full_filename = filefind(fname, [u'.', self.ipython_dir])
192 192 except IOError as e:
193 193 self.log.warn("File not found: %r"%fname)
194 194 return
195 195 # Make sure that the running script gets a proper sys.argv as if it
196 196 # were run from a system shell.
197 197 save_argv = sys.argv
198 198 sys.argv = [full_filename] + self.extra_args[1:]
199 199 try:
200 200 if os.path.isfile(full_filename):
201 201 if full_filename.endswith('.ipy'):
202 202 self.log.info("Running file in user namespace: %s" %
203 203 full_filename)
204 204 self.shell.safe_execfile_ipy(full_filename)
205 205 else:
206 206 # default to python, even without extension
207 207 self.log.info("Running file in user namespace: %s" %
208 208 full_filename)
209 209 # Ensure that __file__ is always defined to match Python behavior
210 210 self.shell.user_ns['__file__'] = fname
211 211 try:
212 212 self.shell.safe_execfile(full_filename, self.shell.user_ns)
213 213 finally:
214 214 del self.shell.user_ns['__file__']
215 215 finally:
216 216 sys.argv = save_argv
217 217
218 218 def _run_exec_files(self):
219 219 """Run files from IPythonApp.exec_files"""
220 220 if not self.exec_files:
221 221 return
222 222
223 223 self.log.debug("Running files in IPythonApp.exec_files...")
224 224 try:
225 225 for fname in self.exec_files:
226 226 self._exec_file(fname)
227 227 except:
228 228 self.log.warn("Unknown error in handling IPythonApp.exec_files:")
229 229 self.shell.showtraceback()
230 230
231 231 def _run_cmd_line_code(self):
232 232 """Run code or file specified at the command-line"""
233 233 if self.code_to_run:
234 234 line = self.code_to_run
235 235 try:
236 236 self.log.info("Running code given at command line (c=): %s" %
237 237 line)
238 238 self.shell.run_cell(line, store_history=False)
239 239 except:
240 240 self.log.warn("Error in executing line in user namespace: %s" %
241 241 line)
242 242 self.shell.showtraceback()
243 243
244 244 # Like Python itself, ignore the second if the first of these is present
245 245 elif self.file_to_run:
246 246 fname = self.file_to_run
247 247 try:
248 248 self._exec_file(fname)
249 249 except:
250 250 self.log.warn("Error in executing file in user namespace: %s" %
251 251 fname)
252 252 self.shell.showtraceback()
253 253
@@ -1,266 +1,266 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 The Base Application class for IPython.parallel apps
5 5
6 6 Authors:
7 7
8 8 * Brian Granger
9 9 * Min RK
10 10
11 11 """
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Copyright (C) 2008-2011 The IPython Development Team
15 15 #
16 16 # Distributed under the terms of the BSD License. The full license is in
17 17 # the file COPYING, distributed as part of this software.
18 18 #-----------------------------------------------------------------------------
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Imports
22 22 #-----------------------------------------------------------------------------
23 23
24 24 from __future__ import with_statement
25 25
26 26 import os
27 27 import logging
28 28 import re
29 29 import sys
30 30
31 31 from subprocess import Popen, PIPE
32 32
33 33 from IPython.core import release
34 34 from IPython.core.crashhandler import CrashHandler
35 35 from IPython.core.application import (
36 36 BaseIPythonApplication,
37 37 base_aliases as base_ip_aliases,
38 38 base_flags as base_ip_flags
39 39 )
40 40 from IPython.utils.path import expand_path
41 41
42 42 from IPython.utils.traitlets import Unicode, Bool, Instance, Dict, List
43 43
44 44 #-----------------------------------------------------------------------------
45 45 # Module errors
46 46 #-----------------------------------------------------------------------------
47 47
48 48 class PIDFileError(Exception):
49 49 pass
50 50
51 51
52 52 #-----------------------------------------------------------------------------
53 53 # Crash handler for this application
54 54 #-----------------------------------------------------------------------------
55 55
56 56
57 57 _message_template = """\
58 58 Oops, $self.app_name crashed. We do our best to make it stable, but...
59 59
60 60 A crash report was automatically generated with the following information:
61 61 - A verbatim copy of the crash traceback.
62 62 - Data on your current $self.app_name configuration.
63 63
64 64 It was left in the file named:
65 65 \t'$self.crash_report_fname'
66 66 If you can email this file to the developers, the information in it will help
67 67 them in understanding and correcting the problem.
68 68
69 69 You can mail it to: $self.contact_name at $self.contact_email
70 70 with the subject '$self.app_name Crash Report'.
71 71
72 72 If you want to do it now, the following command will work (under Unix):
73 73 mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
74 74
75 75 To ensure accurate tracking of this issue, please file a report about it at:
76 76 $self.bug_tracker
77 77 """
78 78
79 79 class ParallelCrashHandler(CrashHandler):
80 80 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
81 81
82 82 message_template = _message_template
83 83
84 84 def __init__(self, app):
85 85 contact_name = release.authors['Min'][0]
86 86 contact_email = release.authors['Min'][1]
87 87 bug_tracker = 'http://github.com/ipython/ipython/issues'
88 88 super(ParallelCrashHandler,self).__init__(
89 89 app, contact_name, contact_email, bug_tracker
90 90 )
91 91
92 92
93 93 #-----------------------------------------------------------------------------
94 94 # Main application
95 95 #-----------------------------------------------------------------------------
96 96 base_aliases = {}
97 97 base_aliases.update(base_ip_aliases)
98 98 base_aliases.update({
99 'profile_dir' : 'ProfileDir.location',
100 'work_dir' : 'BaseParallelApplication.work_dir',
101 'log_to_file' : 'BaseParallelApplication.log_to_file',
102 'clean_logs' : 'BaseParallelApplication.clean_logs',
103 'log_url' : 'BaseParallelApplication.log_url',
99 'profile-dir' : 'ProfileDir.location',
100 'work-dir' : 'BaseParallelApplication.work_dir',
101 'log-to-file' : 'BaseParallelApplication.log_to_file',
102 'clean-logs' : 'BaseParallelApplication.clean_logs',
103 'log-url' : 'BaseParallelApplication.log_url',
104 104 })
105 105
106 106 base_flags = {
107 107 'log-to-file' : (
108 108 {'BaseParallelApplication' : {'log_to_file' : True}},
109 109 "send log output to a file"
110 110 )
111 111 }
112 112 base_flags.update(base_ip_flags)
113 113
114 114 class BaseParallelApplication(BaseIPythonApplication):
115 115 """The base Application for IPython.parallel apps
116 116
117 117 Principle extensions to BaseIPyythonApplication:
118 118
119 119 * work_dir
120 120 * remote logging via pyzmq
121 121 * IOLoop instance
122 122 """
123 123
124 124 crash_handler_class = ParallelCrashHandler
125 125
126 126 def _log_level_default(self):
127 127 # temporarily override default_log_level to INFO
128 128 return logging.INFO
129 129
130 130 work_dir = Unicode(os.getcwdu(), config=True,
131 131 help='Set the working dir for the process.'
132 132 )
133 133 def _work_dir_changed(self, name, old, new):
134 134 self.work_dir = unicode(expand_path(new))
135 135
136 136 log_to_file = Bool(config=True,
137 137 help="whether to log to a file")
138 138
139 139 clean_logs = Bool(False, config=True,
140 140 help="whether to cleanup old logfiles before starting")
141 141
142 142 log_url = Unicode('', config=True,
143 143 help="The ZMQ URL of the iplogger to aggregate logging.")
144 144
145 145 def _config_files_default(self):
146 146 return ['ipcontroller_config.py', 'ipengine_config.py', 'ipcluster_config.py']
147 147
148 148 loop = Instance('zmq.eventloop.ioloop.IOLoop')
149 149 def _loop_default(self):
150 150 from zmq.eventloop.ioloop import IOLoop
151 151 return IOLoop.instance()
152 152
153 153 aliases = Dict(base_aliases)
154 154 flags = Dict(base_flags)
155 155
156 156 def initialize(self, argv=None):
157 157 """initialize the app"""
158 158 super(BaseParallelApplication, self).initialize(argv)
159 159 self.to_work_dir()
160 160 self.reinit_logging()
161 161
162 162 def to_work_dir(self):
163 163 wd = self.work_dir
164 164 if unicode(wd) != os.getcwdu():
165 165 os.chdir(wd)
166 166 self.log.info("Changing to working dir: %s" % wd)
167 167 # This is the working dir by now.
168 168 sys.path.insert(0, '')
169 169
170 170 def reinit_logging(self):
171 171 # Remove old log files
172 172 log_dir = self.profile_dir.log_dir
173 173 if self.clean_logs:
174 174 for f in os.listdir(log_dir):
175 175 if re.match(r'%s-\d+\.(log|err|out)'%self.name,f):
176 176 os.remove(os.path.join(log_dir, f))
177 177 if self.log_to_file:
178 178 # Start logging to the new log file
179 179 log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
180 180 logfile = os.path.join(log_dir, log_filename)
181 181 open_log_file = open(logfile, 'w')
182 182 else:
183 183 open_log_file = None
184 184 if open_log_file is not None:
185 185 self.log.removeHandler(self._log_handler)
186 186 self._log_handler = logging.StreamHandler(open_log_file)
187 187 self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
188 188 self._log_handler.setFormatter(self._log_formatter)
189 189 self.log.addHandler(self._log_handler)
190 190
191 191 def write_pid_file(self, overwrite=False):
192 192 """Create a .pid file in the pid_dir with my pid.
193 193
194 194 This must be called after pre_construct, which sets `self.pid_dir`.
195 195 This raises :exc:`PIDFileError` if the pid file exists already.
196 196 """
197 197 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
198 198 if os.path.isfile(pid_file):
199 199 pid = self.get_pid_from_file()
200 200 if not overwrite:
201 201 raise PIDFileError(
202 202 'The pid file [%s] already exists. \nThis could mean that this '
203 203 'server is already running with [pid=%s].' % (pid_file, pid)
204 204 )
205 205 with open(pid_file, 'w') as f:
206 206 self.log.info("Creating pid file: %s" % pid_file)
207 207 f.write(repr(os.getpid())+'\n')
208 208
209 209 def remove_pid_file(self):
210 210 """Remove the pid file.
211 211
212 212 This should be called at shutdown by registering a callback with
213 213 :func:`reactor.addSystemEventTrigger`. This needs to return
214 214 ``None``.
215 215 """
216 216 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
217 217 if os.path.isfile(pid_file):
218 218 try:
219 219 self.log.info("Removing pid file: %s" % pid_file)
220 220 os.remove(pid_file)
221 221 except:
222 222 self.log.warn("Error removing the pid file: %s" % pid_file)
223 223
224 224 def get_pid_from_file(self):
225 225 """Get the pid from the pid file.
226 226
227 227 If the pid file doesn't exist a :exc:`PIDFileError` is raised.
228 228 """
229 229 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
230 230 if os.path.isfile(pid_file):
231 231 with open(pid_file, 'r') as f:
232 232 s = f.read().strip()
233 233 try:
234 234 pid = int(s)
235 235 except:
236 236 raise PIDFileError("invalid pid file: %s (contents: %r)"%(pid_file, s))
237 237 return pid
238 238 else:
239 239 raise PIDFileError('pid file not found: %s' % pid_file)
240 240
241 241 def check_pid(self, pid):
242 242 if os.name == 'nt':
243 243 try:
244 244 import ctypes
245 245 # returns 0 if no such process (of ours) exists
246 246 # positive int otherwise
247 247 p = ctypes.windll.kernel32.OpenProcess(1,0,pid)
248 248 except Exception:
249 249 self.log.warn(
250 250 "Could not determine whether pid %i is running via `OpenProcess`. "
251 251 " Making the likely assumption that it is."%pid
252 252 )
253 253 return True
254 254 return bool(p)
255 255 else:
256 256 try:
257 257 p = Popen(['ps','x'], stdout=PIPE, stderr=PIPE)
258 258 output,_ = p.communicate()
259 259 except OSError:
260 260 self.log.warn(
261 261 "Could not determine whether pid %i is running via `ps x`. "
262 262 " Making the likely assumption that it is."%pid
263 263 )
264 264 return True
265 265 pids = map(int, re.findall(r'^\W*\d+', output, re.MULTILINE))
266 266 return pid in pids
@@ -1,459 +1,459 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 The ipcluster application.
5 5
6 6 Authors:
7 7
8 8 * Brian Granger
9 9 * MinRK
10 10
11 11 """
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Copyright (C) 2008-2011 The IPython Development Team
15 15 #
16 16 # Distributed under the terms of the BSD License. The full license is in
17 17 # the file COPYING, distributed as part of this software.
18 18 #-----------------------------------------------------------------------------
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Imports
22 22 #-----------------------------------------------------------------------------
23 23
24 24 import errno
25 25 import logging
26 26 import os
27 27 import re
28 28 import signal
29 29
30 30 from subprocess import check_call, CalledProcessError, PIPE
31 31 import zmq
32 32 from zmq.eventloop import ioloop
33 33
34 34 from IPython.config.application import Application, boolean_flag
35 35 from IPython.config.loader import Config
36 36 from IPython.core.application import BaseIPythonApplication
37 37 from IPython.core.profiledir import ProfileDir
38 38 from IPython.utils.daemonize import daemonize
39 39 from IPython.utils.importstring import import_item
40 40 from IPython.utils.traitlets import (Int, Unicode, Bool, CFloat, Dict, List,
41 41 DottedObjectName)
42 42
43 43 from IPython.parallel.apps.baseapp import (
44 44 BaseParallelApplication,
45 45 PIDFileError,
46 46 base_flags, base_aliases
47 47 )
48 48
49 49
50 50 #-----------------------------------------------------------------------------
51 51 # Module level variables
52 52 #-----------------------------------------------------------------------------
53 53
54 54
55 55 default_config_file_name = u'ipcluster_config.py'
56 56
57 57
58 58 _description = """Start an IPython cluster for parallel computing.
59 59
60 60 An IPython cluster consists of 1 controller and 1 or more engines.
61 61 This command automates the startup of these processes using a wide
62 62 range of startup methods (SSH, local processes, PBS, mpiexec,
63 63 Windows HPC Server 2008). To start a cluster with 4 engines on your
64 64 local host simply do 'ipcluster start n=4'. For more complex usage
65 65 you will typically do 'ipcluster create profile=mycluster', then edit
66 66 configuration files, followed by 'ipcluster start profile=mycluster n=4'.
67 67 """
68 68
69 69
70 70 # Exit codes for ipcluster
71 71
72 72 # This will be the exit code if the ipcluster appears to be running because
73 73 # a .pid file exists
74 74 ALREADY_STARTED = 10
75 75
76 76
77 77 # This will be the exit code if ipcluster stop is run, but there is not .pid
78 78 # file to be found.
79 79 ALREADY_STOPPED = 11
80 80
81 81 # This will be the exit code if ipcluster engines is run, but there is not .pid
82 82 # file to be found.
83 83 NO_CLUSTER = 12
84 84
85 85
86 86 #-----------------------------------------------------------------------------
87 87 # Main application
88 88 #-----------------------------------------------------------------------------
89 89 start_help = """Start an IPython cluster for parallel computing
90 90
91 91 Start an ipython cluster by its profile name or cluster
92 92 directory. Cluster directories contain configuration, log and
93 93 security related files and are named using the convention
94 94 'profile_<name>' and should be creating using the 'start'
95 95 subcommand of 'ipcluster'. If your cluster directory is in
96 96 the cwd or the ipython directory, you can simply refer to it
97 97 using its profile name, 'ipcluster start n=4 profile=<profile>`,
98 98 otherwise use the 'profile_dir' option.
99 99 """
100 100 stop_help = """Stop a running IPython cluster
101 101
102 102 Stop a running ipython cluster by its profile name or cluster
103 103 directory. Cluster directories are named using the convention
104 104 'profile_<name>'. If your cluster directory is in
105 105 the cwd or the ipython directory, you can simply refer to it
106 106 using its profile name, 'ipcluster stop profile=<profile>`, otherwise
107 107 use the 'profile_dir' option.
108 108 """
109 109 engines_help = """Start engines connected to an existing IPython cluster
110 110
111 111 Start one or more engines to connect to an existing Cluster
112 112 by profile name or cluster directory.
113 113 Cluster directories contain configuration, log and
114 114 security related files and are named using the convention
115 115 'profile_<name>' and should be creating using the 'start'
116 116 subcommand of 'ipcluster'. If your cluster directory is in
117 117 the cwd or the ipython directory, you can simply refer to it
118 118 using its profile name, 'ipcluster engines n=4 profile=<profile>`,
119 119 otherwise use the 'profile_dir' option.
120 120 """
121 121 stop_aliases = dict(
122 122 signal='IPClusterStop.signal',
123 123 )
124 124 stop_aliases.update(base_aliases)
125 125
126 126 class IPClusterStop(BaseParallelApplication):
127 127 name = u'ipcluster'
128 128 description = stop_help
129 129 config_file_name = Unicode(default_config_file_name)
130 130
131 131 signal = Int(signal.SIGINT, config=True,
132 132 help="signal to use for stopping processes.")
133 133
134 134 aliases = Dict(stop_aliases)
135 135
136 136 def start(self):
137 137 """Start the app for the stop subcommand."""
138 138 try:
139 139 pid = self.get_pid_from_file()
140 140 except PIDFileError:
141 141 self.log.critical(
142 142 'Could not read pid file, cluster is probably not running.'
143 143 )
144 144 # Here I exit with a unusual exit status that other processes
145 145 # can watch for to learn how I existed.
146 146 self.remove_pid_file()
147 147 self.exit(ALREADY_STOPPED)
148 148
149 149 if not self.check_pid(pid):
150 150 self.log.critical(
151 151 'Cluster [pid=%r] is not running.' % pid
152 152 )
153 153 self.remove_pid_file()
154 154 # Here I exit with a unusual exit status that other processes
155 155 # can watch for to learn how I existed.
156 156 self.exit(ALREADY_STOPPED)
157 157
158 158 elif os.name=='posix':
159 159 sig = self.signal
160 160 self.log.info(
161 161 "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
162 162 )
163 163 try:
164 164 os.kill(pid, sig)
165 165 except OSError:
166 166 self.log.error("Stopping cluster failed, assuming already dead.",
167 167 exc_info=True)
168 168 self.remove_pid_file()
169 169 elif os.name=='nt':
170 170 try:
171 171 # kill the whole tree
172 172 p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE)
173 173 except (CalledProcessError, OSError):
174 174 self.log.error("Stopping cluster failed, assuming already dead.",
175 175 exc_info=True)
176 176 self.remove_pid_file()
177 177
178 178 engine_aliases = {}
179 179 engine_aliases.update(base_aliases)
180 180 engine_aliases.update(dict(
181 181 n='IPClusterEngines.n',
182 182 engines = 'IPClusterEngines.engine_launcher_class',
183 183 daemonize = 'IPClusterEngines.daemonize',
184 184 ))
185 185 engine_flags = {}
186 186 engine_flags.update(base_flags)
187 187
188 188 engine_flags.update(dict(
189 189 daemonize=(
190 190 {'IPClusterEngines' : {'daemonize' : True}},
191 191 """run the cluster into the background (not available on Windows)""",
192 192 )
193 193 ))
194 194 class IPClusterEngines(BaseParallelApplication):
195 195
196 196 name = u'ipcluster'
197 197 description = engines_help
198 198 usage = None
199 199 config_file_name = Unicode(default_config_file_name)
200 200 default_log_level = logging.INFO
201 201 classes = List()
202 202 def _classes_default(self):
203 203 from IPython.parallel.apps import launcher
204 204 launchers = launcher.all_launchers
205 205 eslaunchers = [ l for l in launchers if 'EngineSet' in l.__name__]
206 206 return [ProfileDir]+eslaunchers
207 207
208 208 n = Int(2, config=True,
209 209 help="The number of engines to start.")
210 210
211 211 engine_launcher_class = DottedObjectName('LocalEngineSetLauncher',
212 212 config=True,
213 213 help="The class for launching a set of Engines."
214 214 )
215 215 daemonize = Bool(False, config=True,
216 216 help="""Daemonize the ipcluster program. This implies --log-to-file.
217 217 Not available on Windows.
218 218 """)
219 219
220 220 def _daemonize_changed(self, name, old, new):
221 221 if new:
222 222 self.log_to_file = True
223 223
224 224 aliases = Dict(engine_aliases)
225 225 flags = Dict(engine_flags)
226 226 _stopping = False
227 227
228 228 def initialize(self, argv=None):
229 229 super(IPClusterEngines, self).initialize(argv)
230 230 self.init_signal()
231 231 self.init_launchers()
232 232
233 233 def init_launchers(self):
234 234 self.engine_launcher = self.build_launcher(self.engine_launcher_class)
235 235 self.engine_launcher.on_stop(lambda r: self.loop.stop())
236 236
237 237 def init_signal(self):
238 238 # Setup signals
239 239 signal.signal(signal.SIGINT, self.sigint_handler)
240 240
241 241 def build_launcher(self, clsname):
242 242 """import and instantiate a Launcher based on importstring"""
243 243 if '.' not in clsname:
244 244 # not a module, presume it's the raw name in apps.launcher
245 245 clsname = 'IPython.parallel.apps.launcher.'+clsname
246 246 # print repr(clsname)
247 247 klass = import_item(clsname)
248 248
249 249 launcher = klass(
250 250 work_dir=self.profile_dir.location, config=self.config, log=self.log
251 251 )
252 252 return launcher
253 253
254 254 def start_engines(self):
255 255 self.log.info("Starting %i engines"%self.n)
256 256 self.engine_launcher.start(
257 257 self.n,
258 258 self.profile_dir.location
259 259 )
260 260
261 261 def stop_engines(self):
262 262 self.log.info("Stopping Engines...")
263 263 if self.engine_launcher.running:
264 264 d = self.engine_launcher.stop()
265 265 return d
266 266 else:
267 267 return None
268 268
269 269 def stop_launchers(self, r=None):
270 270 if not self._stopping:
271 271 self._stopping = True
272 272 self.log.error("IPython cluster: stopping")
273 273 self.stop_engines()
274 274 # Wait a few seconds to let things shut down.
275 275 dc = ioloop.DelayedCallback(self.loop.stop, 4000, self.loop)
276 276 dc.start()
277 277
278 278 def sigint_handler(self, signum, frame):
279 279 self.log.debug("SIGINT received, stopping launchers...")
280 280 self.stop_launchers()
281 281
282 282 def start_logging(self):
283 283 # Remove old log files of the controller and engine
284 284 if self.clean_logs:
285 285 log_dir = self.profile_dir.log_dir
286 286 for f in os.listdir(log_dir):
287 287 if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f):
288 288 os.remove(os.path.join(log_dir, f))
289 289 # This will remove old log files for ipcluster itself
290 290 # super(IPBaseParallelApplication, self).start_logging()
291 291
292 292 def start(self):
293 293 """Start the app for the engines subcommand."""
294 294 self.log.info("IPython cluster: started")
295 295 # First see if the cluster is already running
296 296
297 297 # Now log and daemonize
298 298 self.log.info(
299 299 'Starting engines with [daemon=%r]' % self.daemonize
300 300 )
301 301 # TODO: Get daemonize working on Windows or as a Windows Server.
302 302 if self.daemonize:
303 303 if os.name=='posix':
304 304 daemonize()
305 305
306 306 dc = ioloop.DelayedCallback(self.start_engines, 0, self.loop)
307 307 dc.start()
308 308 # Now write the new pid file AFTER our new forked pid is active.
309 309 # self.write_pid_file()
310 310 try:
311 311 self.loop.start()
312 312 except KeyboardInterrupt:
313 313 pass
314 314 except zmq.ZMQError as e:
315 315 if e.errno == errno.EINTR:
316 316 pass
317 317 else:
318 318 raise
319 319
320 320 start_aliases = {}
321 321 start_aliases.update(engine_aliases)
322 322 start_aliases.update(dict(
323 323 delay='IPClusterStart.delay',
324 clean_logs='IPClusterStart.clean_logs',
325 324 controller = 'IPClusterStart.controller_launcher_class',
326 325 ))
326 start_aliases['clean-logs'] = 'IPClusterStart.clean_logs'
327 327
328 328 class IPClusterStart(IPClusterEngines):
329 329
330 330 name = u'ipcluster'
331 331 description = start_help
332 332 default_log_level = logging.INFO
333 333 auto_create = Bool(True, config=True,
334 334 help="whether to create the profile_dir if it doesn't exist")
335 335 classes = List()
336 336 def _classes_default(self,):
337 337 from IPython.parallel.apps import launcher
338 338 return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers
339 339
340 340 clean_logs = Bool(True, config=True,
341 341 help="whether to cleanup old logs before starting")
342 342
343 343 delay = CFloat(1., config=True,
344 344 help="delay (in s) between starting the controller and the engines")
345 345
346 346 controller_launcher_class = DottedObjectName('LocalControllerLauncher',
347 347 config=True,
348 348 help="The class for launching a Controller."
349 349 )
350 350 reset = Bool(False, config=True,
351 351 help="Whether to reset config files as part of '--create'."
352 352 )
353 353
354 354 # flags = Dict(flags)
355 355 aliases = Dict(start_aliases)
356 356
357 357 def init_launchers(self):
358 358 self.controller_launcher = self.build_launcher(self.controller_launcher_class)
359 359 self.engine_launcher = self.build_launcher(self.engine_launcher_class)
360 360 self.controller_launcher.on_stop(self.stop_launchers)
361 361
362 362 def start_controller(self):
363 363 self.controller_launcher.start(
364 364 self.profile_dir.location
365 365 )
366 366
367 367 def stop_controller(self):
368 368 # self.log.info("In stop_controller")
369 369 if self.controller_launcher and self.controller_launcher.running:
370 370 return self.controller_launcher.stop()
371 371
372 372 def stop_launchers(self, r=None):
373 373 if not self._stopping:
374 374 self.stop_controller()
375 375 super(IPClusterStart, self).stop_launchers()
376 376
377 377 def start(self):
378 378 """Start the app for the start subcommand."""
379 379 # First see if the cluster is already running
380 380 try:
381 381 pid = self.get_pid_from_file()
382 382 except PIDFileError:
383 383 pass
384 384 else:
385 385 if self.check_pid(pid):
386 386 self.log.critical(
387 387 'Cluster is already running with [pid=%s]. '
388 388 'use "ipcluster stop" to stop the cluster.' % pid
389 389 )
390 390 # Here I exit with a unusual exit status that other processes
391 391 # can watch for to learn how I existed.
392 392 self.exit(ALREADY_STARTED)
393 393 else:
394 394 self.remove_pid_file()
395 395
396 396
397 397 # Now log and daemonize
398 398 self.log.info(
399 399 'Starting ipcluster with [daemon=%r]' % self.daemonize
400 400 )
401 401 # TODO: Get daemonize working on Windows or as a Windows Server.
402 402 if self.daemonize:
403 403 if os.name=='posix':
404 404 daemonize()
405 405
406 406 dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop)
407 407 dc.start()
408 408 dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop)
409 409 dc.start()
410 410 # Now write the new pid file AFTER our new forked pid is active.
411 411 self.write_pid_file()
412 412 try:
413 413 self.loop.start()
414 414 except KeyboardInterrupt:
415 415 pass
416 416 except zmq.ZMQError as e:
417 417 if e.errno == errno.EINTR:
418 418 pass
419 419 else:
420 420 raise
421 421 finally:
422 422 self.remove_pid_file()
423 423
424 424 base='IPython.parallel.apps.ipclusterapp.IPCluster'
425 425
426 426 class IPClusterApp(Application):
427 427 name = u'ipcluster'
428 428 description = _description
429 429
430 430 subcommands = {
431 431 'start' : (base+'Start', start_help),
432 432 'stop' : (base+'Stop', stop_help),
433 433 'engines' : (base+'Engines', engines_help),
434 434 }
435 435
436 436 # no aliases or flags for parent App
437 437 aliases = Dict()
438 438 flags = Dict()
439 439
440 440 def start(self):
441 441 if self.subapp is None:
442 442 print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
443 443 print
444 444 self.print_description()
445 445 self.print_subcommands()
446 446 self.exit(1)
447 447 else:
448 448 return self.subapp.start()
449 449
450 450 def launch_new_instance():
451 451 """Create and run the IPython cluster."""
452 452 app = IPClusterApp.instance()
453 453 app.initialize()
454 454 app.start()
455 455
456 456
457 457 if __name__ == '__main__':
458 458 launch_new_instance()
459 459
@@ -1,422 +1,420 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 The IPython controller application.
5 5
6 6 Authors:
7 7
8 8 * Brian Granger
9 9 * MinRK
10 10
11 11 """
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Copyright (C) 2008-2011 The IPython Development Team
15 15 #
16 16 # Distributed under the terms of the BSD License. The full license is in
17 17 # the file COPYING, distributed as part of this software.
18 18 #-----------------------------------------------------------------------------
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Imports
22 22 #-----------------------------------------------------------------------------
23 23
24 24 from __future__ import with_statement
25 25
26 26 import os
27 27 import socket
28 28 import stat
29 29 import sys
30 30 import uuid
31 31
32 32 from multiprocessing import Process
33 33
34 34 import zmq
35 35 from zmq.devices import ProcessMonitoredQueue
36 36 from zmq.log.handlers import PUBHandler
37 37 from zmq.utils import jsonapi as json
38 38
39 39 from IPython.config.application import boolean_flag
40 40 from IPython.core.profiledir import ProfileDir
41 41
42 42 from IPython.parallel.apps.baseapp import (
43 43 BaseParallelApplication,
44 44 base_aliases,
45 45 base_flags,
46 46 )
47 47 from IPython.utils.importstring import import_item
48 48 from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict
49 49
50 50 # from IPython.parallel.controller.controller import ControllerFactory
51 51 from IPython.zmq.session import Session
52 52 from IPython.parallel.controller.heartmonitor import HeartMonitor
53 53 from IPython.parallel.controller.hub import HubFactory
54 54 from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
55 55 from IPython.parallel.controller.sqlitedb import SQLiteDB
56 56
57 57 from IPython.parallel.util import signal_children, split_url, asbytes
58 58
59 59 # conditional import of MongoDB backend class
60 60
61 61 try:
62 62 from IPython.parallel.controller.mongodb import MongoDB
63 63 except ImportError:
64 64 maybe_mongo = []
65 65 else:
66 66 maybe_mongo = [MongoDB]
67 67
68 68
69 69 #-----------------------------------------------------------------------------
70 70 # Module level variables
71 71 #-----------------------------------------------------------------------------
72 72
73 73
74 74 #: The default config file name for this application
75 75 default_config_file_name = u'ipcontroller_config.py'
76 76
77 77
78 78 _description = """Start the IPython controller for parallel computing.
79 79
80 80 The IPython controller provides a gateway between the IPython engines and
81 81 clients. The controller needs to be started before the engines and can be
82 82 configured using command line options or using a cluster directory. Cluster
83 83 directories contain config, log and security files and are usually located in
84 84 your ipython directory and named as "profile_name". See the `profile`
85 85 and `profile_dir` options for details.
86 86 """
87 87
88 88
89 89
90 90
91 91 #-----------------------------------------------------------------------------
92 92 # The main application
93 93 #-----------------------------------------------------------------------------
94 94 flags = {}
95 95 flags.update(base_flags)
96 96 flags.update({
97 97 'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
98 98 'Use threads instead of processes for the schedulers'),
99 99 'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
100 100 'use the SQLiteDB backend'),
101 101 'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
102 102 'use the MongoDB backend'),
103 103 'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
104 104 'use the in-memory DictDB backend'),
105 105 'reuse' : ({'IPControllerApp' : {'reuse_files' : True}},
106 106 'reuse existing json connection files')
107 107 })
108 108
109 109 flags.update(boolean_flag('secure', 'IPControllerApp.secure',
110 110 "Use HMAC digests for authentication of messages.",
111 111 "Don't authenticate messages."
112 112 ))
113 113 aliases = dict(
114 reuse_files = 'IPControllerApp.reuse_files',
115 114 secure = 'IPControllerApp.secure',
116 115 ssh = 'IPControllerApp.ssh_server',
117 use_threads = 'IPControllerApp.use_threads',
118 116 location = 'IPControllerApp.location',
119 117
120 118 ident = 'Session.session',
121 119 user = 'Session.username',
122 exec_key = 'Session.keyfile',
120 keyfile = 'Session.keyfile',
123 121
124 122 url = 'HubFactory.url',
125 123 ip = 'HubFactory.ip',
126 124 transport = 'HubFactory.transport',
127 125 port = 'HubFactory.regport',
128 126
129 127 ping = 'HeartMonitor.period',
130 128
131 129 scheme = 'TaskScheduler.scheme_name',
132 130 hwm = 'TaskScheduler.hwm',
133 131 )
134 132 aliases.update(base_aliases)
135 133
136 134 class IPControllerApp(BaseParallelApplication):
137 135
138 136 name = u'ipcontroller'
139 137 description = _description
140 138 config_file_name = Unicode(default_config_file_name)
141 139 classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo
142 140
143 141 # change default to True
144 142 auto_create = Bool(True, config=True,
145 143 help="""Whether to create profile dir if it doesn't exist.""")
146 144
147 145 reuse_files = Bool(False, config=True,
148 146 help='Whether to reuse existing json connection files.'
149 147 )
150 148 secure = Bool(True, config=True,
151 149 help='Whether to use HMAC digests for extra message authentication.'
152 150 )
153 151 ssh_server = Unicode(u'', config=True,
154 152 help="""ssh url for clients to use when connecting to the Controller
155 153 processes. It should be of the form: [user@]server[:port]. The
156 154 Controller's listening addresses must be accessible from the ssh server""",
157 155 )
158 156 location = Unicode(u'', config=True,
159 157 help="""The external IP or domain name of the Controller, used for disambiguating
160 158 engine and client connections.""",
161 159 )
162 160 import_statements = List([], config=True,
163 161 help="import statements to be run at startup. Necessary in some environments"
164 162 )
165 163
166 164 use_threads = Bool(False, config=True,
167 165 help='Use threads instead of processes for the schedulers',
168 166 )
169 167
170 168 # internal
171 169 children = List()
172 170 mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')
173 171
174 172 def _use_threads_changed(self, name, old, new):
175 173 self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
176 174
177 175 aliases = Dict(aliases)
178 176 flags = Dict(flags)
179 177
180 178
181 179 def save_connection_dict(self, fname, cdict):
182 180 """save a connection dict to json file."""
183 181 c = self.config
184 182 url = cdict['url']
185 183 location = cdict['location']
186 184 if not location:
187 185 try:
188 186 proto,ip,port = split_url(url)
189 187 except AssertionError:
190 188 pass
191 189 else:
192 190 location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
193 191 cdict['location'] = location
194 192 fname = os.path.join(self.profile_dir.security_dir, fname)
195 193 with open(fname, 'wb') as f:
196 194 f.write(json.dumps(cdict, indent=2))
197 195 os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
198 196
199 197 def load_config_from_json(self):
200 198 """load config from existing json connector files."""
201 199 c = self.config
202 200 # load from engine config
203 201 with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-engine.json')) as f:
204 202 cfg = json.loads(f.read())
205 203 key = c.Session.key = asbytes(cfg['exec_key'])
206 204 xport,addr = cfg['url'].split('://')
207 205 c.HubFactory.engine_transport = xport
208 206 ip,ports = addr.split(':')
209 207 c.HubFactory.engine_ip = ip
210 208 c.HubFactory.regport = int(ports)
211 209 self.location = cfg['location']
212 210 # load client config
213 211 with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-client.json')) as f:
214 212 cfg = json.loads(f.read())
215 213 assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
216 214 xport,addr = cfg['url'].split('://')
217 215 c.HubFactory.client_transport = xport
218 216 ip,ports = addr.split(':')
219 217 c.HubFactory.client_ip = ip
220 218 self.ssh_server = cfg['ssh']
221 219 assert int(ports) == c.HubFactory.regport, "regport mismatch"
222 220
223 221 def init_hub(self):
224 222 c = self.config
225 223
226 224 self.do_import_statements()
227 225 reusing = self.reuse_files
228 226 if reusing:
229 227 try:
230 228 self.load_config_from_json()
231 229 except (AssertionError,IOError):
232 230 reusing=False
233 231 # check again, because reusing may have failed:
234 232 if reusing:
235 233 pass
236 234 elif self.secure:
237 235 key = str(uuid.uuid4())
238 236 # keyfile = os.path.join(self.profile_dir.security_dir, self.exec_key)
239 237 # with open(keyfile, 'w') as f:
240 238 # f.write(key)
241 239 # os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
242 240 c.Session.key = asbytes(key)
243 241 else:
244 242 key = c.Session.key = b''
245 243
246 244 try:
247 245 self.factory = HubFactory(config=c, log=self.log)
248 246 # self.start_logging()
249 247 self.factory.init_hub()
250 248 except:
251 249 self.log.error("Couldn't construct the Controller", exc_info=True)
252 250 self.exit(1)
253 251
254 252 if not reusing:
255 253 # save to new json config files
256 254 f = self.factory
257 255 cdict = {'exec_key' : key,
258 256 'ssh' : self.ssh_server,
259 257 'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
260 258 'location' : self.location
261 259 }
262 260 self.save_connection_dict('ipcontroller-client.json', cdict)
263 261 edict = cdict
264 262 edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
265 263 self.save_connection_dict('ipcontroller-engine.json', edict)
266 264
267 265 #
268 266 def init_schedulers(self):
269 267 children = self.children
270 268 mq = import_item(str(self.mq_class))
271 269
272 270 hub = self.factory
273 271 # maybe_inproc = 'inproc://monitor' if self.use_threads else self.monitor_url
274 272 # IOPub relay (in a Process)
275 273 q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub')
276 274 q.bind_in(hub.client_info['iopub'])
277 275 q.bind_out(hub.engine_info['iopub'])
278 276 q.setsockopt_out(zmq.SUBSCRIBE, b'')
279 277 q.connect_mon(hub.monitor_url)
280 278 q.daemon=True
281 279 children.append(q)
282 280
283 281 # Multiplexer Queue (in a Process)
284 282 q = mq(zmq.XREP, zmq.XREP, zmq.PUB, b'in', b'out')
285 283 q.bind_in(hub.client_info['mux'])
286 284 q.setsockopt_in(zmq.IDENTITY, b'mux')
287 285 q.bind_out(hub.engine_info['mux'])
288 286 q.connect_mon(hub.monitor_url)
289 287 q.daemon=True
290 288 children.append(q)
291 289
292 290 # Control Queue (in a Process)
293 291 q = mq(zmq.XREP, zmq.XREP, zmq.PUB, b'incontrol', b'outcontrol')
294 292 q.bind_in(hub.client_info['control'])
295 293 q.setsockopt_in(zmq.IDENTITY, b'control')
296 294 q.bind_out(hub.engine_info['control'])
297 295 q.connect_mon(hub.monitor_url)
298 296 q.daemon=True
299 297 children.append(q)
300 298 try:
301 299 scheme = self.config.TaskScheduler.scheme_name
302 300 except AttributeError:
303 301 scheme = TaskScheduler.scheme_name.get_default_value()
304 302 # Task Queue (in a Process)
305 303 if scheme == 'pure':
306 304 self.log.warn("task::using pure XREQ Task scheduler")
307 305 q = mq(zmq.XREP, zmq.XREQ, zmq.PUB, b'intask', b'outtask')
308 306 # q.setsockopt_out(zmq.HWM, hub.hwm)
309 307 q.bind_in(hub.client_info['task'][1])
310 308 q.setsockopt_in(zmq.IDENTITY, b'task')
311 309 q.bind_out(hub.engine_info['task'])
312 310 q.connect_mon(hub.monitor_url)
313 311 q.daemon=True
314 312 children.append(q)
315 313 elif scheme == 'none':
316 314 self.log.warn("task::using no Task scheduler")
317 315
318 316 else:
319 317 self.log.info("task::using Python %s Task scheduler"%scheme)
320 318 sargs = (hub.client_info['task'][1], hub.engine_info['task'],
321 319 hub.monitor_url, hub.client_info['notification'])
322 320 kwargs = dict(logname='scheduler', loglevel=self.log_level,
323 321 log_url = self.log_url, config=dict(self.config))
324 322 if 'Process' in self.mq_class:
325 323 # run the Python scheduler in a Process
326 324 q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
327 325 q.daemon=True
328 326 children.append(q)
329 327 else:
330 328 # single-threaded Controller
331 329 kwargs['in_thread'] = True
332 330 launch_scheduler(*sargs, **kwargs)
333 331
334 332
335 333 def save_urls(self):
336 334 """save the registration urls to files."""
337 335 c = self.config
338 336
339 337 sec_dir = self.profile_dir.security_dir
340 338 cf = self.factory
341 339
342 340 with open(os.path.join(sec_dir, 'ipcontroller-engine.url'), 'w') as f:
343 341 f.write("%s://%s:%s"%(cf.engine_transport, cf.engine_ip, cf.regport))
344 342
345 343 with open(os.path.join(sec_dir, 'ipcontroller-client.url'), 'w') as f:
346 344 f.write("%s://%s:%s"%(cf.client_transport, cf.client_ip, cf.regport))
347 345
348 346
349 347 def do_import_statements(self):
350 348 statements = self.import_statements
351 349 for s in statements:
352 350 try:
353 351 self.log.msg("Executing statement: '%s'" % s)
354 352 exec s in globals(), locals()
355 353 except:
356 354 self.log.msg("Error running statement: %s" % s)
357 355
358 356 def forward_logging(self):
359 357 if self.log_url:
360 358 self.log.info("Forwarding logging to %s"%self.log_url)
361 359 context = zmq.Context.instance()
362 360 lsock = context.socket(zmq.PUB)
363 361 lsock.connect(self.log_url)
364 362 handler = PUBHandler(lsock)
365 363 self.log.removeHandler(self._log_handler)
366 364 handler.root_topic = 'controller'
367 365 handler.setLevel(self.log_level)
368 366 self.log.addHandler(handler)
369 367 self._log_handler = handler
370 368 # #
371 369
372 370 def initialize(self, argv=None):
373 371 super(IPControllerApp, self).initialize(argv)
374 372 self.forward_logging()
375 373 self.init_hub()
376 374 self.init_schedulers()
377 375
378 376 def start(self):
379 377 # Start the subprocesses:
380 378 self.factory.start()
381 379 child_procs = []
382 380 for child in self.children:
383 381 child.start()
384 382 if isinstance(child, ProcessMonitoredQueue):
385 383 child_procs.append(child.launcher)
386 384 elif isinstance(child, Process):
387 385 child_procs.append(child)
388 386 if child_procs:
389 387 signal_children(child_procs)
390 388
391 389 self.write_pid_file(overwrite=True)
392 390
393 391 try:
394 392 self.factory.loop.start()
395 393 except KeyboardInterrupt:
396 394 self.log.critical("Interrupted, Exiting...\n")
397 395
398 396
399 397
400 398 def launch_new_instance():
401 399 """Create and run the IPython controller"""
402 400 if sys.platform == 'win32':
403 401 # make sure we don't get called from a multiprocessing subprocess
404 402 # this can result in infinite Controllers being started on Windows
405 403 # which doesn't have a proper fork, so multiprocessing is wonky
406 404
407 405 # this only comes up when IPython has been installed using vanilla
408 406 # setuptools, and *not* distribute.
409 407 import multiprocessing
410 408 p = multiprocessing.current_process()
411 409 # the main process has name 'MainProcess'
412 410 # subprocesses will have names like 'Process-1'
413 411 if p.name != 'MainProcess':
414 412 # we are a subprocess, don't start another Controller!
415 413 return
416 414 app = IPControllerApp.instance()
417 415 app.initialize()
418 416 app.start()
419 417
420 418
421 419 if __name__ == '__main__':
422 420 launch_new_instance()
@@ -1,301 +1,301 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 The IPython engine application
5 5
6 6 Authors:
7 7
8 8 * Brian Granger
9 9 * MinRK
10 10
11 11 """
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Copyright (C) 2008-2011 The IPython Development Team
15 15 #
16 16 # Distributed under the terms of the BSD License. The full license is in
17 17 # the file COPYING, distributed as part of this software.
18 18 #-----------------------------------------------------------------------------
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Imports
22 22 #-----------------------------------------------------------------------------
23 23
24 24 import json
25 25 import os
26 26 import sys
27 27 import time
28 28
29 29 import zmq
30 30 from zmq.eventloop import ioloop
31 31
32 32 from IPython.core.profiledir import ProfileDir
33 33 from IPython.parallel.apps.baseapp import (
34 34 BaseParallelApplication,
35 35 base_aliases,
36 36 base_flags,
37 37 )
38 38 from IPython.zmq.log import EnginePUBHandler
39 39
40 40 from IPython.config.configurable import Configurable
41 41 from IPython.zmq.session import Session
42 42 from IPython.parallel.engine.engine import EngineFactory
43 43 from IPython.parallel.engine.streamkernel import Kernel
44 44 from IPython.parallel.util import disambiguate_url, asbytes
45 45
46 46 from IPython.utils.importstring import import_item
47 47 from IPython.utils.traitlets import Bool, Unicode, Dict, List, Float
48 48
49 49
50 50 #-----------------------------------------------------------------------------
51 51 # Module level variables
52 52 #-----------------------------------------------------------------------------
53 53
54 54 #: The default config file name for this application
55 55 default_config_file_name = u'ipengine_config.py'
56 56
57 57 _description = """Start an IPython engine for parallel computing.
58 58
59 59 IPython engines run in parallel and perform computations on behalf of a client
60 60 and controller. A controller needs to be started before the engines. The
61 61 engine can be configured using command line options or using a cluster
62 62 directory. Cluster directories contain config, log and security files and are
63 63 usually located in your ipython directory and named as "profile_name".
64 64 See the `profile` and `profile_dir` options for details.
65 65 """
66 66
67 67
68 68 #-----------------------------------------------------------------------------
69 69 # MPI configuration
70 70 #-----------------------------------------------------------------------------
71 71
72 72 mpi4py_init = """from mpi4py import MPI as mpi
73 73 mpi.size = mpi.COMM_WORLD.Get_size()
74 74 mpi.rank = mpi.COMM_WORLD.Get_rank()
75 75 """
76 76
77 77
78 78 pytrilinos_init = """from PyTrilinos import Epetra
79 79 class SimpleStruct:
80 80 pass
81 81 mpi = SimpleStruct()
82 82 mpi.rank = 0
83 83 mpi.size = 0
84 84 """
85 85
86 86 class MPI(Configurable):
87 87 """Configurable for MPI initialization"""
88 88 use = Unicode('', config=True,
89 89 help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).'
90 90 )
91 91
92 92 def _on_use_changed(self, old, new):
93 93 # load default init script if it's not set
94 94 if not self.init_script:
95 95 self.init_script = self.default_inits.get(new, '')
96 96
97 97 init_script = Unicode('', config=True,
98 98 help="Initialization code for MPI")
99 99
100 100 default_inits = Dict({'mpi4py' : mpi4py_init, 'pytrilinos':pytrilinos_init},
101 101 config=True)
102 102
103 103
104 104 #-----------------------------------------------------------------------------
105 105 # Main application
106 106 #-----------------------------------------------------------------------------
107 107 aliases = dict(
108 108 file = 'IPEngineApp.url_file',
109 109 c = 'IPEngineApp.startup_command',
110 110 s = 'IPEngineApp.startup_script',
111 111
112 112 ident = 'Session.session',
113 113 user = 'Session.username',
114 exec_key = 'Session.keyfile',
114 keyfile = 'Session.keyfile',
115 115
116 116 url = 'EngineFactory.url',
117 117 ip = 'EngineFactory.ip',
118 118 transport = 'EngineFactory.transport',
119 119 port = 'EngineFactory.regport',
120 120 location = 'EngineFactory.location',
121 121
122 122 timeout = 'EngineFactory.timeout',
123 123
124 124 mpi = 'MPI.use',
125 125
126 126 )
127 127 aliases.update(base_aliases)
128 128
129 129 class IPEngineApp(BaseParallelApplication):
130 130
131 131 name = Unicode(u'ipengine')
132 132 description = Unicode(_description)
133 133 config_file_name = Unicode(default_config_file_name)
134 134 classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI])
135 135
136 136 startup_script = Unicode(u'', config=True,
137 137 help='specify a script to be run at startup')
138 138 startup_command = Unicode('', config=True,
139 139 help='specify a command to be run at startup')
140 140
141 141 url_file = Unicode(u'', config=True,
142 142 help="""The full location of the file containing the connection information for
143 143 the controller. If this is not given, the file must be in the
144 144 security directory of the cluster directory. This location is
145 145 resolved using the `profile` or `profile_dir` options.""",
146 146 )
147 147 wait_for_url_file = Float(5, config=True,
148 148 help="""The maximum number of seconds to wait for url_file to exist.
149 149 This is useful for batch-systems and shared-filesystems where the
150 150 controller and engine are started at the same time and it
151 151 may take a moment for the controller to write the connector files.""")
152 152
153 153 url_file_name = Unicode(u'ipcontroller-engine.json')
154 154 log_url = Unicode('', config=True,
155 155 help="""The URL for the iploggerapp instance, for forwarding
156 156 logging to a central location.""")
157 157
158 158 aliases = Dict(aliases)
159 159
160 160 # def find_key_file(self):
161 161 # """Set the key file.
162 162 #
163 163 # Here we don't try to actually see if it exists for is valid as that
164 164 # is hadled by the connection logic.
165 165 # """
166 166 # config = self.master_config
167 167 # # Find the actual controller key file
168 168 # if not config.Global.key_file:
169 169 # try_this = os.path.join(
170 170 # config.Global.profile_dir,
171 171 # config.Global.security_dir,
172 172 # config.Global.key_file_name
173 173 # )
174 174 # config.Global.key_file = try_this
175 175
176 176 def find_url_file(self):
177 177 """Set the url file.
178 178
179 179 Here we don't try to actually see if it exists for is valid as that
180 180 is hadled by the connection logic.
181 181 """
182 182 config = self.config
183 183 # Find the actual controller key file
184 184 if not self.url_file:
185 185 self.url_file = os.path.join(
186 186 self.profile_dir.security_dir,
187 187 self.url_file_name
188 188 )
189 189 def init_engine(self):
190 190 # This is the working dir by now.
191 191 sys.path.insert(0, '')
192 192 config = self.config
193 193 # print config
194 194 self.find_url_file()
195 195
196 196 # was the url manually specified?
197 197 keys = set(self.config.EngineFactory.keys())
198 198 keys = keys.union(set(self.config.RegistrationFactory.keys()))
199 199
200 200 if keys.intersection(set(['ip', 'url', 'port'])):
201 201 # Connection info was specified, don't wait for the file
202 202 url_specified = True
203 203 self.wait_for_url_file = 0
204 204 else:
205 205 url_specified = False
206 206
207 207 if self.wait_for_url_file and not os.path.exists(self.url_file):
208 208 self.log.warn("url_file %r not found"%self.url_file)
209 209 self.log.warn("Waiting up to %.1f seconds for it to arrive."%self.wait_for_url_file)
210 210 tic = time.time()
211 211 while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file):
212 212 # wait for url_file to exist, for up to 10 seconds
213 213 time.sleep(0.1)
214 214
215 215 if os.path.exists(self.url_file):
216 216 self.log.info("Loading url_file %r"%self.url_file)
217 217 with open(self.url_file) as f:
218 218 d = json.loads(f.read())
219 219 if d['exec_key']:
220 220 config.Session.key = asbytes(d['exec_key'])
221 221 d['url'] = disambiguate_url(d['url'], d['location'])
222 222 config.EngineFactory.url = d['url']
223 223 config.EngineFactory.location = d['location']
224 224 elif not url_specified:
225 225 self.log.critical("Fatal: url file never arrived: %s"%self.url_file)
226 226 self.exit(1)
227 227
228 228
229 229 try:
230 230 exec_lines = config.Kernel.exec_lines
231 231 except AttributeError:
232 232 config.Kernel.exec_lines = []
233 233 exec_lines = config.Kernel.exec_lines
234 234
235 235 if self.startup_script:
236 236 enc = sys.getfilesystemencoding() or 'utf8'
237 237 cmd="execfile(%r)"%self.startup_script.encode(enc)
238 238 exec_lines.append(cmd)
239 239 if self.startup_command:
240 240 exec_lines.append(self.startup_command)
241 241
242 242 # Create the underlying shell class and Engine
243 243 # shell_class = import_item(self.master_config.Global.shell_class)
244 244 # print self.config
245 245 try:
246 246 self.engine = EngineFactory(config=config, log=self.log)
247 247 except:
248 248 self.log.error("Couldn't start the Engine", exc_info=True)
249 249 self.exit(1)
250 250
251 251 def forward_logging(self):
252 252 if self.log_url:
253 253 self.log.info("Forwarding logging to %s"%self.log_url)
254 254 context = self.engine.context
255 255 lsock = context.socket(zmq.PUB)
256 256 lsock.connect(self.log_url)
257 257 self.log.removeHandler(self._log_handler)
258 258 handler = EnginePUBHandler(self.engine, lsock)
259 259 handler.setLevel(self.log_level)
260 260 self.log.addHandler(handler)
261 261 self._log_handler = handler
262 262 #
263 263 def init_mpi(self):
264 264 global mpi
265 265 self.mpi = MPI(config=self.config)
266 266
267 267 mpi_import_statement = self.mpi.init_script
268 268 if mpi_import_statement:
269 269 try:
270 270 self.log.info("Initializing MPI:")
271 271 self.log.info(mpi_import_statement)
272 272 exec mpi_import_statement in globals()
273 273 except:
274 274 mpi = None
275 275 else:
276 276 mpi = None
277 277
278 278 def initialize(self, argv=None):
279 279 super(IPEngineApp, self).initialize(argv)
280 280 self.init_mpi()
281 281 self.init_engine()
282 282 self.forward_logging()
283 283
284 284 def start(self):
285 285 self.engine.start()
286 286 try:
287 287 self.engine.loop.start()
288 288 except KeyboardInterrupt:
289 289 self.log.critical("Engine Interrupted, shutting down...\n")
290 290
291 291
292 292 def launch_new_instance():
293 293 """Create and run the IPython engine"""
294 294 app = IPEngineApp.instance()
295 295 app.initialize()
296 296 app.start()
297 297
298 298
299 299 if __name__ == '__main__':
300 300 launch_new_instance()
301 301
@@ -1,1065 +1,1065 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 Facilities for launching IPython processes asynchronously.
5 5
6 6 Authors:
7 7
8 8 * Brian Granger
9 9 * MinRK
10 10 """
11 11
12 12 #-----------------------------------------------------------------------------
13 13 # Copyright (C) 2008-2011 The IPython Development Team
14 14 #
15 15 # Distributed under the terms of the BSD License. The full license is in
16 16 # the file COPYING, distributed as part of this software.
17 17 #-----------------------------------------------------------------------------
18 18
19 19 #-----------------------------------------------------------------------------
20 20 # Imports
21 21 #-----------------------------------------------------------------------------
22 22
23 23 import copy
24 24 import logging
25 25 import os
26 26 import re
27 27 import stat
28 28
29 29 # signal imports, handling various platforms, versions
30 30
31 31 from signal import SIGINT, SIGTERM
32 32 try:
33 33 from signal import SIGKILL
34 34 except ImportError:
35 35 # Windows
36 36 SIGKILL=SIGTERM
37 37
38 38 try:
39 39 # Windows >= 2.7, 3.2
40 40 from signal import CTRL_C_EVENT as SIGINT
41 41 except ImportError:
42 42 pass
43 43
44 44 from subprocess import Popen, PIPE, STDOUT
45 45 try:
46 46 from subprocess import check_output
47 47 except ImportError:
48 48 # pre-2.7, define check_output with Popen
49 49 def check_output(*args, **kwargs):
50 50 kwargs.update(dict(stdout=PIPE))
51 51 p = Popen(*args, **kwargs)
52 52 out,err = p.communicate()
53 53 return out
54 54
55 55 from zmq.eventloop import ioloop
56 56
57 57 from IPython.config.application import Application
58 58 from IPython.config.configurable import LoggingConfigurable
59 59 from IPython.utils.text import EvalFormatter
60 60 from IPython.utils.traitlets import Any, Int, List, Unicode, Dict, Instance
61 61 from IPython.utils.path import get_ipython_module_path
62 62 from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError
63 63
64 64 from .win32support import forward_read_events
65 65
66 66 from .winhpcjob import IPControllerTask, IPEngineTask, IPControllerJob, IPEngineSetJob
67 67
68 68 WINDOWS = os.name == 'nt'
69 69
70 70 #-----------------------------------------------------------------------------
71 71 # Paths to the kernel apps
72 72 #-----------------------------------------------------------------------------
73 73
74 74
75 75 ipcluster_cmd_argv = pycmd2argv(get_ipython_module_path(
76 76 'IPython.parallel.apps.ipclusterapp'
77 77 ))
78 78
79 79 ipengine_cmd_argv = pycmd2argv(get_ipython_module_path(
80 80 'IPython.parallel.apps.ipengineapp'
81 81 ))
82 82
83 83 ipcontroller_cmd_argv = pycmd2argv(get_ipython_module_path(
84 84 'IPython.parallel.apps.ipcontrollerapp'
85 85 ))
86 86
87 87 #-----------------------------------------------------------------------------
88 88 # Base launchers and errors
89 89 #-----------------------------------------------------------------------------
90 90
91 91
92 92 class LauncherError(Exception):
93 93 pass
94 94
95 95
96 96 class ProcessStateError(LauncherError):
97 97 pass
98 98
99 99
100 100 class UnknownStatus(LauncherError):
101 101 pass
102 102
103 103
104 104 class BaseLauncher(LoggingConfigurable):
105 105 """An asbtraction for starting, stopping and signaling a process."""
106 106
107 107 # In all of the launchers, the work_dir is where child processes will be
108 108 # run. This will usually be the profile_dir, but may not be. any work_dir
109 109 # passed into the __init__ method will override the config value.
110 110 # This should not be used to set the work_dir for the actual engine
111 111 # and controller. Instead, use their own config files or the
112 112 # controller_args, engine_args attributes of the launchers to add
113 113 # the work_dir option.
114 114 work_dir = Unicode(u'.')
115 115 loop = Instance('zmq.eventloop.ioloop.IOLoop')
116 116
117 117 start_data = Any()
118 118 stop_data = Any()
119 119
120 120 def _loop_default(self):
121 121 return ioloop.IOLoop.instance()
122 122
123 123 def __init__(self, work_dir=u'.', config=None, **kwargs):
124 124 super(BaseLauncher, self).__init__(work_dir=work_dir, config=config, **kwargs)
125 125 self.state = 'before' # can be before, running, after
126 126 self.stop_callbacks = []
127 127 self.start_data = None
128 128 self.stop_data = None
129 129
130 130 @property
131 131 def args(self):
132 132 """A list of cmd and args that will be used to start the process.
133 133
134 134 This is what is passed to :func:`spawnProcess` and the first element
135 135 will be the process name.
136 136 """
137 137 return self.find_args()
138 138
139 139 def find_args(self):
140 140 """The ``.args`` property calls this to find the args list.
141 141
142 142 Subcommand should implement this to construct the cmd and args.
143 143 """
144 144 raise NotImplementedError('find_args must be implemented in a subclass')
145 145
146 146 @property
147 147 def arg_str(self):
148 148 """The string form of the program arguments."""
149 149 return ' '.join(self.args)
150 150
151 151 @property
152 152 def running(self):
153 153 """Am I running."""
154 154 if self.state == 'running':
155 155 return True
156 156 else:
157 157 return False
158 158
159 159 def start(self):
160 160 """Start the process."""
161 161 raise NotImplementedError('start must be implemented in a subclass')
162 162
163 163 def stop(self):
164 164 """Stop the process and notify observers of stopping.
165 165
166 166 This method will return None immediately.
167 167 To observe the actual process stopping, see :meth:`on_stop`.
168 168 """
169 169 raise NotImplementedError('stop must be implemented in a subclass')
170 170
171 171 def on_stop(self, f):
172 172 """Register a callback to be called with this Launcher's stop_data
173 173 when the process actually finishes.
174 174 """
175 175 if self.state=='after':
176 176 return f(self.stop_data)
177 177 else:
178 178 self.stop_callbacks.append(f)
179 179
180 180 def notify_start(self, data):
181 181 """Call this to trigger startup actions.
182 182
183 183 This logs the process startup and sets the state to 'running'. It is
184 184 a pass-through so it can be used as a callback.
185 185 """
186 186
187 187 self.log.info('Process %r started: %r' % (self.args[0], data))
188 188 self.start_data = data
189 189 self.state = 'running'
190 190 return data
191 191
192 192 def notify_stop(self, data):
193 193 """Call this to trigger process stop actions.
194 194
195 195 This logs the process stopping and sets the state to 'after'. Call
196 196 this to trigger callbacks registered via :meth:`on_stop`."""
197 197
198 198 self.log.info('Process %r stopped: %r' % (self.args[0], data))
199 199 self.stop_data = data
200 200 self.state = 'after'
201 201 for i in range(len(self.stop_callbacks)):
202 202 d = self.stop_callbacks.pop()
203 203 d(data)
204 204 return data
205 205
206 206 def signal(self, sig):
207 207 """Signal the process.
208 208
209 209 Parameters
210 210 ----------
211 211 sig : str or int
212 212 'KILL', 'INT', etc., or any signal number
213 213 """
214 214 raise NotImplementedError('signal must be implemented in a subclass')
215 215
216 216
217 217 #-----------------------------------------------------------------------------
218 218 # Local process launchers
219 219 #-----------------------------------------------------------------------------
220 220
221 221
222 222 class LocalProcessLauncher(BaseLauncher):
223 223 """Start and stop an external process in an asynchronous manner.
224 224
225 225 This will launch the external process with a working directory of
226 226 ``self.work_dir``.
227 227 """
228 228
229 229 # This is used to to construct self.args, which is passed to
230 230 # spawnProcess.
231 231 cmd_and_args = List([])
232 232 poll_frequency = Int(100) # in ms
233 233
234 234 def __init__(self, work_dir=u'.', config=None, **kwargs):
235 235 super(LocalProcessLauncher, self).__init__(
236 236 work_dir=work_dir, config=config, **kwargs
237 237 )
238 238 self.process = None
239 239 self.poller = None
240 240
241 241 def find_args(self):
242 242 return self.cmd_and_args
243 243
244 244 def start(self):
245 245 if self.state == 'before':
246 246 self.process = Popen(self.args,
247 247 stdout=PIPE,stderr=PIPE,stdin=PIPE,
248 248 env=os.environ,
249 249 cwd=self.work_dir
250 250 )
251 251 if WINDOWS:
252 252 self.stdout = forward_read_events(self.process.stdout)
253 253 self.stderr = forward_read_events(self.process.stderr)
254 254 else:
255 255 self.stdout = self.process.stdout.fileno()
256 256 self.stderr = self.process.stderr.fileno()
257 257 self.loop.add_handler(self.stdout, self.handle_stdout, self.loop.READ)
258 258 self.loop.add_handler(self.stderr, self.handle_stderr, self.loop.READ)
259 259 self.poller = ioloop.PeriodicCallback(self.poll, self.poll_frequency, self.loop)
260 260 self.poller.start()
261 261 self.notify_start(self.process.pid)
262 262 else:
263 263 s = 'The process was already started and has state: %r' % self.state
264 264 raise ProcessStateError(s)
265 265
266 266 def stop(self):
267 267 return self.interrupt_then_kill()
268 268
269 269 def signal(self, sig):
270 270 if self.state == 'running':
271 271 if WINDOWS and sig != SIGINT:
272 272 # use Windows tree-kill for better child cleanup
273 273 check_output(['taskkill', '-pid', str(self.process.pid), '-t', '-f'])
274 274 else:
275 275 self.process.send_signal(sig)
276 276
277 277 def interrupt_then_kill(self, delay=2.0):
278 278 """Send INT, wait a delay and then send KILL."""
279 279 try:
280 280 self.signal(SIGINT)
281 281 except Exception:
282 282 self.log.debug("interrupt failed")
283 283 pass
284 284 self.killer = ioloop.DelayedCallback(lambda : self.signal(SIGKILL), delay*1000, self.loop)
285 285 self.killer.start()
286 286
287 287 # callbacks, etc:
288 288
289 289 def handle_stdout(self, fd, events):
290 290 if WINDOWS:
291 291 line = self.stdout.recv()
292 292 else:
293 293 line = self.process.stdout.readline()
294 294 # a stopped process will be readable but return empty strings
295 295 if line:
296 296 self.log.info(line[:-1])
297 297 else:
298 298 self.poll()
299 299
300 300 def handle_stderr(self, fd, events):
301 301 if WINDOWS:
302 302 line = self.stderr.recv()
303 303 else:
304 304 line = self.process.stderr.readline()
305 305 # a stopped process will be readable but return empty strings
306 306 if line:
307 307 self.log.error(line[:-1])
308 308 else:
309 309 self.poll()
310 310
311 311 def poll(self):
312 312 status = self.process.poll()
313 313 if status is not None:
314 314 self.poller.stop()
315 315 self.loop.remove_handler(self.stdout)
316 316 self.loop.remove_handler(self.stderr)
317 317 self.notify_stop(dict(exit_code=status, pid=self.process.pid))
318 318 return status
319 319
320 320 class LocalControllerLauncher(LocalProcessLauncher):
321 321 """Launch a controller as a regular external process."""
322 322
323 323 controller_cmd = List(ipcontroller_cmd_argv, config=True,
324 324 help="""Popen command to launch ipcontroller.""")
325 325 # Command line arguments to ipcontroller.
326 controller_args = List(['--log-to-file','--log_level=%i'%logging.INFO], config=True,
326 controller_args = List(['--log-to-file','--log-level=%i'%logging.INFO], config=True,
327 327 help="""command-line args to pass to ipcontroller""")
328 328
329 329 def find_args(self):
330 330 return self.controller_cmd + self.controller_args
331 331
332 332 def start(self, profile_dir):
333 333 """Start the controller by profile_dir."""
334 self.controller_args.extend(['--profile_dir=%s'%profile_dir])
334 self.controller_args.extend(['--profile-dir=%s'%profile_dir])
335 335 self.profile_dir = unicode(profile_dir)
336 336 self.log.info("Starting LocalControllerLauncher: %r" % self.args)
337 337 return super(LocalControllerLauncher, self).start()
338 338
339 339
340 340 class LocalEngineLauncher(LocalProcessLauncher):
341 341 """Launch a single engine as a regular externall process."""
342 342
343 343 engine_cmd = List(ipengine_cmd_argv, config=True,
344 344 help="""command to launch the Engine.""")
345 345 # Command line arguments for ipengine.
346 engine_args = List(['--log-to-file','--log_level=%i'%logging.INFO], config=True,
346 engine_args = List(['--log-to-file','--log-level=%i'%logging.INFO], config=True,
347 347 help="command-line arguments to pass to ipengine"
348 348 )
349 349
350 350 def find_args(self):
351 351 return self.engine_cmd + self.engine_args
352 352
353 353 def start(self, profile_dir):
354 354 """Start the engine by profile_dir."""
355 self.engine_args.extend(['--profile_dir=%s'%profile_dir])
355 self.engine_args.extend(['--profile-dir=%s'%profile_dir])
356 356 self.profile_dir = unicode(profile_dir)
357 357 return super(LocalEngineLauncher, self).start()
358 358
359 359
360 360 class LocalEngineSetLauncher(BaseLauncher):
361 361 """Launch a set of engines as regular external processes."""
362 362
363 363 # Command line arguments for ipengine.
364 364 engine_args = List(
365 ['--log-to-file','--log_level=%i'%logging.INFO], config=True,
365 ['--log-to-file','--log-level=%i'%logging.INFO], config=True,
366 366 help="command-line arguments to pass to ipengine"
367 367 )
368 368 # launcher class
369 369 launcher_class = LocalEngineLauncher
370 370
371 371 launchers = Dict()
372 372 stop_data = Dict()
373 373
374 374 def __init__(self, work_dir=u'.', config=None, **kwargs):
375 375 super(LocalEngineSetLauncher, self).__init__(
376 376 work_dir=work_dir, config=config, **kwargs
377 377 )
378 378 self.stop_data = {}
379 379
380 380 def start(self, n, profile_dir):
381 381 """Start n engines by profile or profile_dir."""
382 382 self.profile_dir = unicode(profile_dir)
383 383 dlist = []
384 384 for i in range(n):
385 385 el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log)
386 386 # Copy the engine args over to each engine launcher.
387 387 el.engine_args = copy.deepcopy(self.engine_args)
388 388 el.on_stop(self._notice_engine_stopped)
389 389 d = el.start(profile_dir)
390 390 if i==0:
391 391 self.log.info("Starting LocalEngineSetLauncher: %r" % el.args)
392 392 self.launchers[i] = el
393 393 dlist.append(d)
394 394 self.notify_start(dlist)
395 395 # The consumeErrors here could be dangerous
396 396 # dfinal = gatherBoth(dlist, consumeErrors=True)
397 397 # dfinal.addCallback(self.notify_start)
398 398 return dlist
399 399
400 400 def find_args(self):
401 401 return ['engine set']
402 402
403 403 def signal(self, sig):
404 404 dlist = []
405 405 for el in self.launchers.itervalues():
406 406 d = el.signal(sig)
407 407 dlist.append(d)
408 408 # dfinal = gatherBoth(dlist, consumeErrors=True)
409 409 return dlist
410 410
411 411 def interrupt_then_kill(self, delay=1.0):
412 412 dlist = []
413 413 for el in self.launchers.itervalues():
414 414 d = el.interrupt_then_kill(delay)
415 415 dlist.append(d)
416 416 # dfinal = gatherBoth(dlist, consumeErrors=True)
417 417 return dlist
418 418
419 419 def stop(self):
420 420 return self.interrupt_then_kill()
421 421
422 422 def _notice_engine_stopped(self, data):
423 423 pid = data['pid']
424 424 for idx,el in self.launchers.iteritems():
425 425 if el.process.pid == pid:
426 426 break
427 427 self.launchers.pop(idx)
428 428 self.stop_data[idx] = data
429 429 if not self.launchers:
430 430 self.notify_stop(self.stop_data)
431 431
432 432
433 433 #-----------------------------------------------------------------------------
434 434 # MPIExec launchers
435 435 #-----------------------------------------------------------------------------
436 436
437 437
438 438 class MPIExecLauncher(LocalProcessLauncher):
439 439 """Launch an external process using mpiexec."""
440 440
441 441 mpi_cmd = List(['mpiexec'], config=True,
442 442 help="The mpiexec command to use in starting the process."
443 443 )
444 444 mpi_args = List([], config=True,
445 445 help="The command line arguments to pass to mpiexec."
446 446 )
447 447 program = List(['date'], config=True,
448 448 help="The program to start via mpiexec.")
449 449 program_args = List([], config=True,
450 450 help="The command line argument to the program."
451 451 )
452 452 n = Int(1)
453 453
454 454 def find_args(self):
455 455 """Build self.args using all the fields."""
456 456 return self.mpi_cmd + ['-n', str(self.n)] + self.mpi_args + \
457 457 self.program + self.program_args
458 458
459 459 def start(self, n):
460 460 """Start n instances of the program using mpiexec."""
461 461 self.n = n
462 462 return super(MPIExecLauncher, self).start()
463 463
464 464
465 465 class MPIExecControllerLauncher(MPIExecLauncher):
466 466 """Launch a controller using mpiexec."""
467 467
468 468 controller_cmd = List(ipcontroller_cmd_argv, config=True,
469 469 help="Popen command to launch the Contropper"
470 470 )
471 controller_args = List(['--log-to-file','--log_level=%i'%logging.INFO], config=True,
471 controller_args = List(['--log-to-file','--log-level=%i'%logging.INFO], config=True,
472 472 help="Command line arguments to pass to ipcontroller."
473 473 )
474 474 n = Int(1)
475 475
476 476 def start(self, profile_dir):
477 477 """Start the controller by profile_dir."""
478 self.controller_args.extend(['--profile_dir=%s'%profile_dir])
478 self.controller_args.extend(['--profile-dir=%s'%profile_dir])
479 479 self.profile_dir = unicode(profile_dir)
480 480 self.log.info("Starting MPIExecControllerLauncher: %r" % self.args)
481 481 return super(MPIExecControllerLauncher, self).start(1)
482 482
483 483 def find_args(self):
484 484 return self.mpi_cmd + ['-n', str(self.n)] + self.mpi_args + \
485 485 self.controller_cmd + self.controller_args
486 486
487 487
488 488 class MPIExecEngineSetLauncher(MPIExecLauncher):
489 489
490 490 program = List(ipengine_cmd_argv, config=True,
491 491 help="Popen command for ipengine"
492 492 )
493 493 program_args = List(
494 ['--log-to-file','--log_level=%i'%logging.INFO], config=True,
494 ['--log-to-file','--log-level=%i'%logging.INFO], config=True,
495 495 help="Command line arguments for ipengine."
496 496 )
497 497 n = Int(1)
498 498
499 499 def start(self, n, profile_dir):
500 500 """Start n engines by profile or profile_dir."""
501 self.program_args.extend(['--profile_dir=%s'%profile_dir])
501 self.program_args.extend(['--profile-dir=%s'%profile_dir])
502 502 self.profile_dir = unicode(profile_dir)
503 503 self.n = n
504 504 self.log.info('Starting MPIExecEngineSetLauncher: %r' % self.args)
505 505 return super(MPIExecEngineSetLauncher, self).start(n)
506 506
507 507 #-----------------------------------------------------------------------------
508 508 # SSH launchers
509 509 #-----------------------------------------------------------------------------
510 510
511 511 # TODO: Get SSH Launcher back to level of sshx in 0.10.2
512 512
513 513 class SSHLauncher(LocalProcessLauncher):
514 514 """A minimal launcher for ssh.
515 515
516 516 To be useful this will probably have to be extended to use the ``sshx``
517 517 idea for environment variables. There could be other things this needs
518 518 as well.
519 519 """
520 520
521 521 ssh_cmd = List(['ssh'], config=True,
522 522 help="command for starting ssh")
523 523 ssh_args = List(['-tt'], config=True,
524 524 help="args to pass to ssh")
525 525 program = List(['date'], config=True,
526 526 help="Program to launch via ssh")
527 527 program_args = List([], config=True,
528 528 help="args to pass to remote program")
529 529 hostname = Unicode('', config=True,
530 530 help="hostname on which to launch the program")
531 531 user = Unicode('', config=True,
532 532 help="username for ssh")
533 533 location = Unicode('', config=True,
534 534 help="user@hostname location for ssh in one setting")
535 535
536 536 def _hostname_changed(self, name, old, new):
537 537 if self.user:
538 538 self.location = u'%s@%s' % (self.user, new)
539 539 else:
540 540 self.location = new
541 541
542 542 def _user_changed(self, name, old, new):
543 543 self.location = u'%s@%s' % (new, self.hostname)
544 544
545 545 def find_args(self):
546 546 return self.ssh_cmd + self.ssh_args + [self.location] + \
547 547 self.program + self.program_args
548 548
549 549 def start(self, profile_dir, hostname=None, user=None):
550 550 self.profile_dir = unicode(profile_dir)
551 551 if hostname is not None:
552 552 self.hostname = hostname
553 553 if user is not None:
554 554 self.user = user
555 555
556 556 return super(SSHLauncher, self).start()
557 557
558 558 def signal(self, sig):
559 559 if self.state == 'running':
560 560 # send escaped ssh connection-closer
561 561 self.process.stdin.write('~.')
562 562 self.process.stdin.flush()
563 563
564 564
565 565
566 566 class SSHControllerLauncher(SSHLauncher):
567 567
568 568 program = List(ipcontroller_cmd_argv, config=True,
569 569 help="remote ipcontroller command.")
570 program_args = List(['--reuse-files', '--log-to-file','--log_level=%i'%logging.INFO], config=True,
570 program_args = List(['--reuse-files', '--log-to-file','--log-level=%i'%logging.INFO], config=True,
571 571 help="Command line arguments to ipcontroller.")
572 572
573 573
574 574 class SSHEngineLauncher(SSHLauncher):
575 575 program = List(ipengine_cmd_argv, config=True,
576 576 help="remote ipengine command.")
577 577 # Command line arguments for ipengine.
578 578 program_args = List(
579 579 ['--log-to-file','log_level=%i'%logging.INFO], config=True,
580 580 help="Command line arguments to ipengine."
581 581 )
582 582
583 583 class SSHEngineSetLauncher(LocalEngineSetLauncher):
584 584 launcher_class = SSHEngineLauncher
585 585 engines = Dict(config=True,
586 586 help="""dict of engines to launch. This is a dict by hostname of ints,
587 587 corresponding to the number of engines to start on that host.""")
588 588
589 589 def start(self, n, profile_dir):
590 590 """Start engines by profile or profile_dir.
591 591 `n` is ignored, and the `engines` config property is used instead.
592 592 """
593 593
594 594 self.profile_dir = unicode(profile_dir)
595 595 dlist = []
596 596 for host, n in self.engines.iteritems():
597 597 if isinstance(n, (tuple, list)):
598 598 n, args = n
599 599 else:
600 600 args = copy.deepcopy(self.engine_args)
601 601
602 602 if '@' in host:
603 603 user,host = host.split('@',1)
604 604 else:
605 605 user=None
606 606 for i in range(n):
607 607 el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log)
608 608
609 609 # Copy the engine args over to each engine launcher.
610 610 i
611 611 el.program_args = args
612 612 el.on_stop(self._notice_engine_stopped)
613 613 d = el.start(profile_dir, user=user, hostname=host)
614 614 if i==0:
615 615 self.log.info("Starting SSHEngineSetLauncher: %r" % el.args)
616 616 self.launchers[host+str(i)] = el
617 617 dlist.append(d)
618 618 self.notify_start(dlist)
619 619 return dlist
620 620
621 621
622 622
623 623 #-----------------------------------------------------------------------------
624 624 # Windows HPC Server 2008 scheduler launchers
625 625 #-----------------------------------------------------------------------------
626 626
627 627
628 628 # This is only used on Windows.
629 629 def find_job_cmd():
630 630 if WINDOWS:
631 631 try:
632 632 return find_cmd('job')
633 633 except (FindCmdError, ImportError):
634 634 # ImportError will be raised if win32api is not installed
635 635 return 'job'
636 636 else:
637 637 return 'job'
638 638
639 639
640 640 class WindowsHPCLauncher(BaseLauncher):
641 641
642 642 job_id_regexp = Unicode(r'\d+', config=True,
643 643 help="""A regular expression used to get the job id from the output of the
644 644 submit_command. """
645 645 )
646 646 job_file_name = Unicode(u'ipython_job.xml', config=True,
647 647 help="The filename of the instantiated job script.")
648 648 # The full path to the instantiated job script. This gets made dynamically
649 649 # by combining the work_dir with the job_file_name.
650 650 job_file = Unicode(u'')
651 651 scheduler = Unicode('', config=True,
652 652 help="The hostname of the scheduler to submit the job to.")
653 653 job_cmd = Unicode(find_job_cmd(), config=True,
654 654 help="The command for submitting jobs.")
655 655
656 656 def __init__(self, work_dir=u'.', config=None, **kwargs):
657 657 super(WindowsHPCLauncher, self).__init__(
658 658 work_dir=work_dir, config=config, **kwargs
659 659 )
660 660
661 661 @property
662 662 def job_file(self):
663 663 return os.path.join(self.work_dir, self.job_file_name)
664 664
665 665 def write_job_file(self, n):
666 666 raise NotImplementedError("Implement write_job_file in a subclass.")
667 667
668 668 def find_args(self):
669 669 return [u'job.exe']
670 670
671 671 def parse_job_id(self, output):
672 672 """Take the output of the submit command and return the job id."""
673 673 m = re.search(self.job_id_regexp, output)
674 674 if m is not None:
675 675 job_id = m.group()
676 676 else:
677 677 raise LauncherError("Job id couldn't be determined: %s" % output)
678 678 self.job_id = job_id
679 679 self.log.info('Job started with job id: %r' % job_id)
680 680 return job_id
681 681
682 682 def start(self, n):
683 683 """Start n copies of the process using the Win HPC job scheduler."""
684 684 self.write_job_file(n)
685 685 args = [
686 686 'submit',
687 687 '/jobfile:%s' % self.job_file,
688 688 '/scheduler:%s' % self.scheduler
689 689 ]
690 690 self.log.info("Starting Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
691 691
692 692 output = check_output([self.job_cmd]+args,
693 693 env=os.environ,
694 694 cwd=self.work_dir,
695 695 stderr=STDOUT
696 696 )
697 697 job_id = self.parse_job_id(output)
698 698 self.notify_start(job_id)
699 699 return job_id
700 700
701 701 def stop(self):
702 702 args = [
703 703 'cancel',
704 704 self.job_id,
705 705 '/scheduler:%s' % self.scheduler
706 706 ]
707 707 self.log.info("Stopping Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
708 708 try:
709 709 output = check_output([self.job_cmd]+args,
710 710 env=os.environ,
711 711 cwd=self.work_dir,
712 712 stderr=STDOUT
713 713 )
714 714 except:
715 715 output = 'The job already appears to be stoppped: %r' % self.job_id
716 716 self.notify_stop(dict(job_id=self.job_id, output=output)) # Pass the output of the kill cmd
717 717 return output
718 718
719 719
720 720 class WindowsHPCControllerLauncher(WindowsHPCLauncher):
721 721
722 722 job_file_name = Unicode(u'ipcontroller_job.xml', config=True,
723 723 help="WinHPC xml job file.")
724 724 extra_args = List([], config=False,
725 725 help="extra args to pass to ipcontroller")
726 726
727 727 def write_job_file(self, n):
728 728 job = IPControllerJob(config=self.config)
729 729
730 730 t = IPControllerTask(config=self.config)
731 731 # The tasks work directory is *not* the actual work directory of
732 732 # the controller. It is used as the base path for the stdout/stderr
733 733 # files that the scheduler redirects to.
734 734 t.work_directory = self.profile_dir
735 735 # Add the profile_dir and from self.start().
736 736 t.controller_args.extend(self.extra_args)
737 737 job.add_task(t)
738 738
739 739 self.log.info("Writing job description file: %s" % self.job_file)
740 740 job.write(self.job_file)
741 741
742 742 @property
743 743 def job_file(self):
744 744 return os.path.join(self.profile_dir, self.job_file_name)
745 745
746 746 def start(self, profile_dir):
747 747 """Start the controller by profile_dir."""
748 self.extra_args = ['--profile_dir=%s'%profile_dir]
748 self.extra_args = ['--profile-dir=%s'%profile_dir]
749 749 self.profile_dir = unicode(profile_dir)
750 750 return super(WindowsHPCControllerLauncher, self).start(1)
751 751
752 752
753 753 class WindowsHPCEngineSetLauncher(WindowsHPCLauncher):
754 754
755 755 job_file_name = Unicode(u'ipengineset_job.xml', config=True,
756 756 help="jobfile for ipengines job")
757 757 extra_args = List([], config=False,
758 758 help="extra args to pas to ipengine")
759 759
760 760 def write_job_file(self, n):
761 761 job = IPEngineSetJob(config=self.config)
762 762
763 763 for i in range(n):
764 764 t = IPEngineTask(config=self.config)
765 765 # The tasks work directory is *not* the actual work directory of
766 766 # the engine. It is used as the base path for the stdout/stderr
767 767 # files that the scheduler redirects to.
768 768 t.work_directory = self.profile_dir
769 769 # Add the profile_dir and from self.start().
770 770 t.engine_args.extend(self.extra_args)
771 771 job.add_task(t)
772 772
773 773 self.log.info("Writing job description file: %s" % self.job_file)
774 774 job.write(self.job_file)
775 775
776 776 @property
777 777 def job_file(self):
778 778 return os.path.join(self.profile_dir, self.job_file_name)
779 779
780 780 def start(self, n, profile_dir):
781 781 """Start the controller by profile_dir."""
782 self.extra_args = ['--profile_dir=%s'%profile_dir]
782 self.extra_args = ['--profile-dir=%s'%profile_dir]
783 783 self.profile_dir = unicode(profile_dir)
784 784 return super(WindowsHPCEngineSetLauncher, self).start(n)
785 785
786 786
787 787 #-----------------------------------------------------------------------------
788 788 # Batch (PBS) system launchers
789 789 #-----------------------------------------------------------------------------
790 790
791 791 class BatchSystemLauncher(BaseLauncher):
792 792 """Launch an external process using a batch system.
793 793
794 794 This class is designed to work with UNIX batch systems like PBS, LSF,
795 795 GridEngine, etc. The overall model is that there are different commands
796 796 like qsub, qdel, etc. that handle the starting and stopping of the process.
797 797
798 798 This class also has the notion of a batch script. The ``batch_template``
799 799 attribute can be set to a string that is a template for the batch script.
800 800 This template is instantiated using string formatting. Thus the template can
801 801 use {n} fot the number of instances. Subclasses can add additional variables
802 802 to the template dict.
803 803 """
804 804
805 805 # Subclasses must fill these in. See PBSEngineSet
806 806 submit_command = List([''], config=True,
807 807 help="The name of the command line program used to submit jobs.")
808 808 delete_command = List([''], config=True,
809 809 help="The name of the command line program used to delete jobs.")
810 810 job_id_regexp = Unicode('', config=True,
811 811 help="""A regular expression used to get the job id from the output of the
812 812 submit_command.""")
813 813 batch_template = Unicode('', config=True,
814 814 help="The string that is the batch script template itself.")
815 815 batch_template_file = Unicode(u'', config=True,
816 816 help="The file that contains the batch template.")
817 817 batch_file_name = Unicode(u'batch_script', config=True,
818 818 help="The filename of the instantiated batch script.")
819 819 queue = Unicode(u'', config=True,
820 820 help="The PBS Queue.")
821 821
822 822 # not configurable, override in subclasses
823 823 # PBS Job Array regex
824 824 job_array_regexp = Unicode('')
825 825 job_array_template = Unicode('')
826 826 # PBS Queue regex
827 827 queue_regexp = Unicode('')
828 828 queue_template = Unicode('')
829 829 # The default batch template, override in subclasses
830 830 default_template = Unicode('')
831 831 # The full path to the instantiated batch script.
832 832 batch_file = Unicode(u'')
833 833 # the format dict used with batch_template:
834 834 context = Dict()
835 835 # the Formatter instance for rendering the templates:
836 836 formatter = Instance(EvalFormatter, (), {})
837 837
838 838
839 839 def find_args(self):
840 840 return self.submit_command + [self.batch_file]
841 841
842 842 def __init__(self, work_dir=u'.', config=None, **kwargs):
843 843 super(BatchSystemLauncher, self).__init__(
844 844 work_dir=work_dir, config=config, **kwargs
845 845 )
846 846 self.batch_file = os.path.join(self.work_dir, self.batch_file_name)
847 847
848 848 def parse_job_id(self, output):
849 849 """Take the output of the submit command and return the job id."""
850 850 m = re.search(self.job_id_regexp, output)
851 851 if m is not None:
852 852 job_id = m.group()
853 853 else:
854 854 raise LauncherError("Job id couldn't be determined: %s" % output)
855 855 self.job_id = job_id
856 856 self.log.info('Job submitted with job id: %r' % job_id)
857 857 return job_id
858 858
859 859 def write_batch_script(self, n):
860 860 """Instantiate and write the batch script to the work_dir."""
861 861 self.context['n'] = n
862 862 self.context['queue'] = self.queue
863 863 # first priority is batch_template if set
864 864 if self.batch_template_file and not self.batch_template:
865 865 # second priority is batch_template_file
866 866 with open(self.batch_template_file) as f:
867 867 self.batch_template = f.read()
868 868 if not self.batch_template:
869 869 # third (last) priority is default_template
870 870 self.batch_template = self.default_template
871 871
872 872 # add jobarray or queue lines to user-specified template
873 873 # note that this is *only* when user did not specify a template.
874 874 regex = re.compile(self.job_array_regexp)
875 875 # print regex.search(self.batch_template)
876 876 if not regex.search(self.batch_template):
877 877 self.log.info("adding job array settings to batch script")
878 878 firstline, rest = self.batch_template.split('\n',1)
879 879 self.batch_template = u'\n'.join([firstline, self.job_array_template, rest])
880 880
881 881 regex = re.compile(self.queue_regexp)
882 882 # print regex.search(self.batch_template)
883 883 if self.queue and not regex.search(self.batch_template):
884 884 self.log.info("adding PBS queue settings to batch script")
885 885 firstline, rest = self.batch_template.split('\n',1)
886 886 self.batch_template = u'\n'.join([firstline, self.queue_template, rest])
887 887
888 888 script_as_string = self.formatter.format(self.batch_template, **self.context)
889 889 self.log.info('Writing instantiated batch script: %s' % self.batch_file)
890 890
891 891 with open(self.batch_file, 'w') as f:
892 892 f.write(script_as_string)
893 893 os.chmod(self.batch_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
894 894
895 895 def start(self, n, profile_dir):
896 896 """Start n copies of the process using a batch system."""
897 897 # Here we save profile_dir in the context so they
898 898 # can be used in the batch script template as {profile_dir}
899 899 self.context['profile_dir'] = profile_dir
900 900 self.profile_dir = unicode(profile_dir)
901 901 self.write_batch_script(n)
902 902 output = check_output(self.args, env=os.environ)
903 903
904 904 job_id = self.parse_job_id(output)
905 905 self.notify_start(job_id)
906 906 return job_id
907 907
908 908 def stop(self):
909 909 output = check_output(self.delete_command+[self.job_id], env=os.environ)
910 910 self.notify_stop(dict(job_id=self.job_id, output=output)) # Pass the output of the kill cmd
911 911 return output
912 912
913 913
914 914 class PBSLauncher(BatchSystemLauncher):
915 915 """A BatchSystemLauncher subclass for PBS."""
916 916
917 917 submit_command = List(['qsub'], config=True,
918 918 help="The PBS submit command ['qsub']")
919 919 delete_command = List(['qdel'], config=True,
920 920 help="The PBS delete command ['qsub']")
921 921 job_id_regexp = Unicode(r'\d+', config=True,
922 922 help="Regular expresion for identifying the job ID [r'\d+']")
923 923
924 924 batch_file = Unicode(u'')
925 925 job_array_regexp = Unicode('#PBS\W+-t\W+[\w\d\-\$]+')
926 926 job_array_template = Unicode('#PBS -t 1-{n}')
927 927 queue_regexp = Unicode('#PBS\W+-q\W+\$?\w+')
928 928 queue_template = Unicode('#PBS -q {queue}')
929 929
930 930
931 931 class PBSControllerLauncher(PBSLauncher):
932 932 """Launch a controller using PBS."""
933 933
934 934 batch_file_name = Unicode(u'pbs_controller', config=True,
935 935 help="batch file name for the controller job.")
936 936 default_template= Unicode("""#!/bin/sh
937 937 #PBS -V
938 938 #PBS -N ipcontroller
939 %s --log-to-file --profile_dir={profile_dir}
939 %s --log-to-file --profile-dir={profile_dir}
940 940 """%(' '.join(ipcontroller_cmd_argv)))
941 941
942 942 def start(self, profile_dir):
943 943 """Start the controller by profile or profile_dir."""
944 944 self.log.info("Starting PBSControllerLauncher: %r" % self.args)
945 945 return super(PBSControllerLauncher, self).start(1, profile_dir)
946 946
947 947
948 948 class PBSEngineSetLauncher(PBSLauncher):
949 949 """Launch Engines using PBS"""
950 950 batch_file_name = Unicode(u'pbs_engines', config=True,
951 951 help="batch file name for the engine(s) job.")
952 952 default_template= Unicode(u"""#!/bin/sh
953 953 #PBS -V
954 954 #PBS -N ipengine
955 %s --profile_dir={profile_dir}
955 %s --profile-dir={profile_dir}
956 956 """%(' '.join(ipengine_cmd_argv)))
957 957
958 958 def start(self, n, profile_dir):
959 959 """Start n engines by profile or profile_dir."""
960 960 self.log.info('Starting %i engines with PBSEngineSetLauncher: %r' % (n, self.args))
961 961 return super(PBSEngineSetLauncher, self).start(n, profile_dir)
962 962
963 963 #SGE is very similar to PBS
964 964
965 965 class SGELauncher(PBSLauncher):
966 966 """Sun GridEngine is a PBS clone with slightly different syntax"""
967 967 job_array_regexp = Unicode('#\$\W+\-t')
968 968 job_array_template = Unicode('#$ -t 1-{n}')
969 969 queue_regexp = Unicode('#\$\W+-q\W+\$?\w+')
970 970 queue_template = Unicode('#$ -q {queue}')
971 971
972 972 class SGEControllerLauncher(SGELauncher):
973 973 """Launch a controller using SGE."""
974 974
975 975 batch_file_name = Unicode(u'sge_controller', config=True,
976 976 help="batch file name for the ipontroller job.")
977 977 default_template= Unicode(u"""#$ -V
978 978 #$ -S /bin/sh
979 979 #$ -N ipcontroller
980 %s --log-to-file --profile_dir={profile_dir}
980 %s --log-to-file --profile-dir={profile_dir}
981 981 """%(' '.join(ipcontroller_cmd_argv)))
982 982
983 983 def start(self, profile_dir):
984 984 """Start the controller by profile or profile_dir."""
985 985 self.log.info("Starting PBSControllerLauncher: %r" % self.args)
986 986 return super(SGEControllerLauncher, self).start(1, profile_dir)
987 987
988 988 class SGEEngineSetLauncher(SGELauncher):
989 989 """Launch Engines with SGE"""
990 990 batch_file_name = Unicode(u'sge_engines', config=True,
991 991 help="batch file name for the engine(s) job.")
992 992 default_template = Unicode("""#$ -V
993 993 #$ -S /bin/sh
994 994 #$ -N ipengine
995 %s --profile_dir={profile_dir}
995 %s --profile-dir={profile_dir}
996 996 """%(' '.join(ipengine_cmd_argv)))
997 997
998 998 def start(self, n, profile_dir):
999 999 """Start n engines by profile or profile_dir."""
1000 1000 self.log.info('Starting %i engines with SGEEngineSetLauncher: %r' % (n, self.args))
1001 1001 return super(SGEEngineSetLauncher, self).start(n, profile_dir)
1002 1002
1003 1003
1004 1004 #-----------------------------------------------------------------------------
1005 1005 # A launcher for ipcluster itself!
1006 1006 #-----------------------------------------------------------------------------
1007 1007
1008 1008
1009 1009 class IPClusterLauncher(LocalProcessLauncher):
1010 1010 """Launch the ipcluster program in an external process."""
1011 1011
1012 1012 ipcluster_cmd = List(ipcluster_cmd_argv, config=True,
1013 1013 help="Popen command for ipcluster")
1014 1014 ipcluster_args = List(
1015 ['--clean-logs', '--log-to-file', '--log_level=%i'%logging.INFO], config=True,
1015 ['--clean-logs', '--log-to-file', '--log-level=%i'%logging.INFO], config=True,
1016 1016 help="Command line arguments to pass to ipcluster.")
1017 1017 ipcluster_subcommand = Unicode('start')
1018 1018 ipcluster_n = Int(2)
1019 1019
1020 1020 def find_args(self):
1021 1021 return self.ipcluster_cmd + [self.ipcluster_subcommand] + \
1022 1022 ['--n=%i'%self.ipcluster_n] + self.ipcluster_args
1023 1023
1024 1024 def start(self):
1025 1025 self.log.info("Starting ipcluster: %r" % self.args)
1026 1026 return super(IPClusterLauncher, self).start()
1027 1027
1028 1028 #-----------------------------------------------------------------------------
1029 1029 # Collections of launchers
1030 1030 #-----------------------------------------------------------------------------
1031 1031
1032 1032 local_launchers = [
1033 1033 LocalControllerLauncher,
1034 1034 LocalEngineLauncher,
1035 1035 LocalEngineSetLauncher,
1036 1036 ]
1037 1037 mpi_launchers = [
1038 1038 MPIExecLauncher,
1039 1039 MPIExecControllerLauncher,
1040 1040 MPIExecEngineSetLauncher,
1041 1041 ]
1042 1042 ssh_launchers = [
1043 1043 SSHLauncher,
1044 1044 SSHControllerLauncher,
1045 1045 SSHEngineLauncher,
1046 1046 SSHEngineSetLauncher,
1047 1047 ]
1048 1048 winhpc_launchers = [
1049 1049 WindowsHPCLauncher,
1050 1050 WindowsHPCControllerLauncher,
1051 1051 WindowsHPCEngineSetLauncher,
1052 1052 ]
1053 1053 pbs_launchers = [
1054 1054 PBSLauncher,
1055 1055 PBSControllerLauncher,
1056 1056 PBSEngineSetLauncher,
1057 1057 ]
1058 1058 sge_launchers = [
1059 1059 SGELauncher,
1060 1060 SGEControllerLauncher,
1061 1061 SGEEngineSetLauncher,
1062 1062 ]
1063 1063 all_launchers = local_launchers + mpi_launchers + ssh_launchers + winhpc_launchers\
1064 1064 + pbs_launchers + sge_launchers
1065 1065
@@ -1,111 +1,111 b''
1 1 """toplevel setup/teardown for parallel tests."""
2 2
3 3 #-------------------------------------------------------------------------------
4 4 # Copyright (C) 2011 The IPython Development Team
5 5 #
6 6 # Distributed under the terms of the BSD License. The full license is in
7 7 # the file COPYING, distributed as part of this software.
8 8 #-------------------------------------------------------------------------------
9 9
10 10 #-------------------------------------------------------------------------------
11 11 # Imports
12 12 #-------------------------------------------------------------------------------
13 13
14 14 import os
15 15 import tempfile
16 16 import time
17 17 from subprocess import Popen
18 18
19 19 from IPython.utils.path import get_ipython_dir
20 20 from IPython.parallel import Client
21 21 from IPython.parallel.apps.launcher import (LocalProcessLauncher,
22 22 ipengine_cmd_argv,
23 23 ipcontroller_cmd_argv,
24 24 SIGKILL)
25 25
26 26 # globals
27 27 launchers = []
28 28 blackhole = open(os.devnull, 'w')
29 29
30 30 # Launcher class
31 31 class TestProcessLauncher(LocalProcessLauncher):
32 32 """subclass LocalProcessLauncher, to prevent extra sockets and threads being created on Windows"""
33 33 def start(self):
34 34 if self.state == 'before':
35 35 self.process = Popen(self.args,
36 36 stdout=blackhole, stderr=blackhole,
37 37 env=os.environ,
38 38 cwd=self.work_dir
39 39 )
40 40 self.notify_start(self.process.pid)
41 41 self.poll = self.process.poll
42 42 else:
43 43 s = 'The process was already started and has state: %r' % self.state
44 44 raise ProcessStateError(s)
45 45
46 46 # nose setup/teardown
47 47
48 48 def setup():
49 49 cluster_dir = os.path.join(get_ipython_dir(), 'profile_iptest')
50 50 engine_json = os.path.join(cluster_dir, 'security', 'ipcontroller-engine.json')
51 51 client_json = os.path.join(cluster_dir, 'security', 'ipcontroller-client.json')
52 52 for json in (engine_json, client_json):
53 53 if os.path.exists(json):
54 54 os.remove(json)
55 55
56 56 cp = TestProcessLauncher()
57 57 cp.cmd_and_args = ipcontroller_cmd_argv + \
58 ['--profile=iptest', '--log_level=50']
58 ['--profile=iptest', '--log-level=50']
59 59 cp.start()
60 60 launchers.append(cp)
61 61 tic = time.time()
62 62 while not os.path.exists(engine_json) or not os.path.exists(client_json):
63 63 if cp.poll() is not None:
64 64 print cp.poll()
65 65 raise RuntimeError("The test controller failed to start.")
66 66 elif time.time()-tic > 10:
67 67 raise RuntimeError("Timeout waiting for the test controller to start.")
68 68 time.sleep(0.1)
69 69 add_engines(1)
70 70
71 71 def add_engines(n=1, profile='iptest'):
72 72 rc = Client(profile=profile)
73 73 base = len(rc)
74 74 eps = []
75 75 for i in range(n):
76 76 ep = TestProcessLauncher()
77 ep.cmd_and_args = ipengine_cmd_argv + ['--profile=%s'%profile, '--log_level=50']
77 ep.cmd_and_args = ipengine_cmd_argv + ['--profile=%s'%profile, '--log-level=50']
78 78 ep.start()
79 79 launchers.append(ep)
80 80 eps.append(ep)
81 81 tic = time.time()
82 82 while len(rc) < base+n:
83 83 if any([ ep.poll() is not None for ep in eps ]):
84 84 raise RuntimeError("A test engine failed to start.")
85 85 elif time.time()-tic > 10:
86 86 raise RuntimeError("Timeout waiting for engines to connect.")
87 87 time.sleep(.1)
88 88 rc.spin()
89 89 rc.close()
90 90 return eps
91 91
92 92 def teardown():
93 93 time.sleep(1)
94 94 while launchers:
95 95 p = launchers.pop()
96 96 if p.poll() is None:
97 97 try:
98 98 p.stop()
99 99 except Exception, e:
100 100 print e
101 101 pass
102 102 if p.poll() is None:
103 103 time.sleep(.25)
104 104 if p.poll() is None:
105 105 try:
106 106 print 'cleaning up test process...'
107 107 p.signal(SIGKILL)
108 108 except:
109 109 print "couldn't shutdown process: ", p
110 110 blackhole.close()
111 111
@@ -1,309 +1,321 b''
1 1 """Generic testing tools that do NOT depend on Twisted.
2 2
3 3 In particular, this module exposes a set of top-level assert* functions that
4 4 can be used in place of nose.tools.assert* in method generators (the ones in
5 5 nose can not, at least as of nose 0.10.4).
6 6
7 7 Note: our testing package contains testing.util, which does depend on Twisted
8 8 and provides utilities for tests that manage Deferreds. All testing support
9 9 tools that only depend on nose, IPython or the standard library should go here
10 10 instead.
11 11
12 12
13 13 Authors
14 14 -------
15 15 - Fernando Perez <Fernando.Perez@berkeley.edu>
16 16 """
17 17
18 18 from __future__ import absolute_import
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Copyright (C) 2009 The IPython Development Team
22 22 #
23 23 # Distributed under the terms of the BSD License. The full license is in
24 24 # the file COPYING, distributed as part of this software.
25 25 #-----------------------------------------------------------------------------
26 26
27 27 #-----------------------------------------------------------------------------
28 28 # Imports
29 29 #-----------------------------------------------------------------------------
30 30
31 31 import os
32 32 import re
33 33 import sys
34 34
35 from contextlib import contextmanager
36
35 37 try:
36 38 # These tools are used by parts of the runtime, so we make the nose
37 39 # dependency optional at this point. Nose is a hard dependency to run the
38 40 # test suite, but NOT to use ipython itself.
39 41 import nose.tools as nt
40 42 has_nose = True
41 43 except ImportError:
42 44 has_nose = False
43 45
44 46 from IPython.config.loader import Config
45 47 from IPython.utils.process import find_cmd, getoutputerror
46 48 from IPython.utils.text import list_strings
47 49 from IPython.utils.io import temp_pyfile
48 50
49 51 from . import decorators as dec
50 52 from . import skipdoctest
51 53
52 54 #-----------------------------------------------------------------------------
53 55 # Globals
54 56 #-----------------------------------------------------------------------------
55 57
56 58 # Make a bunch of nose.tools assert wrappers that can be used in test
57 59 # generators. This will expose an assert* function for each one in nose.tools.
58 60
59 61 _tpl = """
60 62 def %(name)s(*a,**kw):
61 63 return nt.%(name)s(*a,**kw)
62 64 """
63 65
64 66 if has_nose:
65 67 for _x in [a for a in dir(nt) if a.startswith('assert')]:
66 68 exec _tpl % dict(name=_x)
67 69
68 70 #-----------------------------------------------------------------------------
69 71 # Functions and classes
70 72 #-----------------------------------------------------------------------------
71 73
72 74 # The docstring for full_path doctests differently on win32 (different path
73 75 # separator) so just skip the doctest there. The example remains informative.
74 76 doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
75 77
76 78 @doctest_deco
77 79 def full_path(startPath,files):
78 80 """Make full paths for all the listed files, based on startPath.
79 81
80 82 Only the base part of startPath is kept, since this routine is typically
81 83 used with a script's __file__ variable as startPath. The base of startPath
82 84 is then prepended to all the listed files, forming the output list.
83 85
84 86 Parameters
85 87 ----------
86 88 startPath : string
87 89 Initial path to use as the base for the results. This path is split
88 90 using os.path.split() and only its first component is kept.
89 91
90 92 files : string or list
91 93 One or more files.
92 94
93 95 Examples
94 96 --------
95 97
96 98 >>> full_path('/foo/bar.py',['a.txt','b.txt'])
97 99 ['/foo/a.txt', '/foo/b.txt']
98 100
99 101 >>> full_path('/foo',['a.txt','b.txt'])
100 102 ['/a.txt', '/b.txt']
101 103
102 104 If a single file is given, the output is still a list:
103 105 >>> full_path('/foo','a.txt')
104 106 ['/a.txt']
105 107 """
106 108
107 109 files = list_strings(files)
108 110 base = os.path.split(startPath)[0]
109 111 return [ os.path.join(base,f) for f in files ]
110 112
111 113
112 114 def parse_test_output(txt):
113 115 """Parse the output of a test run and return errors, failures.
114 116
115 117 Parameters
116 118 ----------
117 119 txt : str
118 120 Text output of a test run, assumed to contain a line of one of the
119 121 following forms::
120 122 'FAILED (errors=1)'
121 123 'FAILED (failures=1)'
122 124 'FAILED (errors=1, failures=1)'
123 125
124 126 Returns
125 127 -------
126 128 nerr, nfail: number of errors and failures.
127 129 """
128 130
129 131 err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
130 132 if err_m:
131 133 nerr = int(err_m.group(1))
132 134 nfail = 0
133 135 return nerr, nfail
134 136
135 137 fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
136 138 if fail_m:
137 139 nerr = 0
138 140 nfail = int(fail_m.group(1))
139 141 return nerr, nfail
140 142
141 143 both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
142 144 re.MULTILINE)
143 145 if both_m:
144 146 nerr = int(both_m.group(1))
145 147 nfail = int(both_m.group(2))
146 148 return nerr, nfail
147 149
148 150 # If the input didn't match any of these forms, assume no error/failures
149 151 return 0, 0
150 152
151 153
152 154 # So nose doesn't think this is a test
153 155 parse_test_output.__test__ = False
154 156
155 157
156 158 def default_argv():
157 159 """Return a valid default argv for creating testing instances of ipython"""
158 160
159 161 return ['--quick', # so no config file is loaded
160 162 # Other defaults to minimize side effects on stdout
161 163 '--colors=NoColor', '--no-term-title','--no-banner',
162 164 '--autocall=0']
163 165
164 166
165 167 def default_config():
166 168 """Return a config object with good defaults for testing."""
167 169 config = Config()
168 170 config.TerminalInteractiveShell.colors = 'NoColor'
169 171 config.TerminalTerminalInteractiveShell.term_title = False,
170 172 config.TerminalInteractiveShell.autocall = 0
171 173 config.HistoryManager.hist_file = u'test_hist.sqlite'
172 174 config.HistoryManager.db_cache_size = 10000
173 175 return config
174 176
175 177
176 178 def ipexec(fname, options=None):
177 179 """Utility to call 'ipython filename'.
178 180
179 181 Starts IPython witha minimal and safe configuration to make startup as fast
180 182 as possible.
181 183
182 184 Note that this starts IPython in a subprocess!
183 185
184 186 Parameters
185 187 ----------
186 188 fname : str
187 189 Name of file to be executed (should have .py or .ipy extension).
188 190
189 191 options : optional, list
190 192 Extra command-line flags to be passed to IPython.
191 193
192 194 Returns
193 195 -------
194 196 (stdout, stderr) of ipython subprocess.
195 197 """
196 198 if options is None: options = []
197 199
198 200 # For these subprocess calls, eliminate all prompt printing so we only see
199 201 # output from script execution
200 202 prompt_opts = [ '--InteractiveShell.prompt_in1=""',
201 203 '--InteractiveShell.prompt_in2=""',
202 204 '--InteractiveShell.prompt_out=""'
203 205 ]
204 206 cmdargs = ' '.join(default_argv() + prompt_opts + options)
205 207
206 208 _ip = get_ipython()
207 209 test_dir = os.path.dirname(__file__)
208 210
209 211 ipython_cmd = find_cmd('ipython')
210 212 # Absolute path for filename
211 213 full_fname = os.path.join(test_dir, fname)
212 214 full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname)
213 215 #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg
214 216 return getoutputerror(full_cmd)
215 217
216 218
217 219 def ipexec_validate(fname, expected_out, expected_err='',
218 220 options=None):
219 221 """Utility to call 'ipython filename' and validate output/error.
220 222
221 223 This function raises an AssertionError if the validation fails.
222 224
223 225 Note that this starts IPython in a subprocess!
224 226
225 227 Parameters
226 228 ----------
227 229 fname : str
228 230 Name of the file to be executed (should have .py or .ipy extension).
229 231
230 232 expected_out : str
231 233 Expected stdout of the process.
232 234
233 235 expected_err : optional, str
234 236 Expected stderr of the process.
235 237
236 238 options : optional, list
237 239 Extra command-line flags to be passed to IPython.
238 240
239 241 Returns
240 242 -------
241 243 None
242 244 """
243 245
244 246 import nose.tools as nt
245 247
246 248 out, err = ipexec(fname)
247 249 #print 'OUT', out # dbg
248 250 #print 'ERR', err # dbg
249 251 # If there are any errors, we must check those befor stdout, as they may be
250 252 # more informative than simply having an empty stdout.
251 253 if err:
252 254 if expected_err:
253 255 nt.assert_equals(err.strip(), expected_err.strip())
254 256 else:
255 257 raise ValueError('Running file %r produced error: %r' %
256 258 (fname, err))
257 259 # If no errors or output on stderr was expected, match stdout
258 260 nt.assert_equals(out.strip(), expected_out.strip())
259 261
260 262
261 263 class TempFileMixin(object):
262 264 """Utility class to create temporary Python/IPython files.
263 265
264 266 Meant as a mixin class for test cases."""
265 267
266 268 def mktmp(self, src, ext='.py'):
267 269 """Make a valid python temp file."""
268 270 fname, f = temp_pyfile(src, ext)
269 271 self.tmpfile = f
270 272 self.fname = fname
271 273
272 274 def tearDown(self):
273 275 if hasattr(self, 'tmpfile'):
274 276 # If the tmpfile wasn't made because of skipped tests, like in
275 277 # win32, there's nothing to cleanup.
276 278 self.tmpfile.close()
277 279 try:
278 280 os.unlink(self.fname)
279 281 except:
280 282 # On Windows, even though we close the file, we still can't
281 283 # delete it. I have no clue why
282 284 pass
283 285
284 286 pair_fail_msg = ("Testing function {0}\n\n"
285 287 "In:\n"
286 288 " {1!r}\n"
287 289 "Expected:\n"
288 290 " {2!r}\n"
289 291 "Got:\n"
290 292 " {3!r}\n")
291 293 def check_pairs(func, pairs):
292 294 """Utility function for the common case of checking a function with a
293 295 sequence of input/output pairs.
294 296
295 297 Parameters
296 298 ----------
297 299 func : callable
298 300 The function to be tested. Should accept a single argument.
299 301 pairs : iterable
300 302 A list of (input, expected_output) tuples.
301 303
302 304 Returns
303 305 -------
304 306 None. Raises an AssertionError if any output does not match the expected
305 307 value.
306 308 """
307 309 for inp, expected in pairs:
308 310 out = func(inp)
309 311 assert out == expected, pair_fail_msg.format(func.func_name, inp, expected, out)
312
313 @contextmanager
314 def mute_warn():
315 from IPython.utils import warn
316 save_warn = warn.warn
317 warn.warn = lambda *a, **kw: None
318 try:
319 yield
320 finally:
321 warn.warn = save_warn No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now