##// END OF EJS Templates
aliases match flag pattern ('-' as wordsep, not '_')...
MinRK -
Show More
@@ -1,404 +1,404 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 A base class for a configurable application.
3 A base class for a configurable application.
4
4
5 Authors:
5 Authors:
6
6
7 * Brian Granger
7 * Brian Granger
8 * Min RK
8 * Min RK
9 """
9 """
10
10
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12 # Copyright (C) 2008-2011 The IPython Development Team
12 # Copyright (C) 2008-2011 The IPython Development Team
13 #
13 #
14 # Distributed under the terms of the BSD License. The full license is in
14 # Distributed under the terms of the BSD License. The full license is in
15 # the file COPYING, distributed as part of this software.
15 # the file COPYING, distributed as part of this software.
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17
17
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19 # Imports
19 # Imports
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21
21
22 import logging
22 import logging
23 import os
23 import os
24 import re
24 import re
25 import sys
25 import sys
26 from copy import deepcopy
26 from copy import deepcopy
27
27
28 from IPython.config.configurable import SingletonConfigurable
28 from IPython.config.configurable import SingletonConfigurable
29 from IPython.config.loader import (
29 from IPython.config.loader import (
30 KeyValueConfigLoader, PyFileConfigLoader, Config, ArgumentError
30 KeyValueConfigLoader, PyFileConfigLoader, Config, ArgumentError
31 )
31 )
32
32
33 from IPython.utils.traitlets import (
33 from IPython.utils.traitlets import (
34 Unicode, List, Int, Enum, Dict, Instance, TraitError
34 Unicode, List, Int, Enum, Dict, Instance, TraitError
35 )
35 )
36 from IPython.utils.importstring import import_item
36 from IPython.utils.importstring import import_item
37 from IPython.utils.text import indent, wrap_paragraphs, dedent
37 from IPython.utils.text import indent, wrap_paragraphs, dedent
38
38
39 #-----------------------------------------------------------------------------
39 #-----------------------------------------------------------------------------
40 # function for re-wrapping a helpstring
40 # function for re-wrapping a helpstring
41 #-----------------------------------------------------------------------------
41 #-----------------------------------------------------------------------------
42
42
43 #-----------------------------------------------------------------------------
43 #-----------------------------------------------------------------------------
44 # Descriptions for the various sections
44 # Descriptions for the various sections
45 #-----------------------------------------------------------------------------
45 #-----------------------------------------------------------------------------
46
46
47 # merge flags&aliases into options
47 # merge flags&aliases into options
48 option_description = """
48 option_description = """
49 IPython command-line arguments are passed as '--<flag>', or '--<name>=<value>'.
49 IPython command-line arguments are passed as '--<flag>', or '--<name>=<value>'.
50
50
51 Arguments that take values are actually aliases to full Configurables, whose
51 Arguments that take values are actually aliases to full Configurables, whose
52 aliases are listed on the help line. For more information on full
52 aliases are listed on the help line. For more information on full
53 configurables, see '--help-all'.
53 configurables, see '--help-all'.
54 """.strip() # trim newlines of front and back
54 """.strip() # trim newlines of front and back
55
55
56 keyvalue_description = """
56 keyvalue_description = """
57 Parameters are set from command-line arguments of the form:
57 Parameters are set from command-line arguments of the form:
58 `--Class.trait=value`.
58 `--Class.trait=value`.
59 This line is evaluated in Python, so simple expressions are allowed, e.g.::
59 This line is evaluated in Python, so simple expressions are allowed, e.g.::
60 `--C.a='range(3)'` For setting C.a=[0,1,2].
60 `--C.a='range(3)'` For setting C.a=[0,1,2].
61 """.strip() # trim newlines of front and back
61 """.strip() # trim newlines of front and back
62
62
63 subcommand_description = """
63 subcommand_description = """
64 Subcommands are launched as `{app} cmd [args]`. For information on using
64 Subcommands are launched as `{app} cmd [args]`. For information on using
65 subcommand 'cmd', do: `{app} cmd -h`.
65 subcommand 'cmd', do: `{app} cmd -h`.
66 """.strip().format(app=os.path.basename(sys.argv[0]))
66 """.strip().format(app=os.path.basename(sys.argv[0]))
67 # get running program name
67 # get running program name
68
68
69 #-----------------------------------------------------------------------------
69 #-----------------------------------------------------------------------------
70 # Application class
70 # Application class
71 #-----------------------------------------------------------------------------
71 #-----------------------------------------------------------------------------
72
72
73
73
74 class ApplicationError(Exception):
74 class ApplicationError(Exception):
75 pass
75 pass
76
76
77
77
78 class Application(SingletonConfigurable):
78 class Application(SingletonConfigurable):
79 """A singleton application with full configuration support."""
79 """A singleton application with full configuration support."""
80
80
81 # The name of the application, will usually match the name of the command
81 # The name of the application, will usually match the name of the command
82 # line application
82 # line application
83 name = Unicode(u'application')
83 name = Unicode(u'application')
84
84
85 # The description of the application that is printed at the beginning
85 # The description of the application that is printed at the beginning
86 # of the help.
86 # of the help.
87 description = Unicode(u'This is an application.')
87 description = Unicode(u'This is an application.')
88 # default section descriptions
88 # default section descriptions
89 option_description = Unicode(option_description)
89 option_description = Unicode(option_description)
90 keyvalue_description = Unicode(keyvalue_description)
90 keyvalue_description = Unicode(keyvalue_description)
91 subcommand_description = Unicode(subcommand_description)
91 subcommand_description = Unicode(subcommand_description)
92
92
93
93
94 # A sequence of Configurable subclasses whose config=True attributes will
94 # A sequence of Configurable subclasses whose config=True attributes will
95 # be exposed at the command line.
95 # be exposed at the command line.
96 classes = List([])
96 classes = List([])
97
97
98 # The version string of this application.
98 # The version string of this application.
99 version = Unicode(u'0.0')
99 version = Unicode(u'0.0')
100
100
101 # The log level for the application
101 # The log level for the application
102 log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
102 log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
103 default_value=logging.WARN,
103 default_value=logging.WARN,
104 config=True,
104 config=True,
105 help="Set the log level by value or name.")
105 help="Set the log level by value or name.")
106 def _log_level_changed(self, name, old, new):
106 def _log_level_changed(self, name, old, new):
107 """Adjust the log level when log_level is set."""
107 """Adjust the log level when log_level is set."""
108 if isinstance(new, basestring):
108 if isinstance(new, basestring):
109 new = getattr(logging, new)
109 new = getattr(logging, new)
110 self.log_level = new
110 self.log_level = new
111 self.log.setLevel(new)
111 self.log.setLevel(new)
112
112
113 # the alias map for configurables
113 # the alias map for configurables
114 aliases = Dict(dict(log_level='Application.log_level'))
114 aliases = Dict({'log-level' : 'Application.log_level'})
115
115
116 # flags for loading Configurables or store_const style flags
116 # flags for loading Configurables or store_const style flags
117 # flags are loaded from this dict by '--key' flags
117 # flags are loaded from this dict by '--key' flags
118 # this must be a dict of two-tuples, the first element being the Config/dict
118 # this must be a dict of two-tuples, the first element being the Config/dict
119 # and the second being the help string for the flag
119 # and the second being the help string for the flag
120 flags = Dict()
120 flags = Dict()
121 def _flags_changed(self, name, old, new):
121 def _flags_changed(self, name, old, new):
122 """ensure flags dict is valid"""
122 """ensure flags dict is valid"""
123 for key,value in new.iteritems():
123 for key,value in new.iteritems():
124 assert len(value) == 2, "Bad flag: %r:%s"%(key,value)
124 assert len(value) == 2, "Bad flag: %r:%s"%(key,value)
125 assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value)
125 assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value)
126 assert isinstance(value[1], basestring), "Bad flag: %r:%s"%(key,value)
126 assert isinstance(value[1], basestring), "Bad flag: %r:%s"%(key,value)
127
127
128
128
129 # subcommands for launching other applications
129 # subcommands for launching other applications
130 # if this is not empty, this will be a parent Application
130 # if this is not empty, this will be a parent Application
131 # this must be a dict of two-tuples,
131 # this must be a dict of two-tuples,
132 # the first element being the application class/import string
132 # the first element being the application class/import string
133 # and the second being the help string for the subcommand
133 # and the second being the help string for the subcommand
134 subcommands = Dict()
134 subcommands = Dict()
135 # parse_command_line will initialize a subapp, if requested
135 # parse_command_line will initialize a subapp, if requested
136 subapp = Instance('IPython.config.application.Application', allow_none=True)
136 subapp = Instance('IPython.config.application.Application', allow_none=True)
137
137
138 # extra command-line arguments that don't set config values
138 # extra command-line arguments that don't set config values
139 extra_args = List(Unicode)
139 extra_args = List(Unicode)
140
140
141
141
142 def __init__(self, **kwargs):
142 def __init__(self, **kwargs):
143 SingletonConfigurable.__init__(self, **kwargs)
143 SingletonConfigurable.__init__(self, **kwargs)
144 # Add my class to self.classes so my attributes appear in command line
144 # Add my class to self.classes so my attributes appear in command line
145 # options.
145 # options.
146 self.classes.insert(0, self.__class__)
146 self.classes.insert(0, self.__class__)
147
147
148 self.init_logging()
148 self.init_logging()
149
149
150 def _config_changed(self, name, old, new):
150 def _config_changed(self, name, old, new):
151 SingletonConfigurable._config_changed(self, name, old, new)
151 SingletonConfigurable._config_changed(self, name, old, new)
152 self.log.debug('Config changed:')
152 self.log.debug('Config changed:')
153 self.log.debug(repr(new))
153 self.log.debug(repr(new))
154
154
155 def init_logging(self):
155 def init_logging(self):
156 """Start logging for this application.
156 """Start logging for this application.
157
157
158 The default is to log to stdout using a StreaHandler. The log level
158 The default is to log to stdout using a StreaHandler. The log level
159 starts at loggin.WARN, but this can be adjusted by setting the
159 starts at loggin.WARN, but this can be adjusted by setting the
160 ``log_level`` attribute.
160 ``log_level`` attribute.
161 """
161 """
162 self.log = logging.getLogger(self.__class__.__name__)
162 self.log = logging.getLogger(self.__class__.__name__)
163 self.log.setLevel(self.log_level)
163 self.log.setLevel(self.log_level)
164 if sys.executable.endswith('pythonw.exe'):
164 if sys.executable.endswith('pythonw.exe'):
165 # this should really go to a file, but file-logging is only
165 # this should really go to a file, but file-logging is only
166 # hooked up in parallel applications
166 # hooked up in parallel applications
167 self._log_handler = logging.StreamHandler(open(os.devnull, 'w'))
167 self._log_handler = logging.StreamHandler(open(os.devnull, 'w'))
168 else:
168 else:
169 self._log_handler = logging.StreamHandler()
169 self._log_handler = logging.StreamHandler()
170 self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
170 self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
171 self._log_handler.setFormatter(self._log_formatter)
171 self._log_handler.setFormatter(self._log_formatter)
172 self.log.addHandler(self._log_handler)
172 self.log.addHandler(self._log_handler)
173
173
174 def initialize(self, argv=None):
174 def initialize(self, argv=None):
175 """Do the basic steps to configure me.
175 """Do the basic steps to configure me.
176
176
177 Override in subclasses.
177 Override in subclasses.
178 """
178 """
179 self.parse_command_line(argv)
179 self.parse_command_line(argv)
180
180
181
181
182 def start(self):
182 def start(self):
183 """Start the app mainloop.
183 """Start the app mainloop.
184
184
185 Override in subclasses.
185 Override in subclasses.
186 """
186 """
187 if self.subapp is not None:
187 if self.subapp is not None:
188 return self.subapp.start()
188 return self.subapp.start()
189
189
190 def print_alias_help(self):
190 def print_alias_help(self):
191 """Print the alias part of the help."""
191 """Print the alias part of the help."""
192 if not self.aliases:
192 if not self.aliases:
193 return
193 return
194
194
195 lines = []
195 lines = []
196 classdict = {}
196 classdict = {}
197 for cls in self.classes:
197 for cls in self.classes:
198 # include all parents (up to, but excluding Configurable) in available names
198 # include all parents (up to, but excluding Configurable) in available names
199 for c in cls.mro()[:-3]:
199 for c in cls.mro()[:-3]:
200 classdict[c.__name__] = c
200 classdict[c.__name__] = c
201
201
202 for alias, longname in self.aliases.iteritems():
202 for alias, longname in self.aliases.iteritems():
203 classname, traitname = longname.split('.',1)
203 classname, traitname = longname.split('.',1)
204 cls = classdict[classname]
204 cls = classdict[classname]
205
205
206 trait = cls.class_traits(config=True)[traitname]
206 trait = cls.class_traits(config=True)[traitname]
207 help = cls.class_get_trait_help(trait).splitlines()
207 help = cls.class_get_trait_help(trait).splitlines()
208 # reformat first line
208 # reformat first line
209 help[0] = help[0].replace(longname, alias) + ' (%s)'%longname
209 help[0] = help[0].replace(longname, alias) + ' (%s)'%longname
210 lines.extend(help)
210 lines.extend(help)
211 # lines.append('')
211 # lines.append('')
212 print os.linesep.join(lines)
212 print os.linesep.join(lines)
213
213
214 def print_flag_help(self):
214 def print_flag_help(self):
215 """Print the flag part of the help."""
215 """Print the flag part of the help."""
216 if not self.flags:
216 if not self.flags:
217 return
217 return
218
218
219 lines = []
219 lines = []
220 for m, (cfg,help) in self.flags.iteritems():
220 for m, (cfg,help) in self.flags.iteritems():
221 lines.append('--'+m)
221 lines.append('--'+m)
222 lines.append(indent(dedent(help.strip())))
222 lines.append(indent(dedent(help.strip())))
223 # lines.append('')
223 # lines.append('')
224 print os.linesep.join(lines)
224 print os.linesep.join(lines)
225
225
226 def print_options(self):
226 def print_options(self):
227 if not self.flags and not self.aliases:
227 if not self.flags and not self.aliases:
228 return
228 return
229 lines = ['Options']
229 lines = ['Options']
230 lines.append('-'*len(lines[0]))
230 lines.append('-'*len(lines[0]))
231 lines.append('')
231 lines.append('')
232 for p in wrap_paragraphs(self.option_description):
232 for p in wrap_paragraphs(self.option_description):
233 lines.append(p)
233 lines.append(p)
234 lines.append('')
234 lines.append('')
235 print os.linesep.join(lines)
235 print os.linesep.join(lines)
236 self.print_flag_help()
236 self.print_flag_help()
237 self.print_alias_help()
237 self.print_alias_help()
238 print
238 print
239
239
240 def print_subcommands(self):
240 def print_subcommands(self):
241 """Print the subcommand part of the help."""
241 """Print the subcommand part of the help."""
242 if not self.subcommands:
242 if not self.subcommands:
243 return
243 return
244
244
245 lines = ["Subcommands"]
245 lines = ["Subcommands"]
246 lines.append('-'*len(lines[0]))
246 lines.append('-'*len(lines[0]))
247 lines.append('')
247 lines.append('')
248 for p in wrap_paragraphs(self.subcommand_description):
248 for p in wrap_paragraphs(self.subcommand_description):
249 lines.append(p)
249 lines.append(p)
250 lines.append('')
250 lines.append('')
251 for subc, (cls,help) in self.subcommands.iteritems():
251 for subc, (cls,help) in self.subcommands.iteritems():
252 lines.append("%s : %s"%(subc, cls))
252 lines.append("%s : %s"%(subc, cls))
253 if help:
253 if help:
254 lines.append(indent(dedent(help.strip())))
254 lines.append(indent(dedent(help.strip())))
255 lines.append('')
255 lines.append('')
256 print os.linesep.join(lines)
256 print os.linesep.join(lines)
257
257
258 def print_help(self, classes=False):
258 def print_help(self, classes=False):
259 """Print the help for each Configurable class in self.classes.
259 """Print the help for each Configurable class in self.classes.
260
260
261 If classes=False (the default), only flags and aliases are printed.
261 If classes=False (the default), only flags and aliases are printed.
262 """
262 """
263 self.print_subcommands()
263 self.print_subcommands()
264 self.print_options()
264 self.print_options()
265
265
266 if classes:
266 if classes:
267 if self.classes:
267 if self.classes:
268 print "Class parameters"
268 print "Class parameters"
269 print "----------------"
269 print "----------------"
270 print
270 print
271 for p in wrap_paragraphs(self.keyvalue_description):
271 for p in wrap_paragraphs(self.keyvalue_description):
272 print p
272 print p
273 print
273 print
274
274
275 for cls in self.classes:
275 for cls in self.classes:
276 cls.class_print_help()
276 cls.class_print_help()
277 print
277 print
278 else:
278 else:
279 print "To see all available configurables, use `--help-all`"
279 print "To see all available configurables, use `--help-all`"
280 print
280 print
281
281
282 def print_description(self):
282 def print_description(self):
283 """Print the application description."""
283 """Print the application description."""
284 for p in wrap_paragraphs(self.description):
284 for p in wrap_paragraphs(self.description):
285 print p
285 print p
286 print
286 print
287
287
288 def print_version(self):
288 def print_version(self):
289 """Print the version string."""
289 """Print the version string."""
290 print self.version
290 print self.version
291
291
292 def update_config(self, config):
292 def update_config(self, config):
293 """Fire the traits events when the config is updated."""
293 """Fire the traits events when the config is updated."""
294 # Save a copy of the current config.
294 # Save a copy of the current config.
295 newconfig = deepcopy(self.config)
295 newconfig = deepcopy(self.config)
296 # Merge the new config into the current one.
296 # Merge the new config into the current one.
297 newconfig._merge(config)
297 newconfig._merge(config)
298 # Save the combined config as self.config, which triggers the traits
298 # Save the combined config as self.config, which triggers the traits
299 # events.
299 # events.
300 self.config = newconfig
300 self.config = newconfig
301
301
302 def initialize_subcommand(self, subc, argv=None):
302 def initialize_subcommand(self, subc, argv=None):
303 """Initialize a subcommand with argv."""
303 """Initialize a subcommand with argv."""
304 subapp,help = self.subcommands.get(subc)
304 subapp,help = self.subcommands.get(subc)
305
305
306 if isinstance(subapp, basestring):
306 if isinstance(subapp, basestring):
307 subapp = import_item(subapp)
307 subapp = import_item(subapp)
308
308
309 # clear existing instances
309 # clear existing instances
310 self.__class__.clear_instance()
310 self.__class__.clear_instance()
311 # instantiate
311 # instantiate
312 self.subapp = subapp.instance()
312 self.subapp = subapp.instance()
313 # and initialize subapp
313 # and initialize subapp
314 self.subapp.initialize(argv)
314 self.subapp.initialize(argv)
315
315
316 def parse_command_line(self, argv=None):
316 def parse_command_line(self, argv=None):
317 """Parse the command line arguments."""
317 """Parse the command line arguments."""
318 argv = sys.argv[1:] if argv is None else argv
318 argv = sys.argv[1:] if argv is None else argv
319
319
320 if self.subcommands and len(argv) > 0:
320 if self.subcommands and len(argv) > 0:
321 # we have subcommands, and one may have been specified
321 # we have subcommands, and one may have been specified
322 subc, subargv = argv[0], argv[1:]
322 subc, subargv = argv[0], argv[1:]
323 if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands:
323 if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands:
324 # it's a subcommand, and *not* a flag or class parameter
324 # it's a subcommand, and *not* a flag or class parameter
325 return self.initialize_subcommand(subc, subargv)
325 return self.initialize_subcommand(subc, subargv)
326
326
327 if '-h' in argv or '--help' in argv or '--help-all' in argv:
327 if '-h' in argv or '--help' in argv or '--help-all' in argv:
328 self.print_description()
328 self.print_description()
329 self.print_help('--help-all' in argv)
329 self.print_help('--help-all' in argv)
330 self.exit(0)
330 self.exit(0)
331
331
332 if '--version' in argv:
332 if '--version' in argv:
333 self.print_version()
333 self.print_version()
334 self.exit(0)
334 self.exit(0)
335
335
336 loader = KeyValueConfigLoader(argv=argv, aliases=self.aliases,
336 loader = KeyValueConfigLoader(argv=argv, aliases=self.aliases,
337 flags=self.flags)
337 flags=self.flags)
338 try:
338 try:
339 config = loader.load_config()
339 config = loader.load_config()
340 self.update_config(config)
340 self.update_config(config)
341 except (TraitError, ArgumentError) as e:
341 except (TraitError, ArgumentError) as e:
342 self.print_description()
342 self.print_description()
343 self.print_help()
343 self.print_help()
344 self.log.fatal(str(e))
344 self.log.fatal(str(e))
345 self.exit(1)
345 self.exit(1)
346 # store unparsed args in extra_args
346 # store unparsed args in extra_args
347 self.extra_args = loader.extra_args
347 self.extra_args = loader.extra_args
348
348
349 def load_config_file(self, filename, path=None):
349 def load_config_file(self, filename, path=None):
350 """Load a .py based config file by filename and path."""
350 """Load a .py based config file by filename and path."""
351 loader = PyFileConfigLoader(filename, path=path)
351 loader = PyFileConfigLoader(filename, path=path)
352 config = loader.load_config()
352 config = loader.load_config()
353 self.update_config(config)
353 self.update_config(config)
354
354
355 def generate_config_file(self):
355 def generate_config_file(self):
356 """generate default config file from Configurables"""
356 """generate default config file from Configurables"""
357 lines = ["# Configuration file for %s."%self.name]
357 lines = ["# Configuration file for %s."%self.name]
358 lines.append('')
358 lines.append('')
359 lines.append('c = get_config()')
359 lines.append('c = get_config()')
360 lines.append('')
360 lines.append('')
361 for cls in self.classes:
361 for cls in self.classes:
362 lines.append(cls.class_config_section())
362 lines.append(cls.class_config_section())
363 return '\n'.join(lines)
363 return '\n'.join(lines)
364
364
365 def exit(self, exit_status=0):
365 def exit(self, exit_status=0):
366 self.log.debug("Exiting application: %s" % self.name)
366 self.log.debug("Exiting application: %s" % self.name)
367 sys.exit(exit_status)
367 sys.exit(exit_status)
368
368
369 #-----------------------------------------------------------------------------
369 #-----------------------------------------------------------------------------
370 # utility functions, for convenience
370 # utility functions, for convenience
371 #-----------------------------------------------------------------------------
371 #-----------------------------------------------------------------------------
372
372
373 def boolean_flag(name, configurable, set_help='', unset_help=''):
373 def boolean_flag(name, configurable, set_help='', unset_help=''):
374 """Helper for building basic --trait, --no-trait flags.
374 """Helper for building basic --trait, --no-trait flags.
375
375
376 Parameters
376 Parameters
377 ----------
377 ----------
378
378
379 name : str
379 name : str
380 The name of the flag.
380 The name of the flag.
381 configurable : str
381 configurable : str
382 The 'Class.trait' string of the trait to be set/unset with the flag
382 The 'Class.trait' string of the trait to be set/unset with the flag
383 set_help : unicode
383 set_help : unicode
384 help string for --name flag
384 help string for --name flag
385 unset_help : unicode
385 unset_help : unicode
386 help string for --no-name flag
386 help string for --no-name flag
387
387
388 Returns
388 Returns
389 -------
389 -------
390
390
391 cfg : dict
391 cfg : dict
392 A dict with two keys: 'name', and 'no-name', for setting and unsetting
392 A dict with two keys: 'name', and 'no-name', for setting and unsetting
393 the trait, respectively.
393 the trait, respectively.
394 """
394 """
395 # default helpstrings
395 # default helpstrings
396 set_help = set_help or "set %s=True"%configurable
396 set_help = set_help or "set %s=True"%configurable
397 unset_help = unset_help or "set %s=False"%configurable
397 unset_help = unset_help or "set %s=False"%configurable
398
398
399 cls,trait = configurable.split('.')
399 cls,trait = configurable.split('.')
400
400
401 setter = {cls : {trait : True}}
401 setter = {cls : {trait : True}}
402 unsetter = {cls : {trait : False}}
402 unsetter = {cls : {trait : False}}
403 return {name : (setter, set_help), 'no-'+name : (unsetter, unset_help)}
403 return {name : (setter, set_help), 'no-'+name : (unsetter, unset_help)}
404
404
@@ -1,569 +1,588 b''
1 """A simple configuration system.
1 """A simple configuration system.
2
2
3 Authors
3 Authors
4 -------
4 -------
5 * Brian Granger
5 * Brian Granger
6 * Fernando Perez
6 * Fernando Perez
7 * Min RK
7 * Min RK
8 """
8 """
9
9
10 #-----------------------------------------------------------------------------
10 #-----------------------------------------------------------------------------
11 # Copyright (C) 2008-2011 The IPython Development Team
11 # Copyright (C) 2008-2011 The IPython Development Team
12 #
12 #
13 # Distributed under the terms of the BSD License. The full license is in
13 # Distributed under the terms of the BSD License. The full license is in
14 # the file COPYING, distributed as part of this software.
14 # the file COPYING, distributed as part of this software.
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16
16
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18 # Imports
18 # Imports
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20
20
21 import __builtin__
21 import __builtin__
22 import re
22 import re
23 import sys
23 import sys
24
24
25 from IPython.external import argparse
25 from IPython.external import argparse
26 from IPython.utils.path import filefind, get_ipython_dir
26 from IPython.utils.path import filefind, get_ipython_dir
27 from IPython.utils import warn
27
28
28 #-----------------------------------------------------------------------------
29 #-----------------------------------------------------------------------------
29 # Exceptions
30 # Exceptions
30 #-----------------------------------------------------------------------------
31 #-----------------------------------------------------------------------------
31
32
32
33
33 class ConfigError(Exception):
34 class ConfigError(Exception):
34 pass
35 pass
35
36
36
37
37 class ConfigLoaderError(ConfigError):
38 class ConfigLoaderError(ConfigError):
38 pass
39 pass
39
40
40 class ArgumentError(ConfigLoaderError):
41 class ArgumentError(ConfigLoaderError):
41 pass
42 pass
42
43
43 #-----------------------------------------------------------------------------
44 #-----------------------------------------------------------------------------
44 # Argparse fix
45 # Argparse fix
45 #-----------------------------------------------------------------------------
46 #-----------------------------------------------------------------------------
46
47
47 # Unfortunately argparse by default prints help messages to stderr instead of
48 # Unfortunately argparse by default prints help messages to stderr instead of
48 # stdout. This makes it annoying to capture long help screens at the command
49 # stdout. This makes it annoying to capture long help screens at the command
49 # line, since one must know how to pipe stderr, which many users don't know how
50 # line, since one must know how to pipe stderr, which many users don't know how
50 # to do. So we override the print_help method with one that defaults to
51 # to do. So we override the print_help method with one that defaults to
51 # stdout and use our class instead.
52 # stdout and use our class instead.
52
53
53 class ArgumentParser(argparse.ArgumentParser):
54 class ArgumentParser(argparse.ArgumentParser):
54 """Simple argparse subclass that prints help to stdout by default."""
55 """Simple argparse subclass that prints help to stdout by default."""
55
56
56 def print_help(self, file=None):
57 def print_help(self, file=None):
57 if file is None:
58 if file is None:
58 file = sys.stdout
59 file = sys.stdout
59 return super(ArgumentParser, self).print_help(file)
60 return super(ArgumentParser, self).print_help(file)
60
61
61 print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
62 print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
62
63
63 #-----------------------------------------------------------------------------
64 #-----------------------------------------------------------------------------
64 # Config class for holding config information
65 # Config class for holding config information
65 #-----------------------------------------------------------------------------
66 #-----------------------------------------------------------------------------
66
67
67
68
68 class Config(dict):
69 class Config(dict):
69 """An attribute based dict that can do smart merges."""
70 """An attribute based dict that can do smart merges."""
70
71
71 def __init__(self, *args, **kwds):
72 def __init__(self, *args, **kwds):
72 dict.__init__(self, *args, **kwds)
73 dict.__init__(self, *args, **kwds)
73 # This sets self.__dict__ = self, but it has to be done this way
74 # This sets self.__dict__ = self, but it has to be done this way
74 # because we are also overriding __setattr__.
75 # because we are also overriding __setattr__.
75 dict.__setattr__(self, '__dict__', self)
76 dict.__setattr__(self, '__dict__', self)
76
77
77 def _merge(self, other):
78 def _merge(self, other):
78 to_update = {}
79 to_update = {}
79 for k, v in other.iteritems():
80 for k, v in other.iteritems():
80 if not self.has_key(k):
81 if not self.has_key(k):
81 to_update[k] = v
82 to_update[k] = v
82 else: # I have this key
83 else: # I have this key
83 if isinstance(v, Config):
84 if isinstance(v, Config):
84 # Recursively merge common sub Configs
85 # Recursively merge common sub Configs
85 self[k]._merge(v)
86 self[k]._merge(v)
86 else:
87 else:
87 # Plain updates for non-Configs
88 # Plain updates for non-Configs
88 to_update[k] = v
89 to_update[k] = v
89
90
90 self.update(to_update)
91 self.update(to_update)
91
92
92 def _is_section_key(self, key):
93 def _is_section_key(self, key):
93 if key[0].upper()==key[0] and not key.startswith('_'):
94 if key[0].upper()==key[0] and not key.startswith('_'):
94 return True
95 return True
95 else:
96 else:
96 return False
97 return False
97
98
98 def __contains__(self, key):
99 def __contains__(self, key):
99 if self._is_section_key(key):
100 if self._is_section_key(key):
100 return True
101 return True
101 else:
102 else:
102 return super(Config, self).__contains__(key)
103 return super(Config, self).__contains__(key)
103 # .has_key is deprecated for dictionaries.
104 # .has_key is deprecated for dictionaries.
104 has_key = __contains__
105 has_key = __contains__
105
106
106 def _has_section(self, key):
107 def _has_section(self, key):
107 if self._is_section_key(key):
108 if self._is_section_key(key):
108 if super(Config, self).__contains__(key):
109 if super(Config, self).__contains__(key):
109 return True
110 return True
110 return False
111 return False
111
112
112 def copy(self):
113 def copy(self):
113 return type(self)(dict.copy(self))
114 return type(self)(dict.copy(self))
114
115
115 def __copy__(self):
116 def __copy__(self):
116 return self.copy()
117 return self.copy()
117
118
118 def __deepcopy__(self, memo):
119 def __deepcopy__(self, memo):
119 import copy
120 import copy
120 return type(self)(copy.deepcopy(self.items()))
121 return type(self)(copy.deepcopy(self.items()))
121
122
122 def __getitem__(self, key):
123 def __getitem__(self, key):
123 # We cannot use directly self._is_section_key, because it triggers
124 # We cannot use directly self._is_section_key, because it triggers
124 # infinite recursion on top of PyPy. Instead, we manually fish the
125 # infinite recursion on top of PyPy. Instead, we manually fish the
125 # bound method.
126 # bound method.
126 is_section_key = self.__class__._is_section_key.__get__(self)
127 is_section_key = self.__class__._is_section_key.__get__(self)
127
128
128 # Because we use this for an exec namespace, we need to delegate
129 # Because we use this for an exec namespace, we need to delegate
129 # the lookup of names in __builtin__ to itself. This means
130 # the lookup of names in __builtin__ to itself. This means
130 # that you can't have section or attribute names that are
131 # that you can't have section or attribute names that are
131 # builtins.
132 # builtins.
132 try:
133 try:
133 return getattr(__builtin__, key)
134 return getattr(__builtin__, key)
134 except AttributeError:
135 except AttributeError:
135 pass
136 pass
136 if is_section_key(key):
137 if is_section_key(key):
137 try:
138 try:
138 return dict.__getitem__(self, key)
139 return dict.__getitem__(self, key)
139 except KeyError:
140 except KeyError:
140 c = Config()
141 c = Config()
141 dict.__setitem__(self, key, c)
142 dict.__setitem__(self, key, c)
142 return c
143 return c
143 else:
144 else:
144 return dict.__getitem__(self, key)
145 return dict.__getitem__(self, key)
145
146
146 def __setitem__(self, key, value):
147 def __setitem__(self, key, value):
147 # Don't allow names in __builtin__ to be modified.
148 # Don't allow names in __builtin__ to be modified.
148 if hasattr(__builtin__, key):
149 if hasattr(__builtin__, key):
149 raise ConfigError('Config variable names cannot have the same name '
150 raise ConfigError('Config variable names cannot have the same name '
150 'as a Python builtin: %s' % key)
151 'as a Python builtin: %s' % key)
151 if self._is_section_key(key):
152 if self._is_section_key(key):
152 if not isinstance(value, Config):
153 if not isinstance(value, Config):
153 raise ValueError('values whose keys begin with an uppercase '
154 raise ValueError('values whose keys begin with an uppercase '
154 'char must be Config instances: %r, %r' % (key, value))
155 'char must be Config instances: %r, %r' % (key, value))
155 else:
156 else:
156 dict.__setitem__(self, key, value)
157 dict.__setitem__(self, key, value)
157
158
158 def __getattr__(self, key):
159 def __getattr__(self, key):
159 try:
160 try:
160 return self.__getitem__(key)
161 return self.__getitem__(key)
161 except KeyError, e:
162 except KeyError, e:
162 raise AttributeError(e)
163 raise AttributeError(e)
163
164
164 def __setattr__(self, key, value):
165 def __setattr__(self, key, value):
165 try:
166 try:
166 self.__setitem__(key, value)
167 self.__setitem__(key, value)
167 except KeyError, e:
168 except KeyError, e:
168 raise AttributeError(e)
169 raise AttributeError(e)
169
170
170 def __delattr__(self, key):
171 def __delattr__(self, key):
171 try:
172 try:
172 dict.__delitem__(self, key)
173 dict.__delitem__(self, key)
173 except KeyError, e:
174 except KeyError, e:
174 raise AttributeError(e)
175 raise AttributeError(e)
175
176
176
177
177 #-----------------------------------------------------------------------------
178 #-----------------------------------------------------------------------------
178 # Config loading classes
179 # Config loading classes
179 #-----------------------------------------------------------------------------
180 #-----------------------------------------------------------------------------
180
181
181
182
182 class ConfigLoader(object):
183 class ConfigLoader(object):
183 """A object for loading configurations from just about anywhere.
184 """A object for loading configurations from just about anywhere.
184
185
185 The resulting configuration is packaged as a :class:`Struct`.
186 The resulting configuration is packaged as a :class:`Struct`.
186
187
187 Notes
188 Notes
188 -----
189 -----
189 A :class:`ConfigLoader` does one thing: load a config from a source
190 A :class:`ConfigLoader` does one thing: load a config from a source
190 (file, command line arguments) and returns the data as a :class:`Struct`.
191 (file, command line arguments) and returns the data as a :class:`Struct`.
191 There are lots of things that :class:`ConfigLoader` does not do. It does
192 There are lots of things that :class:`ConfigLoader` does not do. It does
192 not implement complex logic for finding config files. It does not handle
193 not implement complex logic for finding config files. It does not handle
193 default values or merge multiple configs. These things need to be
194 default values or merge multiple configs. These things need to be
194 handled elsewhere.
195 handled elsewhere.
195 """
196 """
196
197
197 def __init__(self):
198 def __init__(self):
198 """A base class for config loaders.
199 """A base class for config loaders.
199
200
200 Examples
201 Examples
201 --------
202 --------
202
203
203 >>> cl = ConfigLoader()
204 >>> cl = ConfigLoader()
204 >>> config = cl.load_config()
205 >>> config = cl.load_config()
205 >>> config
206 >>> config
206 {}
207 {}
207 """
208 """
208 self.clear()
209 self.clear()
209
210
210 def clear(self):
211 def clear(self):
211 self.config = Config()
212 self.config = Config()
212
213
213 def load_config(self):
214 def load_config(self):
214 """Load a config from somewhere, return a :class:`Config` instance.
215 """Load a config from somewhere, return a :class:`Config` instance.
215
216
216 Usually, this will cause self.config to be set and then returned.
217 Usually, this will cause self.config to be set and then returned.
217 However, in most cases, :meth:`ConfigLoader.clear` should be called
218 However, in most cases, :meth:`ConfigLoader.clear` should be called
218 to erase any previous state.
219 to erase any previous state.
219 """
220 """
220 self.clear()
221 self.clear()
221 return self.config
222 return self.config
222
223
223
224
224 class FileConfigLoader(ConfigLoader):
225 class FileConfigLoader(ConfigLoader):
225 """A base class for file based configurations.
226 """A base class for file based configurations.
226
227
227 As we add more file based config loaders, the common logic should go
228 As we add more file based config loaders, the common logic should go
228 here.
229 here.
229 """
230 """
230 pass
231 pass
231
232
232
233
233 class PyFileConfigLoader(FileConfigLoader):
234 class PyFileConfigLoader(FileConfigLoader):
234 """A config loader for pure python files.
235 """A config loader for pure python files.
235
236
236 This calls execfile on a plain python file and looks for attributes
237 This calls execfile on a plain python file and looks for attributes
237 that are all caps. These attribute are added to the config Struct.
238 that are all caps. These attribute are added to the config Struct.
238 """
239 """
239
240
240 def __init__(self, filename, path=None):
241 def __init__(self, filename, path=None):
241 """Build a config loader for a filename and path.
242 """Build a config loader for a filename and path.
242
243
243 Parameters
244 Parameters
244 ----------
245 ----------
245 filename : str
246 filename : str
246 The file name of the config file.
247 The file name of the config file.
247 path : str, list, tuple
248 path : str, list, tuple
248 The path to search for the config file on, or a sequence of
249 The path to search for the config file on, or a sequence of
249 paths to try in order.
250 paths to try in order.
250 """
251 """
251 super(PyFileConfigLoader, self).__init__()
252 super(PyFileConfigLoader, self).__init__()
252 self.filename = filename
253 self.filename = filename
253 self.path = path
254 self.path = path
254 self.full_filename = ''
255 self.full_filename = ''
255 self.data = None
256 self.data = None
256
257
257 def load_config(self):
258 def load_config(self):
258 """Load the config from a file and return it as a Struct."""
259 """Load the config from a file and return it as a Struct."""
259 self.clear()
260 self.clear()
260 self._find_file()
261 self._find_file()
261 self._read_file_as_dict()
262 self._read_file_as_dict()
262 self._convert_to_config()
263 self._convert_to_config()
263 return self.config
264 return self.config
264
265
265 def _find_file(self):
266 def _find_file(self):
266 """Try to find the file by searching the paths."""
267 """Try to find the file by searching the paths."""
267 self.full_filename = filefind(self.filename, self.path)
268 self.full_filename = filefind(self.filename, self.path)
268
269
269 def _read_file_as_dict(self):
270 def _read_file_as_dict(self):
270 """Load the config file into self.config, with recursive loading."""
271 """Load the config file into self.config, with recursive loading."""
271 # This closure is made available in the namespace that is used
272 # This closure is made available in the namespace that is used
272 # to exec the config file. It allows users to call
273 # to exec the config file. It allows users to call
273 # load_subconfig('myconfig.py') to load config files recursively.
274 # load_subconfig('myconfig.py') to load config files recursively.
274 # It needs to be a closure because it has references to self.path
275 # It needs to be a closure because it has references to self.path
275 # and self.config. The sub-config is loaded with the same path
276 # and self.config. The sub-config is loaded with the same path
276 # as the parent, but it uses an empty config which is then merged
277 # as the parent, but it uses an empty config which is then merged
277 # with the parents.
278 # with the parents.
278
279
279 # If a profile is specified, the config file will be loaded
280 # If a profile is specified, the config file will be loaded
280 # from that profile
281 # from that profile
281
282
282 def load_subconfig(fname, profile=None):
283 def load_subconfig(fname, profile=None):
283 # import here to prevent circular imports
284 # import here to prevent circular imports
284 from IPython.core.profiledir import ProfileDir, ProfileDirError
285 from IPython.core.profiledir import ProfileDir, ProfileDirError
285 if profile is not None:
286 if profile is not None:
286 try:
287 try:
287 profile_dir = ProfileDir.find_profile_dir_by_name(
288 profile_dir = ProfileDir.find_profile_dir_by_name(
288 get_ipython_dir(),
289 get_ipython_dir(),
289 profile,
290 profile,
290 )
291 )
291 except ProfileDirError:
292 except ProfileDirError:
292 return
293 return
293 path = profile_dir.location
294 path = profile_dir.location
294 else:
295 else:
295 path = self.path
296 path = self.path
296 loader = PyFileConfigLoader(fname, path)
297 loader = PyFileConfigLoader(fname, path)
297 try:
298 try:
298 sub_config = loader.load_config()
299 sub_config = loader.load_config()
299 except IOError:
300 except IOError:
300 # Pass silently if the sub config is not there. This happens
301 # Pass silently if the sub config is not there. This happens
301 # when a user s using a profile, but not the default config.
302 # when a user s using a profile, but not the default config.
302 pass
303 pass
303 else:
304 else:
304 self.config._merge(sub_config)
305 self.config._merge(sub_config)
305
306
306 # Again, this needs to be a closure and should be used in config
307 # Again, this needs to be a closure and should be used in config
307 # files to get the config being loaded.
308 # files to get the config being loaded.
308 def get_config():
309 def get_config():
309 return self.config
310 return self.config
310
311
311 namespace = dict(load_subconfig=load_subconfig, get_config=get_config)
312 namespace = dict(load_subconfig=load_subconfig, get_config=get_config)
312 fs_encoding = sys.getfilesystemencoding() or 'ascii'
313 fs_encoding = sys.getfilesystemencoding() or 'ascii'
313 conf_filename = self.full_filename.encode(fs_encoding)
314 conf_filename = self.full_filename.encode(fs_encoding)
314 execfile(conf_filename, namespace)
315 execfile(conf_filename, namespace)
315
316
316 def _convert_to_config(self):
317 def _convert_to_config(self):
317 if self.data is None:
318 if self.data is None:
318 ConfigLoaderError('self.data does not exist')
319 ConfigLoaderError('self.data does not exist')
319
320
320
321
321 class CommandLineConfigLoader(ConfigLoader):
322 class CommandLineConfigLoader(ConfigLoader):
322 """A config loader for command line arguments.
323 """A config loader for command line arguments.
323
324
324 As we add more command line based loaders, the common logic should go
325 As we add more command line based loaders, the common logic should go
325 here.
326 here.
326 """
327 """
327
328
328 kv_pattern = re.compile(r'\-\-[A-Za-z]\w*(\.\w+)*\=.*')
329 # raw --identifier=value pattern
330 # but *also* accept '-' as wordsep, for aliases
331 # accepts: --foo=a
332 # --Class.trait=value
333 # --alias-name=value
334 # rejects: -foo=value
335 # --foo
336 # --Class.trait
337 kv_pattern = re.compile(r'\-\-[A-Za-z][\w\-]*(\.[\w\-]+)*\=.*')
338
339 # just flags, no assignments, with two *or one* leading '-'
340 # accepts: --foo
341 # -foo-bar-again
342 # rejects: --anything=anything
343 # --two.word
344
329 flag_pattern = re.compile(r'\-\-?\w+[\-\w]*$')
345 flag_pattern = re.compile(r'\-\-?\w+[\-\w]*$')
330
346
331 class KeyValueConfigLoader(CommandLineConfigLoader):
347 class KeyValueConfigLoader(CommandLineConfigLoader):
332 """A config loader that loads key value pairs from the command line.
348 """A config loader that loads key value pairs from the command line.
333
349
334 This allows command line options to be gives in the following form::
350 This allows command line options to be gives in the following form::
335
351
336 ipython --profile="foo" --InteractiveShell.autocall=False
352 ipython --profile="foo" --InteractiveShell.autocall=False
337 """
353 """
338
354
339 def __init__(self, argv=None, aliases=None, flags=None):
355 def __init__(self, argv=None, aliases=None, flags=None):
340 """Create a key value pair config loader.
356 """Create a key value pair config loader.
341
357
342 Parameters
358 Parameters
343 ----------
359 ----------
344 argv : list
360 argv : list
345 A list that has the form of sys.argv[1:] which has unicode
361 A list that has the form of sys.argv[1:] which has unicode
346 elements of the form u"key=value". If this is None (default),
362 elements of the form u"key=value". If this is None (default),
347 then sys.argv[1:] will be used.
363 then sys.argv[1:] will be used.
348 aliases : dict
364 aliases : dict
349 A dict of aliases for configurable traits.
365 A dict of aliases for configurable traits.
350 Keys are the short aliases, Values are the resolved trait.
366 Keys are the short aliases, Values are the resolved trait.
351 Of the form: `{'alias' : 'Configurable.trait'}`
367 Of the form: `{'alias' : 'Configurable.trait'}`
352 flags : dict
368 flags : dict
353 A dict of flags, keyed by str name. Vaues can be Config objects,
369 A dict of flags, keyed by str name. Vaues can be Config objects,
354 dicts, or "key=value" strings. If Config or dict, when the flag
370 dicts, or "key=value" strings. If Config or dict, when the flag
355 is triggered, The flag is loaded as `self.config.update(m)`.
371 is triggered, The flag is loaded as `self.config.update(m)`.
356
372
357 Returns
373 Returns
358 -------
374 -------
359 config : Config
375 config : Config
360 The resulting Config object.
376 The resulting Config object.
361
377
362 Examples
378 Examples
363 --------
379 --------
364
380
365 >>> from IPython.config.loader import KeyValueConfigLoader
381 >>> from IPython.config.loader import KeyValueConfigLoader
366 >>> cl = KeyValueConfigLoader()
382 >>> cl = KeyValueConfigLoader()
367 >>> cl.load_config(["--foo='bar'","--A.name='brian'","--B.number=0"])
383 >>> cl.load_config(["--A.name='brian'","--B.number=0"])
368 {'A': {'name': 'brian'}, 'B': {'number': 0}, 'foo': 'bar'}
384 {'A': {'name': 'brian'}, 'B': {'number': 0}}
369 """
385 """
370 self.clear()
386 self.clear()
371 if argv is None:
387 if argv is None:
372 argv = sys.argv[1:]
388 argv = sys.argv[1:]
373 self.argv = argv
389 self.argv = argv
374 self.aliases = aliases or {}
390 self.aliases = aliases or {}
375 self.flags = flags or {}
391 self.flags = flags or {}
376
392
377
393
378 def clear(self):
394 def clear(self):
379 super(KeyValueConfigLoader, self).clear()
395 super(KeyValueConfigLoader, self).clear()
380 self.extra_args = []
396 self.extra_args = []
381
397
382
398
383 def _decode_argv(self, argv, enc=None):
399 def _decode_argv(self, argv, enc=None):
384 """decode argv if bytes, using stin.encoding, falling back on default enc"""
400 """decode argv if bytes, using stin.encoding, falling back on default enc"""
385 uargv = []
401 uargv = []
386 if enc is None:
402 if enc is None:
387 enc = sys.stdin.encoding or sys.getdefaultencoding()
403 enc = sys.stdin.encoding or sys.getdefaultencoding()
388 for arg in argv:
404 for arg in argv:
389 if not isinstance(arg, unicode):
405 if not isinstance(arg, unicode):
390 # only decode if not already decoded
406 # only decode if not already decoded
391 arg = arg.decode(enc)
407 arg = arg.decode(enc)
392 uargv.append(arg)
408 uargv.append(arg)
393 return uargv
409 return uargv
394
410
395
411
396 def load_config(self, argv=None, aliases=None, flags=None):
412 def load_config(self, argv=None, aliases=None, flags=None):
397 """Parse the configuration and generate the Config object.
413 """Parse the configuration and generate the Config object.
398
414
399 After loading, any arguments that are not key-value or
415 After loading, any arguments that are not key-value or
400 flags will be stored in self.extra_args - a list of
416 flags will be stored in self.extra_args - a list of
401 unparsed command-line arguments. This is used for
417 unparsed command-line arguments. This is used for
402 arguments such as input files or subcommands.
418 arguments such as input files or subcommands.
403
419
404 Parameters
420 Parameters
405 ----------
421 ----------
406 argv : list, optional
422 argv : list, optional
407 A list that has the form of sys.argv[1:] which has unicode
423 A list that has the form of sys.argv[1:] which has unicode
408 elements of the form u"key=value". If this is None (default),
424 elements of the form u"key=value". If this is None (default),
409 then self.argv will be used.
425 then self.argv will be used.
410 aliases : dict
426 aliases : dict
411 A dict of aliases for configurable traits.
427 A dict of aliases for configurable traits.
412 Keys are the short aliases, Values are the resolved trait.
428 Keys are the short aliases, Values are the resolved trait.
413 Of the form: `{'alias' : 'Configurable.trait'}`
429 Of the form: `{'alias' : 'Configurable.trait'}`
414 flags : dict
430 flags : dict
415 A dict of flags, keyed by str name. Values can be Config objects
431 A dict of flags, keyed by str name. Values can be Config objects
416 or dicts. When the flag is triggered, The config is loaded as
432 or dicts. When the flag is triggered, The config is loaded as
417 `self.config.update(cfg)`.
433 `self.config.update(cfg)`.
418 """
434 """
419 from IPython.config.configurable import Configurable
435 from IPython.config.configurable import Configurable
420
436
421 self.clear()
437 self.clear()
422 if argv is None:
438 if argv is None:
423 argv = self.argv
439 argv = self.argv
424 if aliases is None:
440 if aliases is None:
425 aliases = self.aliases
441 aliases = self.aliases
426 if flags is None:
442 if flags is None:
427 flags = self.flags
443 flags = self.flags
428
444
429 # ensure argv is a list of unicode strings:
445 # ensure argv is a list of unicode strings:
430 uargv = self._decode_argv(argv)
446 uargv = self._decode_argv(argv)
431 for idx,raw in enumerate(uargv):
447 for idx,raw in enumerate(uargv):
432 # strip leading '-'
448 # strip leading '-'
433 item = raw.lstrip('-')
449 item = raw.lstrip('-')
434
450
435 if raw == '--':
451 if raw == '--':
436 # don't parse arguments after '--'
452 # don't parse arguments after '--'
437 # this is useful for relaying arguments to scripts, e.g.
453 # this is useful for relaying arguments to scripts, e.g.
438 # ipython -i foo.py --pylab=qt -- args after '--' go-to-foo.py
454 # ipython -i foo.py --pylab=qt -- args after '--' go-to-foo.py
439 self.extra_args.extend(uargv[idx+1:])
455 self.extra_args.extend(uargv[idx+1:])
440 break
456 break
441
457
442 if kv_pattern.match(raw):
458 if kv_pattern.match(raw):
443 lhs,rhs = item.split('=',1)
459 lhs,rhs = item.split('=',1)
444 # Substitute longnames for aliases.
460 # Substitute longnames for aliases.
445 if lhs in aliases:
461 if lhs in aliases:
446 lhs = aliases[lhs]
462 lhs = aliases[lhs]
463 if '.' not in lhs:
464 # probably a mistyped alias, but not technically illegal
465 warn.warn("Unrecognized alias: '%s', it will probably have no effect."%lhs)
447 exec_str = 'self.config.' + lhs + '=' + rhs
466 exec_str = 'self.config.' + lhs + '=' + rhs
448 try:
467 try:
449 # Try to see if regular Python syntax will work. This
468 # Try to see if regular Python syntax will work. This
450 # won't handle strings as the quote marks are removed
469 # won't handle strings as the quote marks are removed
451 # by the system shell.
470 # by the system shell.
452 exec exec_str in locals(), globals()
471 exec exec_str in locals(), globals()
453 except (NameError, SyntaxError):
472 except (NameError, SyntaxError):
454 # This case happens if the rhs is a string but without
473 # This case happens if the rhs is a string but without
455 # the quote marks. Use repr, to get quote marks, and
474 # the quote marks. Use repr, to get quote marks, and
456 # 'u' prefix and see if
475 # 'u' prefix and see if
457 # it succeeds. If it still fails, we let it raise.
476 # it succeeds. If it still fails, we let it raise.
458 exec_str = u'self.config.' + lhs + '=' + repr(rhs)
477 exec_str = u'self.config.' + lhs + '=' + repr(rhs)
459 exec exec_str in locals(), globals()
478 exec exec_str in locals(), globals()
460 elif flag_pattern.match(raw):
479 elif flag_pattern.match(raw):
461 if item in flags:
480 if item in flags:
462 cfg,help = flags[item]
481 cfg,help = flags[item]
463 if isinstance(cfg, (dict, Config)):
482 if isinstance(cfg, (dict, Config)):
464 # don't clobber whole config sections, update
483 # don't clobber whole config sections, update
465 # each section from config:
484 # each section from config:
466 for sec,c in cfg.iteritems():
485 for sec,c in cfg.iteritems():
467 self.config[sec].update(c)
486 self.config[sec].update(c)
468 else:
487 else:
469 raise ValueError("Invalid flag: '%s'"%raw)
488 raise ValueError("Invalid flag: '%s'"%raw)
470 else:
489 else:
471 raise ArgumentError("Unrecognized flag: '%s'"%raw)
490 raise ArgumentError("Unrecognized flag: '%s'"%raw)
472 elif raw.startswith('-'):
491 elif raw.startswith('-'):
473 kv = '--'+item
492 kv = '--'+item
474 if kv_pattern.match(kv):
493 if kv_pattern.match(kv):
475 raise ArgumentError("Invalid argument: '%s', did you mean '%s'?"%(raw, kv))
494 raise ArgumentError("Invalid argument: '%s', did you mean '%s'?"%(raw, kv))
476 else:
495 else:
477 raise ArgumentError("Invalid argument: '%s'"%raw)
496 raise ArgumentError("Invalid argument: '%s'"%raw)
478 else:
497 else:
479 # keep all args that aren't valid in a list,
498 # keep all args that aren't valid in a list,
480 # in case our parent knows what to do with them.
499 # in case our parent knows what to do with them.
481 self.extra_args.append(item)
500 self.extra_args.append(item)
482 return self.config
501 return self.config
483
502
484 class ArgParseConfigLoader(CommandLineConfigLoader):
503 class ArgParseConfigLoader(CommandLineConfigLoader):
485 """A loader that uses the argparse module to load from the command line."""
504 """A loader that uses the argparse module to load from the command line."""
486
505
487 def __init__(self, argv=None, *parser_args, **parser_kw):
506 def __init__(self, argv=None, *parser_args, **parser_kw):
488 """Create a config loader for use with argparse.
507 """Create a config loader for use with argparse.
489
508
490 Parameters
509 Parameters
491 ----------
510 ----------
492
511
493 argv : optional, list
512 argv : optional, list
494 If given, used to read command-line arguments from, otherwise
513 If given, used to read command-line arguments from, otherwise
495 sys.argv[1:] is used.
514 sys.argv[1:] is used.
496
515
497 parser_args : tuple
516 parser_args : tuple
498 A tuple of positional arguments that will be passed to the
517 A tuple of positional arguments that will be passed to the
499 constructor of :class:`argparse.ArgumentParser`.
518 constructor of :class:`argparse.ArgumentParser`.
500
519
501 parser_kw : dict
520 parser_kw : dict
502 A tuple of keyword arguments that will be passed to the
521 A tuple of keyword arguments that will be passed to the
503 constructor of :class:`argparse.ArgumentParser`.
522 constructor of :class:`argparse.ArgumentParser`.
504
523
505 Returns
524 Returns
506 -------
525 -------
507 config : Config
526 config : Config
508 The resulting Config object.
527 The resulting Config object.
509 """
528 """
510 super(CommandLineConfigLoader, self).__init__()
529 super(CommandLineConfigLoader, self).__init__()
511 if argv == None:
530 if argv == None:
512 argv = sys.argv[1:]
531 argv = sys.argv[1:]
513 self.argv = argv
532 self.argv = argv
514 self.parser_args = parser_args
533 self.parser_args = parser_args
515 self.version = parser_kw.pop("version", None)
534 self.version = parser_kw.pop("version", None)
516 kwargs = dict(argument_default=argparse.SUPPRESS)
535 kwargs = dict(argument_default=argparse.SUPPRESS)
517 kwargs.update(parser_kw)
536 kwargs.update(parser_kw)
518 self.parser_kw = kwargs
537 self.parser_kw = kwargs
519
538
520 def load_config(self, argv=None):
539 def load_config(self, argv=None):
521 """Parse command line arguments and return as a Config object.
540 """Parse command line arguments and return as a Config object.
522
541
523 Parameters
542 Parameters
524 ----------
543 ----------
525
544
526 args : optional, list
545 args : optional, list
527 If given, a list with the structure of sys.argv[1:] to parse
546 If given, a list with the structure of sys.argv[1:] to parse
528 arguments from. If not given, the instance's self.argv attribute
547 arguments from. If not given, the instance's self.argv attribute
529 (given at construction time) is used."""
548 (given at construction time) is used."""
530 self.clear()
549 self.clear()
531 if argv is None:
550 if argv is None:
532 argv = self.argv
551 argv = self.argv
533 self._create_parser()
552 self._create_parser()
534 self._parse_args(argv)
553 self._parse_args(argv)
535 self._convert_to_config()
554 self._convert_to_config()
536 return self.config
555 return self.config
537
556
538 def get_extra_args(self):
557 def get_extra_args(self):
539 if hasattr(self, 'extra_args'):
558 if hasattr(self, 'extra_args'):
540 return self.extra_args
559 return self.extra_args
541 else:
560 else:
542 return []
561 return []
543
562
544 def _create_parser(self):
563 def _create_parser(self):
545 self.parser = ArgumentParser(*self.parser_args, **self.parser_kw)
564 self.parser = ArgumentParser(*self.parser_args, **self.parser_kw)
546 self._add_arguments()
565 self._add_arguments()
547
566
548 def _add_arguments(self):
567 def _add_arguments(self):
549 raise NotImplementedError("subclasses must implement _add_arguments")
568 raise NotImplementedError("subclasses must implement _add_arguments")
550
569
551 def _parse_args(self, args):
570 def _parse_args(self, args):
552 """self.parser->self.parsed_data"""
571 """self.parser->self.parsed_data"""
553 # decode sys.argv to support unicode command-line options
572 # decode sys.argv to support unicode command-line options
554 uargs = []
573 uargs = []
555 for a in args:
574 for a in args:
556 if isinstance(a, str):
575 if isinstance(a, str):
557 # don't decode if we already got unicode
576 # don't decode if we already got unicode
558 a = a.decode(sys.stdin.encoding or
577 a = a.decode(sys.stdin.encoding or
559 sys.getdefaultencoding())
578 sys.getdefaultencoding())
560 uargs.append(a)
579 uargs.append(a)
561 self.parsed_data, self.extra_args = self.parser.parse_known_args(uargs)
580 self.parsed_data, self.extra_args = self.parser.parse_known_args(uargs)
562
581
563 def _convert_to_config(self):
582 def _convert_to_config(self):
564 """self.parsed_data->self.config"""
583 """self.parsed_data->self.config"""
565 for k, v in vars(self.parsed_data).iteritems():
584 for k, v in vars(self.parsed_data).iteritems():
566 exec_str = 'self.config.' + k + '= v'
585 exec_str = 'self.config.' + k + '= v'
567 exec exec_str in locals(), globals()
586 exec exec_str in locals(), globals()
568
587
569
588
@@ -1,141 +1,146 b''
1 """
1 """
2 Tests for IPython.config.application.Application
2 Tests for IPython.config.application.Application
3
3
4 Authors:
4 Authors:
5
5
6 * Brian Granger
6 * Brian Granger
7 """
7 """
8
8
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10 # Copyright (C) 2008-2011 The IPython Development Team
10 # Copyright (C) 2008-2011 The IPython Development Team
11 #
11 #
12 # Distributed under the terms of the BSD License. The full license is in
12 # Distributed under the terms of the BSD License. The full license is in
13 # the file COPYING, distributed as part of this software.
13 # the file COPYING, distributed as part of this software.
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15
15
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17 # Imports
17 # Imports
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 from unittest import TestCase
20 from unittest import TestCase
21
21
22 from IPython.config.configurable import Configurable
22 from IPython.config.configurable import Configurable
23
23
24 from IPython.config.application import (
24 from IPython.config.application import (
25 Application
25 Application
26 )
26 )
27
27
28 from IPython.utils.traitlets import (
28 from IPython.utils.traitlets import (
29 Bool, Unicode, Int, Float, List, Dict
29 Bool, Unicode, Int, Float, List, Dict
30 )
30 )
31
31
32 #-----------------------------------------------------------------------------
32 #-----------------------------------------------------------------------------
33 # Code
33 # Code
34 #-----------------------------------------------------------------------------
34 #-----------------------------------------------------------------------------
35
35
36 class Foo(Configurable):
36 class Foo(Configurable):
37
37
38 i = Int(0, config=True, help="The integer i.")
38 i = Int(0, config=True, help="The integer i.")
39 j = Int(1, config=True, help="The integer j.")
39 j = Int(1, config=True, help="The integer j.")
40 name = Unicode(u'Brian', config=True, help="First name.")
40 name = Unicode(u'Brian', config=True, help="First name.")
41
41
42
42
43 class Bar(Configurable):
43 class Bar(Configurable):
44
44
45 b = Int(0, config=True, help="The integer b.")
45 b = Int(0, config=True, help="The integer b.")
46 enabled = Bool(True, config=True, help="Enable bar.")
46 enabled = Bool(True, config=True, help="Enable bar.")
47
47
48
48
49 class MyApp(Application):
49 class MyApp(Application):
50
50
51 name = Unicode(u'myapp')
51 name = Unicode(u'myapp')
52 running = Bool(False, config=True,
52 running = Bool(False, config=True,
53 help="Is the app running?")
53 help="Is the app running?")
54 classes = List([Bar, Foo])
54 classes = List([Bar, Foo])
55 config_file = Unicode(u'', config=True,
55 config_file = Unicode(u'', config=True,
56 help="Load this config file")
56 help="Load this config file")
57
57
58 aliases = Dict(dict(i='Foo.i',j='Foo.j',name='Foo.name',
58 aliases = Dict({
59 enabled='Bar.enabled', log_level='MyApp.log_level'))
59 'i' : 'Foo.i',
60 'j' : 'Foo.j',
61 'name' : 'Foo.name',
62 'enabled' : 'Bar.enabled',
63 'log-level' : 'MyApp.log_level',
64 })
60
65
61 flags = Dict(dict(enable=({'Bar': {'enabled' : True}}, "Set Bar.enabled to True"),
66 flags = Dict(dict(enable=({'Bar': {'enabled' : True}}, "Set Bar.enabled to True"),
62 disable=({'Bar': {'enabled' : False}}, "Set Bar.enabled to False")))
67 disable=({'Bar': {'enabled' : False}}, "Set Bar.enabled to False")))
63
68
64 def init_foo(self):
69 def init_foo(self):
65 self.foo = Foo(config=self.config)
70 self.foo = Foo(config=self.config)
66
71
67 def init_bar(self):
72 def init_bar(self):
68 self.bar = Bar(config=self.config)
73 self.bar = Bar(config=self.config)
69
74
70
75
71 class TestApplication(TestCase):
76 class TestApplication(TestCase):
72
77
73 def test_basic(self):
78 def test_basic(self):
74 app = MyApp()
79 app = MyApp()
75 self.assertEquals(app.name, u'myapp')
80 self.assertEquals(app.name, u'myapp')
76 self.assertEquals(app.running, False)
81 self.assertEquals(app.running, False)
77 self.assertEquals(app.classes, [MyApp,Bar,Foo])
82 self.assertEquals(app.classes, [MyApp,Bar,Foo])
78 self.assertEquals(app.config_file, u'')
83 self.assertEquals(app.config_file, u'')
79
84
80 def test_config(self):
85 def test_config(self):
81 app = MyApp()
86 app = MyApp()
82 app.parse_command_line(["--i=10","--Foo.j=10","--enabled=False","--log_level=50"])
87 app.parse_command_line(["--i=10","--Foo.j=10","--enabled=False","--log-level=50"])
83 config = app.config
88 config = app.config
84 self.assertEquals(config.Foo.i, 10)
89 self.assertEquals(config.Foo.i, 10)
85 self.assertEquals(config.Foo.j, 10)
90 self.assertEquals(config.Foo.j, 10)
86 self.assertEquals(config.Bar.enabled, False)
91 self.assertEquals(config.Bar.enabled, False)
87 self.assertEquals(config.MyApp.log_level,50)
92 self.assertEquals(config.MyApp.log_level,50)
88
93
89 def test_config_propagation(self):
94 def test_config_propagation(self):
90 app = MyApp()
95 app = MyApp()
91 app.parse_command_line(["--i=10","--Foo.j=10","--enabled=False","--log_level=50"])
96 app.parse_command_line(["--i=10","--Foo.j=10","--enabled=False","--log-level=50"])
92 app.init_foo()
97 app.init_foo()
93 app.init_bar()
98 app.init_bar()
94 self.assertEquals(app.foo.i, 10)
99 self.assertEquals(app.foo.i, 10)
95 self.assertEquals(app.foo.j, 10)
100 self.assertEquals(app.foo.j, 10)
96 self.assertEquals(app.bar.enabled, False)
101 self.assertEquals(app.bar.enabled, False)
97
102
98 def test_flags(self):
103 def test_flags(self):
99 app = MyApp()
104 app = MyApp()
100 app.parse_command_line(["--disable"])
105 app.parse_command_line(["--disable"])
101 app.init_bar()
106 app.init_bar()
102 self.assertEquals(app.bar.enabled, False)
107 self.assertEquals(app.bar.enabled, False)
103 app.parse_command_line(["--enable"])
108 app.parse_command_line(["--enable"])
104 app.init_bar()
109 app.init_bar()
105 self.assertEquals(app.bar.enabled, True)
110 self.assertEquals(app.bar.enabled, True)
106
111
107 def test_aliases(self):
112 def test_aliases(self):
108 app = MyApp()
113 app = MyApp()
109 app.parse_command_line(["--i=5", "--j=10"])
114 app.parse_command_line(["--i=5", "--j=10"])
110 app.init_foo()
115 app.init_foo()
111 self.assertEquals(app.foo.i, 5)
116 self.assertEquals(app.foo.i, 5)
112 app.init_foo()
117 app.init_foo()
113 self.assertEquals(app.foo.j, 10)
118 self.assertEquals(app.foo.j, 10)
114
119
115 def test_flag_clobber(self):
120 def test_flag_clobber(self):
116 """test that setting flags doesn't clobber existing settings"""
121 """test that setting flags doesn't clobber existing settings"""
117 app = MyApp()
122 app = MyApp()
118 app.parse_command_line(["--Bar.b=5", "--disable"])
123 app.parse_command_line(["--Bar.b=5", "--disable"])
119 app.init_bar()
124 app.init_bar()
120 self.assertEquals(app.bar.enabled, False)
125 self.assertEquals(app.bar.enabled, False)
121 self.assertEquals(app.bar.b, 5)
126 self.assertEquals(app.bar.b, 5)
122 app.parse_command_line(["--enable", "--Bar.b=10"])
127 app.parse_command_line(["--enable", "--Bar.b=10"])
123 app.init_bar()
128 app.init_bar()
124 self.assertEquals(app.bar.enabled, True)
129 self.assertEquals(app.bar.enabled, True)
125 self.assertEquals(app.bar.b, 10)
130 self.assertEquals(app.bar.b, 10)
126
131
127 def test_extra_args(self):
132 def test_extra_args(self):
128 app = MyApp()
133 app = MyApp()
129 app.parse_command_line(["--Bar.b=5", 'extra', "--disable", 'args'])
134 app.parse_command_line(["--Bar.b=5", 'extra', "--disable", 'args'])
130 app.init_bar()
135 app.init_bar()
131 self.assertEquals(app.bar.enabled, False)
136 self.assertEquals(app.bar.enabled, False)
132 self.assertEquals(app.bar.b, 5)
137 self.assertEquals(app.bar.b, 5)
133 self.assertEquals(app.extra_args, ['extra', 'args'])
138 self.assertEquals(app.extra_args, ['extra', 'args'])
134 app = MyApp()
139 app = MyApp()
135 app.parse_command_line(["--Bar.b=5", '--', 'extra', "--disable", 'args'])
140 app.parse_command_line(["--Bar.b=5", '--', 'extra', "--disable", 'args'])
136 app.init_bar()
141 app.init_bar()
137 self.assertEquals(app.bar.enabled, True)
142 self.assertEquals(app.bar.enabled, True)
138 self.assertEquals(app.bar.b, 5)
143 self.assertEquals(app.bar.b, 5)
139 self.assertEquals(app.extra_args, ['extra', '--disable', 'args'])
144 self.assertEquals(app.extra_args, ['extra', '--disable', 'args'])
140
145
141
146
@@ -1,219 +1,226 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 Tests for IPython.config.loader
4 Tests for IPython.config.loader
5
5
6 Authors:
6 Authors:
7
7
8 * Brian Granger
8 * Brian Granger
9 * Fernando Perez (design help)
9 * Fernando Perez (design help)
10 """
10 """
11
11
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13 # Copyright (C) 2008-2009 The IPython Development Team
13 # Copyright (C) 2008-2009 The IPython Development Team
14 #
14 #
15 # Distributed under the terms of the BSD License. The full license is in
15 # Distributed under the terms of the BSD License. The full license is in
16 # the file COPYING, distributed as part of this software.
16 # the file COPYING, distributed as part of this software.
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20 # Imports
20 # Imports
21 #-----------------------------------------------------------------------------
21 #-----------------------------------------------------------------------------
22
22
23 import os
23 import os
24 import sys
24 import sys
25 from tempfile import mkstemp
25 from tempfile import mkstemp
26 from unittest import TestCase
26 from unittest import TestCase
27
27
28 from nose import SkipTest
28 from nose import SkipTest
29
29
30 from IPython.testing.tools import mute_warn
31
30 from IPython.utils.traitlets import Int, Unicode
32 from IPython.utils.traitlets import Int, Unicode
31 from IPython.config.configurable import Configurable
33 from IPython.config.configurable import Configurable
32 from IPython.config.loader import (
34 from IPython.config.loader import (
33 Config,
35 Config,
34 PyFileConfigLoader,
36 PyFileConfigLoader,
35 KeyValueConfigLoader,
37 KeyValueConfigLoader,
36 ArgParseConfigLoader,
38 ArgParseConfigLoader,
37 ConfigError
39 ConfigError
38 )
40 )
39
41
40 #-----------------------------------------------------------------------------
42 #-----------------------------------------------------------------------------
41 # Actual tests
43 # Actual tests
42 #-----------------------------------------------------------------------------
44 #-----------------------------------------------------------------------------
43
45
44
46
45 pyfile = """
47 pyfile = """
46 c = get_config()
48 c = get_config()
47 c.a=10
49 c.a=10
48 c.b=20
50 c.b=20
49 c.Foo.Bar.value=10
51 c.Foo.Bar.value=10
50 c.Foo.Bam.value=range(10)
52 c.Foo.Bam.value=range(10)
51 c.D.C.value='hi there'
53 c.D.C.value='hi there'
52 """
54 """
53
55
54 class TestPyFileCL(TestCase):
56 class TestPyFileCL(TestCase):
55
57
56 def test_basic(self):
58 def test_basic(self):
57 fd, fname = mkstemp('.py')
59 fd, fname = mkstemp('.py')
58 f = os.fdopen(fd, 'w')
60 f = os.fdopen(fd, 'w')
59 f.write(pyfile)
61 f.write(pyfile)
60 f.close()
62 f.close()
61 # Unlink the file
63 # Unlink the file
62 cl = PyFileConfigLoader(fname)
64 cl = PyFileConfigLoader(fname)
63 config = cl.load_config()
65 config = cl.load_config()
64 self.assertEquals(config.a, 10)
66 self.assertEquals(config.a, 10)
65 self.assertEquals(config.b, 20)
67 self.assertEquals(config.b, 20)
66 self.assertEquals(config.Foo.Bar.value, 10)
68 self.assertEquals(config.Foo.Bar.value, 10)
67 self.assertEquals(config.Foo.Bam.value, range(10))
69 self.assertEquals(config.Foo.Bam.value, range(10))
68 self.assertEquals(config.D.C.value, 'hi there')
70 self.assertEquals(config.D.C.value, 'hi there')
69
71
70 class MyLoader1(ArgParseConfigLoader):
72 class MyLoader1(ArgParseConfigLoader):
71 def _add_arguments(self):
73 def _add_arguments(self):
72 p = self.parser
74 p = self.parser
73 p.add_argument('-f', '--foo', dest='Global.foo', type=str)
75 p.add_argument('-f', '--foo', dest='Global.foo', type=str)
74 p.add_argument('-b', dest='MyClass.bar', type=int)
76 p.add_argument('-b', dest='MyClass.bar', type=int)
75 p.add_argument('-n', dest='n', action='store_true')
77 p.add_argument('-n', dest='n', action='store_true')
76 p.add_argument('Global.bam', type=str)
78 p.add_argument('Global.bam', type=str)
77
79
78 class MyLoader2(ArgParseConfigLoader):
80 class MyLoader2(ArgParseConfigLoader):
79 def _add_arguments(self):
81 def _add_arguments(self):
80 subparsers = self.parser.add_subparsers(dest='subparser_name')
82 subparsers = self.parser.add_subparsers(dest='subparser_name')
81 subparser1 = subparsers.add_parser('1')
83 subparser1 = subparsers.add_parser('1')
82 subparser1.add_argument('-x',dest='Global.x')
84 subparser1.add_argument('-x',dest='Global.x')
83 subparser2 = subparsers.add_parser('2')
85 subparser2 = subparsers.add_parser('2')
84 subparser2.add_argument('y')
86 subparser2.add_argument('y')
85
87
86 class TestArgParseCL(TestCase):
88 class TestArgParseCL(TestCase):
87
89
88 def test_basic(self):
90 def test_basic(self):
89 cl = MyLoader1()
91 cl = MyLoader1()
90 config = cl.load_config('-f hi -b 10 -n wow'.split())
92 config = cl.load_config('-f hi -b 10 -n wow'.split())
91 self.assertEquals(config.Global.foo, 'hi')
93 self.assertEquals(config.Global.foo, 'hi')
92 self.assertEquals(config.MyClass.bar, 10)
94 self.assertEquals(config.MyClass.bar, 10)
93 self.assertEquals(config.n, True)
95 self.assertEquals(config.n, True)
94 self.assertEquals(config.Global.bam, 'wow')
96 self.assertEquals(config.Global.bam, 'wow')
95 config = cl.load_config(['wow'])
97 config = cl.load_config(['wow'])
96 self.assertEquals(config.keys(), ['Global'])
98 self.assertEquals(config.keys(), ['Global'])
97 self.assertEquals(config.Global.keys(), ['bam'])
99 self.assertEquals(config.Global.keys(), ['bam'])
98 self.assertEquals(config.Global.bam, 'wow')
100 self.assertEquals(config.Global.bam, 'wow')
99
101
100 def test_add_arguments(self):
102 def test_add_arguments(self):
101 cl = MyLoader2()
103 cl = MyLoader2()
102 config = cl.load_config('2 frobble'.split())
104 config = cl.load_config('2 frobble'.split())
103 self.assertEquals(config.subparser_name, '2')
105 self.assertEquals(config.subparser_name, '2')
104 self.assertEquals(config.y, 'frobble')
106 self.assertEquals(config.y, 'frobble')
105 config = cl.load_config('1 -x frobble'.split())
107 config = cl.load_config('1 -x frobble'.split())
106 self.assertEquals(config.subparser_name, '1')
108 self.assertEquals(config.subparser_name, '1')
107 self.assertEquals(config.Global.x, 'frobble')
109 self.assertEquals(config.Global.x, 'frobble')
108
110
109 def test_argv(self):
111 def test_argv(self):
110 cl = MyLoader1(argv='-f hi -b 10 -n wow'.split())
112 cl = MyLoader1(argv='-f hi -b 10 -n wow'.split())
111 config = cl.load_config()
113 config = cl.load_config()
112 self.assertEquals(config.Global.foo, 'hi')
114 self.assertEquals(config.Global.foo, 'hi')
113 self.assertEquals(config.MyClass.bar, 10)
115 self.assertEquals(config.MyClass.bar, 10)
114 self.assertEquals(config.n, True)
116 self.assertEquals(config.n, True)
115 self.assertEquals(config.Global.bam, 'wow')
117 self.assertEquals(config.Global.bam, 'wow')
116
118
117
119
118 class TestKeyValueCL(TestCase):
120 class TestKeyValueCL(TestCase):
119
121
120 def test_basic(self):
122 def test_basic(self):
121 cl = KeyValueConfigLoader()
123 cl = KeyValueConfigLoader()
122 argv = ['--'+s.strip('c.') for s in pyfile.split('\n')[2:-1]]
124 argv = ['--'+s.strip('c.') for s in pyfile.split('\n')[2:-1]]
125 with mute_warn():
123 config = cl.load_config(argv)
126 config = cl.load_config(argv)
124 self.assertEquals(config.a, 10)
127 self.assertEquals(config.a, 10)
125 self.assertEquals(config.b, 20)
128 self.assertEquals(config.b, 20)
126 self.assertEquals(config.Foo.Bar.value, 10)
129 self.assertEquals(config.Foo.Bar.value, 10)
127 self.assertEquals(config.Foo.Bam.value, range(10))
130 self.assertEquals(config.Foo.Bam.value, range(10))
128 self.assertEquals(config.D.C.value, 'hi there')
131 self.assertEquals(config.D.C.value, 'hi there')
129
132
130 def test_extra_args(self):
133 def test_extra_args(self):
131 cl = KeyValueConfigLoader()
134 cl = KeyValueConfigLoader()
135 with mute_warn():
132 config = cl.load_config(['--a=5', 'b', '--c=10', 'd'])
136 config = cl.load_config(['--a=5', 'b', '--c=10', 'd'])
133 self.assertEquals(cl.extra_args, ['b', 'd'])
137 self.assertEquals(cl.extra_args, ['b', 'd'])
134 self.assertEquals(config.a, 5)
138 self.assertEquals(config.a, 5)
135 self.assertEquals(config.c, 10)
139 self.assertEquals(config.c, 10)
140 with mute_warn():
136 config = cl.load_config(['--', '--a=5', '--c=10'])
141 config = cl.load_config(['--', '--a=5', '--c=10'])
137 self.assertEquals(cl.extra_args, ['--a=5', '--c=10'])
142 self.assertEquals(cl.extra_args, ['--a=5', '--c=10'])
138
143
139 def test_unicode_args(self):
144 def test_unicode_args(self):
140 cl = KeyValueConfigLoader()
145 cl = KeyValueConfigLoader()
141 argv = [u'--a=épsîlön']
146 argv = [u'--a=épsîlön']
147 with mute_warn():
142 config = cl.load_config(argv)
148 config = cl.load_config(argv)
143 self.assertEquals(config.a, u'épsîlön')
149 self.assertEquals(config.a, u'épsîlön')
144
150
145 def test_unicode_bytes_args(self):
151 def test_unicode_bytes_args(self):
146 uarg = u'--a=é'
152 uarg = u'--a=é'
147 try:
153 try:
148 barg = uarg.encode(sys.stdin.encoding)
154 barg = uarg.encode(sys.stdin.encoding)
149 except (TypeError, UnicodeEncodeError):
155 except (TypeError, UnicodeEncodeError):
150 raise SkipTest("sys.stdin.encoding can't handle 'é'")
156 raise SkipTest("sys.stdin.encoding can't handle 'é'")
151
157
152 cl = KeyValueConfigLoader()
158 cl = KeyValueConfigLoader()
159 with mute_warn():
153 config = cl.load_config([barg])
160 config = cl.load_config([barg])
154 self.assertEquals(config.a, u'é')
161 self.assertEquals(config.a, u'é')
155
162
156
163
157 class TestConfig(TestCase):
164 class TestConfig(TestCase):
158
165
159 def test_setget(self):
166 def test_setget(self):
160 c = Config()
167 c = Config()
161 c.a = 10
168 c.a = 10
162 self.assertEquals(c.a, 10)
169 self.assertEquals(c.a, 10)
163 self.assertEquals(c.has_key('b'), False)
170 self.assertEquals(c.has_key('b'), False)
164
171
165 def test_auto_section(self):
172 def test_auto_section(self):
166 c = Config()
173 c = Config()
167 self.assertEquals(c.has_key('A'), True)
174 self.assertEquals(c.has_key('A'), True)
168 self.assertEquals(c._has_section('A'), False)
175 self.assertEquals(c._has_section('A'), False)
169 A = c.A
176 A = c.A
170 A.foo = 'hi there'
177 A.foo = 'hi there'
171 self.assertEquals(c._has_section('A'), True)
178 self.assertEquals(c._has_section('A'), True)
172 self.assertEquals(c.A.foo, 'hi there')
179 self.assertEquals(c.A.foo, 'hi there')
173 del c.A
180 del c.A
174 self.assertEquals(len(c.A.keys()),0)
181 self.assertEquals(len(c.A.keys()),0)
175
182
176 def test_merge_doesnt_exist(self):
183 def test_merge_doesnt_exist(self):
177 c1 = Config()
184 c1 = Config()
178 c2 = Config()
185 c2 = Config()
179 c2.bar = 10
186 c2.bar = 10
180 c2.Foo.bar = 10
187 c2.Foo.bar = 10
181 c1._merge(c2)
188 c1._merge(c2)
182 self.assertEquals(c1.Foo.bar, 10)
189 self.assertEquals(c1.Foo.bar, 10)
183 self.assertEquals(c1.bar, 10)
190 self.assertEquals(c1.bar, 10)
184 c2.Bar.bar = 10
191 c2.Bar.bar = 10
185 c1._merge(c2)
192 c1._merge(c2)
186 self.assertEquals(c1.Bar.bar, 10)
193 self.assertEquals(c1.Bar.bar, 10)
187
194
188 def test_merge_exists(self):
195 def test_merge_exists(self):
189 c1 = Config()
196 c1 = Config()
190 c2 = Config()
197 c2 = Config()
191 c1.Foo.bar = 10
198 c1.Foo.bar = 10
192 c1.Foo.bam = 30
199 c1.Foo.bam = 30
193 c2.Foo.bar = 20
200 c2.Foo.bar = 20
194 c2.Foo.wow = 40
201 c2.Foo.wow = 40
195 c1._merge(c2)
202 c1._merge(c2)
196 self.assertEquals(c1.Foo.bam, 30)
203 self.assertEquals(c1.Foo.bam, 30)
197 self.assertEquals(c1.Foo.bar, 20)
204 self.assertEquals(c1.Foo.bar, 20)
198 self.assertEquals(c1.Foo.wow, 40)
205 self.assertEquals(c1.Foo.wow, 40)
199 c2.Foo.Bam.bam = 10
206 c2.Foo.Bam.bam = 10
200 c1._merge(c2)
207 c1._merge(c2)
201 self.assertEquals(c1.Foo.Bam.bam, 10)
208 self.assertEquals(c1.Foo.Bam.bam, 10)
202
209
203 def test_deepcopy(self):
210 def test_deepcopy(self):
204 c1 = Config()
211 c1 = Config()
205 c1.Foo.bar = 10
212 c1.Foo.bar = 10
206 c1.Foo.bam = 30
213 c1.Foo.bam = 30
207 c1.a = 'asdf'
214 c1.a = 'asdf'
208 c1.b = range(10)
215 c1.b = range(10)
209 import copy
216 import copy
210 c2 = copy.deepcopy(c1)
217 c2 = copy.deepcopy(c1)
211 self.assertEquals(c1, c2)
218 self.assertEquals(c1, c2)
212 self.assert_(c1 is not c2)
219 self.assert_(c1 is not c2)
213 self.assert_(c1.Foo is not c2.Foo)
220 self.assert_(c1.Foo is not c2.Foo)
214
221
215 def test_builtin(self):
222 def test_builtin(self):
216 c1 = Config()
223 c1 = Config()
217 exec 'foo = True' in c1
224 exec 'foo = True' in c1
218 self.assertEquals(c1.foo, True)
225 self.assertEquals(c1.foo, True)
219 self.assertRaises(ConfigError, setattr, c1, 'ValueError', 10)
226 self.assertRaises(ConfigError, setattr, c1, 'ValueError', 10)
@@ -1,307 +1,307 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 An application for IPython.
3 An application for IPython.
4
4
5 All top-level applications should use the classes in this module for
5 All top-level applications should use the classes in this module for
6 handling configuration and creating componenets.
6 handling configuration and creating componenets.
7
7
8 The job of an :class:`Application` is to create the master configuration
8 The job of an :class:`Application` is to create the master configuration
9 object and then create the configurable objects, passing the config to them.
9 object and then create the configurable objects, passing the config to them.
10
10
11 Authors:
11 Authors:
12
12
13 * Brian Granger
13 * Brian Granger
14 * Fernando Perez
14 * Fernando Perez
15 * Min RK
15 * Min RK
16
16
17 """
17 """
18
18
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20 # Copyright (C) 2008-2011 The IPython Development Team
20 # Copyright (C) 2008-2011 The IPython Development Team
21 #
21 #
22 # Distributed under the terms of the BSD License. The full license is in
22 # Distributed under the terms of the BSD License. The full license is in
23 # the file COPYING, distributed as part of this software.
23 # the file COPYING, distributed as part of this software.
24 #-----------------------------------------------------------------------------
24 #-----------------------------------------------------------------------------
25
25
26 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
27 # Imports
27 # Imports
28 #-----------------------------------------------------------------------------
28 #-----------------------------------------------------------------------------
29
29
30 import glob
30 import glob
31 import logging
31 import logging
32 import os
32 import os
33 import shutil
33 import shutil
34 import sys
34 import sys
35
35
36 from IPython.config.application import Application
36 from IPython.config.application import Application
37 from IPython.config.configurable import Configurable
37 from IPython.config.configurable import Configurable
38 from IPython.config.loader import Config
38 from IPython.config.loader import Config
39 from IPython.core import release, crashhandler
39 from IPython.core import release, crashhandler
40 from IPython.core.profiledir import ProfileDir, ProfileDirError
40 from IPython.core.profiledir import ProfileDir, ProfileDirError
41 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
41 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
42 from IPython.utils.traitlets import List, Unicode, Type, Bool, Dict
42 from IPython.utils.traitlets import List, Unicode, Type, Bool, Dict
43
43
44 #-----------------------------------------------------------------------------
44 #-----------------------------------------------------------------------------
45 # Classes and functions
45 # Classes and functions
46 #-----------------------------------------------------------------------------
46 #-----------------------------------------------------------------------------
47
47
48
48
49 #-----------------------------------------------------------------------------
49 #-----------------------------------------------------------------------------
50 # Base Application Class
50 # Base Application Class
51 #-----------------------------------------------------------------------------
51 #-----------------------------------------------------------------------------
52
52
53 # aliases and flags
53 # aliases and flags
54
54
55 base_aliases = dict(
55 base_aliases = {
56 profile='BaseIPythonApplication.profile',
56 'profile' : 'BaseIPythonApplication.profile',
57 ipython_dir='BaseIPythonApplication.ipython_dir',
57 'ipython-dir' : 'BaseIPythonApplication.ipython_dir',
58 log_level='Application.log_level',
58 'log-level' : 'Application.log_level',
59 )
59 }
60
60
61 base_flags = dict(
61 base_flags = dict(
62 debug = ({'Application' : {'log_level' : logging.DEBUG}},
62 debug = ({'Application' : {'log_level' : logging.DEBUG}},
63 "set log level to logging.DEBUG (maximize logging output)"),
63 "set log level to logging.DEBUG (maximize logging output)"),
64 quiet = ({'Application' : {'log_level' : logging.CRITICAL}},
64 quiet = ({'Application' : {'log_level' : logging.CRITICAL}},
65 "set log level to logging.CRITICAL (minimize logging output)"),
65 "set log level to logging.CRITICAL (minimize logging output)"),
66 init = ({'BaseIPythonApplication' : {
66 init = ({'BaseIPythonApplication' : {
67 'copy_config_files' : True,
67 'copy_config_files' : True,
68 'auto_create' : True}
68 'auto_create' : True}
69 }, "Initialize profile with default config files")
69 }, "Initialize profile with default config files")
70 )
70 )
71
71
72
72
73 class BaseIPythonApplication(Application):
73 class BaseIPythonApplication(Application):
74
74
75 name = Unicode(u'ipython')
75 name = Unicode(u'ipython')
76 description = Unicode(u'IPython: an enhanced interactive Python shell.')
76 description = Unicode(u'IPython: an enhanced interactive Python shell.')
77 version = Unicode(release.version)
77 version = Unicode(release.version)
78
78
79 aliases = Dict(base_aliases)
79 aliases = Dict(base_aliases)
80 flags = Dict(base_flags)
80 flags = Dict(base_flags)
81 classes = List([ProfileDir])
81 classes = List([ProfileDir])
82
82
83 # Track whether the config_file has changed,
83 # Track whether the config_file has changed,
84 # because some logic happens only if we aren't using the default.
84 # because some logic happens only if we aren't using the default.
85 config_file_specified = Bool(False)
85 config_file_specified = Bool(False)
86
86
87 config_file_name = Unicode(u'ipython_config.py')
87 config_file_name = Unicode(u'ipython_config.py')
88 def _config_file_name_default(self):
88 def _config_file_name_default(self):
89 return self.name.replace('-','_') + u'_config.py'
89 return self.name.replace('-','_') + u'_config.py'
90 def _config_file_name_changed(self, name, old, new):
90 def _config_file_name_changed(self, name, old, new):
91 if new != old:
91 if new != old:
92 self.config_file_specified = True
92 self.config_file_specified = True
93
93
94 # The directory that contains IPython's builtin profiles.
94 # The directory that contains IPython's builtin profiles.
95 builtin_profile_dir = Unicode(
95 builtin_profile_dir = Unicode(
96 os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
96 os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
97 )
97 )
98
98
99 config_file_paths = List(Unicode)
99 config_file_paths = List(Unicode)
100 def _config_file_paths_default(self):
100 def _config_file_paths_default(self):
101 return [os.getcwdu()]
101 return [os.getcwdu()]
102
102
103 profile = Unicode(u'default', config=True,
103 profile = Unicode(u'default', config=True,
104 help="""The IPython profile to use."""
104 help="""The IPython profile to use."""
105 )
105 )
106 def _profile_changed(self, name, old, new):
106 def _profile_changed(self, name, old, new):
107 self.builtin_profile_dir = os.path.join(
107 self.builtin_profile_dir = os.path.join(
108 get_ipython_package_dir(), u'config', u'profile', new
108 get_ipython_package_dir(), u'config', u'profile', new
109 )
109 )
110
110
111 ipython_dir = Unicode(get_ipython_dir(), config=True,
111 ipython_dir = Unicode(get_ipython_dir(), config=True,
112 help="""
112 help="""
113 The name of the IPython directory. This directory is used for logging
113 The name of the IPython directory. This directory is used for logging
114 configuration (through profiles), history storage, etc. The default
114 configuration (through profiles), history storage, etc. The default
115 is usually $HOME/.ipython. This options can also be specified through
115 is usually $HOME/.ipython. This options can also be specified through
116 the environment variable IPYTHON_DIR.
116 the environment variable IPYTHON_DIR.
117 """
117 """
118 )
118 )
119
119
120 overwrite = Bool(False, config=True,
120 overwrite = Bool(False, config=True,
121 help="""Whether to overwrite existing config files when copying""")
121 help="""Whether to overwrite existing config files when copying""")
122 auto_create = Bool(False, config=True,
122 auto_create = Bool(False, config=True,
123 help="""Whether to create profile dir if it doesn't exist""")
123 help="""Whether to create profile dir if it doesn't exist""")
124
124
125 config_files = List(Unicode)
125 config_files = List(Unicode)
126 def _config_files_default(self):
126 def _config_files_default(self):
127 return [u'ipython_config.py']
127 return [u'ipython_config.py']
128
128
129 copy_config_files = Bool(False, config=True,
129 copy_config_files = Bool(False, config=True,
130 help="""Whether to install the default config files into the profile dir.
130 help="""Whether to install the default config files into the profile dir.
131 If a new profile is being created, and IPython contains config files for that
131 If a new profile is being created, and IPython contains config files for that
132 profile, then they will be staged into the new directory. Otherwise,
132 profile, then they will be staged into the new directory. Otherwise,
133 default config files will be automatically generated.
133 default config files will be automatically generated.
134 """)
134 """)
135
135
136 # The class to use as the crash handler.
136 # The class to use as the crash handler.
137 crash_handler_class = Type(crashhandler.CrashHandler)
137 crash_handler_class = Type(crashhandler.CrashHandler)
138
138
139 def __init__(self, **kwargs):
139 def __init__(self, **kwargs):
140 super(BaseIPythonApplication, self).__init__(**kwargs)
140 super(BaseIPythonApplication, self).__init__(**kwargs)
141 # ensure even default IPYTHON_DIR exists
141 # ensure even default IPYTHON_DIR exists
142 if not os.path.exists(self.ipython_dir):
142 if not os.path.exists(self.ipython_dir):
143 self._ipython_dir_changed('ipython_dir', self.ipython_dir, self.ipython_dir)
143 self._ipython_dir_changed('ipython_dir', self.ipython_dir, self.ipython_dir)
144
144
145 #-------------------------------------------------------------------------
145 #-------------------------------------------------------------------------
146 # Various stages of Application creation
146 # Various stages of Application creation
147 #-------------------------------------------------------------------------
147 #-------------------------------------------------------------------------
148
148
149 def init_crash_handler(self):
149 def init_crash_handler(self):
150 """Create a crash handler, typically setting sys.excepthook to it."""
150 """Create a crash handler, typically setting sys.excepthook to it."""
151 self.crash_handler = self.crash_handler_class(self)
151 self.crash_handler = self.crash_handler_class(self)
152 sys.excepthook = self.crash_handler
152 sys.excepthook = self.crash_handler
153
153
154 def _ipython_dir_changed(self, name, old, new):
154 def _ipython_dir_changed(self, name, old, new):
155 if old in sys.path:
155 if old in sys.path:
156 sys.path.remove(old)
156 sys.path.remove(old)
157 sys.path.append(os.path.abspath(new))
157 sys.path.append(os.path.abspath(new))
158 if not os.path.isdir(new):
158 if not os.path.isdir(new):
159 os.makedirs(new, mode=0777)
159 os.makedirs(new, mode=0777)
160 readme = os.path.join(new, 'README')
160 readme = os.path.join(new, 'README')
161 if not os.path.exists(readme):
161 if not os.path.exists(readme):
162 path = os.path.join(get_ipython_package_dir(), u'config', u'profile')
162 path = os.path.join(get_ipython_package_dir(), u'config', u'profile')
163 shutil.copy(os.path.join(path, 'README'), readme)
163 shutil.copy(os.path.join(path, 'README'), readme)
164 self.log.debug("IPYTHON_DIR set to: %s" % new)
164 self.log.debug("IPYTHON_DIR set to: %s" % new)
165
165
166 def load_config_file(self, suppress_errors=True):
166 def load_config_file(self, suppress_errors=True):
167 """Load the config file.
167 """Load the config file.
168
168
169 By default, errors in loading config are handled, and a warning
169 By default, errors in loading config are handled, and a warning
170 printed on screen. For testing, the suppress_errors option is set
170 printed on screen. For testing, the suppress_errors option is set
171 to False, so errors will make tests fail.
171 to False, so errors will make tests fail.
172 """
172 """
173 base_config = 'ipython_config.py'
173 base_config = 'ipython_config.py'
174 self.log.debug("Attempting to load config file: %s" %
174 self.log.debug("Attempting to load config file: %s" %
175 base_config)
175 base_config)
176 try:
176 try:
177 Application.load_config_file(
177 Application.load_config_file(
178 self,
178 self,
179 base_config,
179 base_config,
180 path=self.config_file_paths
180 path=self.config_file_paths
181 )
181 )
182 except IOError:
182 except IOError:
183 # ignore errors loading parent
183 # ignore errors loading parent
184 pass
184 pass
185 if self.config_file_name == base_config:
185 if self.config_file_name == base_config:
186 # don't load secondary config
186 # don't load secondary config
187 return
187 return
188 self.log.debug("Attempting to load config file: %s" %
188 self.log.debug("Attempting to load config file: %s" %
189 self.config_file_name)
189 self.config_file_name)
190 try:
190 try:
191 Application.load_config_file(
191 Application.load_config_file(
192 self,
192 self,
193 self.config_file_name,
193 self.config_file_name,
194 path=self.config_file_paths
194 path=self.config_file_paths
195 )
195 )
196 except IOError:
196 except IOError:
197 # Only warn if the default config file was NOT being used.
197 # Only warn if the default config file was NOT being used.
198 if self.config_file_specified:
198 if self.config_file_specified:
199 self.log.warn("Config file not found, skipping: %s" %
199 self.log.warn("Config file not found, skipping: %s" %
200 self.config_file_name)
200 self.config_file_name)
201 except:
201 except:
202 # For testing purposes.
202 # For testing purposes.
203 if not suppress_errors:
203 if not suppress_errors:
204 raise
204 raise
205 self.log.warn("Error loading config file: %s" %
205 self.log.warn("Error loading config file: %s" %
206 self.config_file_name, exc_info=True)
206 self.config_file_name, exc_info=True)
207
207
208 def init_profile_dir(self):
208 def init_profile_dir(self):
209 """initialize the profile dir"""
209 """initialize the profile dir"""
210 try:
210 try:
211 # location explicitly specified:
211 # location explicitly specified:
212 location = self.config.ProfileDir.location
212 location = self.config.ProfileDir.location
213 except AttributeError:
213 except AttributeError:
214 # location not specified, find by profile name
214 # location not specified, find by profile name
215 try:
215 try:
216 p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
216 p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
217 except ProfileDirError:
217 except ProfileDirError:
218 # not found, maybe create it (always create default profile)
218 # not found, maybe create it (always create default profile)
219 if self.auto_create or self.profile=='default':
219 if self.auto_create or self.profile=='default':
220 try:
220 try:
221 p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
221 p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
222 except ProfileDirError:
222 except ProfileDirError:
223 self.log.fatal("Could not create profile: %r"%self.profile)
223 self.log.fatal("Could not create profile: %r"%self.profile)
224 self.exit(1)
224 self.exit(1)
225 else:
225 else:
226 self.log.info("Created profile dir: %r"%p.location)
226 self.log.info("Created profile dir: %r"%p.location)
227 else:
227 else:
228 self.log.fatal("Profile %r not found."%self.profile)
228 self.log.fatal("Profile %r not found."%self.profile)
229 self.exit(1)
229 self.exit(1)
230 else:
230 else:
231 self.log.info("Using existing profile dir: %r"%p.location)
231 self.log.info("Using existing profile dir: %r"%p.location)
232 else:
232 else:
233 # location is fully specified
233 # location is fully specified
234 try:
234 try:
235 p = ProfileDir.find_profile_dir(location, self.config)
235 p = ProfileDir.find_profile_dir(location, self.config)
236 except ProfileDirError:
236 except ProfileDirError:
237 # not found, maybe create it
237 # not found, maybe create it
238 if self.auto_create:
238 if self.auto_create:
239 try:
239 try:
240 p = ProfileDir.create_profile_dir(location, self.config)
240 p = ProfileDir.create_profile_dir(location, self.config)
241 except ProfileDirError:
241 except ProfileDirError:
242 self.log.fatal("Could not create profile directory: %r"%location)
242 self.log.fatal("Could not create profile directory: %r"%location)
243 self.exit(1)
243 self.exit(1)
244 else:
244 else:
245 self.log.info("Creating new profile dir: %r"%location)
245 self.log.info("Creating new profile dir: %r"%location)
246 else:
246 else:
247 self.log.fatal("Profile directory %r not found."%location)
247 self.log.fatal("Profile directory %r not found."%location)
248 self.exit(1)
248 self.exit(1)
249 else:
249 else:
250 self.log.info("Using existing profile dir: %r"%location)
250 self.log.info("Using existing profile dir: %r"%location)
251
251
252 self.profile_dir = p
252 self.profile_dir = p
253 self.config_file_paths.append(p.location)
253 self.config_file_paths.append(p.location)
254
254
255 def init_config_files(self):
255 def init_config_files(self):
256 """[optionally] copy default config files into profile dir."""
256 """[optionally] copy default config files into profile dir."""
257 # copy config files
257 # copy config files
258 path = self.builtin_profile_dir
258 path = self.builtin_profile_dir
259 if self.copy_config_files:
259 if self.copy_config_files:
260 src = self.profile
260 src = self.profile
261
261
262 cfg = self.config_file_name
262 cfg = self.config_file_name
263 if path and os.path.exists(os.path.join(path, cfg)):
263 if path and os.path.exists(os.path.join(path, cfg)):
264 self.log.warn("Staging %r from %s into %r [overwrite=%s]"%(
264 self.log.warn("Staging %r from %s into %r [overwrite=%s]"%(
265 cfg, src, self.profile_dir.location, self.overwrite)
265 cfg, src, self.profile_dir.location, self.overwrite)
266 )
266 )
267 self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite)
267 self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite)
268 else:
268 else:
269 self.stage_default_config_file()
269 self.stage_default_config_file()
270 else:
270 else:
271 # Still stage *bundled* config files, but not generated ones
271 # Still stage *bundled* config files, but not generated ones
272 # This is necessary for `ipython profile=sympy` to load the profile
272 # This is necessary for `ipython profile=sympy` to load the profile
273 # on the first go
273 # on the first go
274 files = glob.glob(os.path.join(path, '*.py'))
274 files = glob.glob(os.path.join(path, '*.py'))
275 for fullpath in files:
275 for fullpath in files:
276 cfg = os.path.basename(fullpath)
276 cfg = os.path.basename(fullpath)
277 if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False):
277 if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False):
278 # file was copied
278 # file was copied
279 self.log.warn("Staging bundled %s from %s into %r"%(
279 self.log.warn("Staging bundled %s from %s into %r"%(
280 cfg, self.profile, self.profile_dir.location)
280 cfg, self.profile, self.profile_dir.location)
281 )
281 )
282
282
283
283
284 def stage_default_config_file(self):
284 def stage_default_config_file(self):
285 """auto generate default config file, and stage it into the profile."""
285 """auto generate default config file, and stage it into the profile."""
286 s = self.generate_config_file()
286 s = self.generate_config_file()
287 fname = os.path.join(self.profile_dir.location, self.config_file_name)
287 fname = os.path.join(self.profile_dir.location, self.config_file_name)
288 if self.overwrite or not os.path.exists(fname):
288 if self.overwrite or not os.path.exists(fname):
289 self.log.warn("Generating default config file: %r"%(fname))
289 self.log.warn("Generating default config file: %r"%(fname))
290 with open(fname, 'w') as f:
290 with open(fname, 'w') as f:
291 f.write(s)
291 f.write(s)
292
292
293
293
294 def initialize(self, argv=None):
294 def initialize(self, argv=None):
295 # don't hook up crash handler before parsing command-line
295 # don't hook up crash handler before parsing command-line
296 self.parse_command_line(argv)
296 self.parse_command_line(argv)
297 self.init_crash_handler()
297 self.init_crash_handler()
298 if self.subapp is not None:
298 if self.subapp is not None:
299 # stop here if subapp is taking over
299 # stop here if subapp is taking over
300 return
300 return
301 cl_config = self.config
301 cl_config = self.config
302 self.init_profile_dir()
302 self.init_profile_dir()
303 self.init_config_files()
303 self.init_config_files()
304 self.load_config_file()
304 self.load_config_file()
305 # enforce cl-opts override configfile opts:
305 # enforce cl-opts override configfile opts:
306 self.update_config(cl_config)
306 self.update_config(cl_config)
307
307
@@ -1,220 +1,220 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 An application for managing IPython profiles.
3 An application for managing IPython profiles.
4
4
5 To be invoked as the `ipython profile` subcommand.
5 To be invoked as the `ipython profile` subcommand.
6
6
7 Authors:
7 Authors:
8
8
9 * Min RK
9 * Min RK
10
10
11 """
11 """
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Copyright (C) 2008-2011 The IPython Development Team
14 # Copyright (C) 2008-2011 The IPython Development Team
15 #
15 #
16 # Distributed under the terms of the BSD License. The full license is in
16 # Distributed under the terms of the BSD License. The full license is in
17 # the file COPYING, distributed as part of this software.
17 # the file COPYING, distributed as part of this software.
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Imports
21 # Imports
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 import logging
24 import logging
25 import os
25 import os
26
26
27 from IPython.config.application import Application, boolean_flag
27 from IPython.config.application import Application, boolean_flag
28 from IPython.core.application import (
28 from IPython.core.application import (
29 BaseIPythonApplication, base_flags, base_aliases
29 BaseIPythonApplication, base_flags, base_aliases
30 )
30 )
31 from IPython.core.profiledir import ProfileDir
31 from IPython.core.profiledir import ProfileDir
32 from IPython.utils.path import get_ipython_dir
32 from IPython.utils.path import get_ipython_dir
33 from IPython.utils.traitlets import Unicode, Bool, Dict
33 from IPython.utils.traitlets import Unicode, Bool, Dict
34
34
35 #-----------------------------------------------------------------------------
35 #-----------------------------------------------------------------------------
36 # Constants
36 # Constants
37 #-----------------------------------------------------------------------------
37 #-----------------------------------------------------------------------------
38
38
39 create_help = """Create an IPython profile by name
39 create_help = """Create an IPython profile by name
40
40
41 Create an ipython profile directory by its name or
41 Create an ipython profile directory by its name or
42 profile directory path. Profile directories contain
42 profile directory path. Profile directories contain
43 configuration, log and security related files and are named
43 configuration, log and security related files and are named
44 using the convention 'profile_<name>'. By default they are
44 using the convention 'profile_<name>'. By default they are
45 located in your ipython directory. Once created, you will
45 located in your ipython directory. Once created, you will
46 can edit the configuration files in the profile
46 can edit the configuration files in the profile
47 directory to configure IPython. Most users will create a
47 directory to configure IPython. Most users will create a
48 profile directory by name,
48 profile directory by name,
49 `ipython profile create myprofile`, which will put the directory
49 `ipython profile create myprofile`, which will put the directory
50 in `<ipython_dir>/profile_myprofile`.
50 in `<ipython_dir>/profile_myprofile`.
51 """
51 """
52 list_help = """List available IPython profiles
52 list_help = """List available IPython profiles
53
53
54 List all available profiles, by profile location, that can
54 List all available profiles, by profile location, that can
55 be found in the current working directly or in the ipython
55 be found in the current working directly or in the ipython
56 directory. Profile directories are named using the convention
56 directory. Profile directories are named using the convention
57 'profile_<profile>'.
57 'profile_<profile>'.
58 """
58 """
59 profile_help = """Manage IPython profiles
59 profile_help = """Manage IPython profiles
60
60
61 Profile directories contain
61 Profile directories contain
62 configuration, log and security related files and are named
62 configuration, log and security related files and are named
63 using the convention 'profile_<name>'. By default they are
63 using the convention 'profile_<name>'. By default they are
64 located in your ipython directory. You can create profiles
64 located in your ipython directory. You can create profiles
65 with `ipython profile create <name>`, or see the profiles you
65 with `ipython profile create <name>`, or see the profiles you
66 already have with `ipython profile list`
66 already have with `ipython profile list`
67
67
68 To get started configuring IPython, simply do:
68 To get started configuring IPython, simply do:
69
69
70 $> ipython profile create
70 $> ipython profile create
71
71
72 and IPython will create the default profile in <ipython_dir>/profile_default,
72 and IPython will create the default profile in <ipython_dir>/profile_default,
73 where you can edit ipython_config.py to start configuring IPython.
73 where you can edit ipython_config.py to start configuring IPython.
74
74
75 """
75 """
76
76
77 #-----------------------------------------------------------------------------
77 #-----------------------------------------------------------------------------
78 # Profile Application Class (for `ipython profile` subcommand)
78 # Profile Application Class (for `ipython profile` subcommand)
79 #-----------------------------------------------------------------------------
79 #-----------------------------------------------------------------------------
80
80
81
81
82
82
83 class ProfileList(Application):
83 class ProfileList(Application):
84 name = u'ipython-profile'
84 name = u'ipython-profile'
85 description = list_help
85 description = list_help
86
86
87 aliases = Dict(dict(
87 aliases = Dict({
88 ipython_dir = 'ProfileList.ipython_dir',
88 'ipython-dir' : 'ProfileList.ipython_dir',
89 log_level = 'Application.log_level',
89 'log-level' : 'Application.log_level',
90 ))
90 })
91 flags = Dict(dict(
91 flags = Dict(dict(
92 debug = ({'Application' : {'log_level' : 0}},
92 debug = ({'Application' : {'log_level' : 0}},
93 "Set log_level to 0, maximizing log output."
93 "Set Application.log_level to 0, maximizing log output."
94 )
94 )
95 ))
95 ))
96 ipython_dir = Unicode(get_ipython_dir(), config=True,
96 ipython_dir = Unicode(get_ipython_dir(), config=True,
97 help="""
97 help="""
98 The name of the IPython directory. This directory is used for logging
98 The name of the IPython directory. This directory is used for logging
99 configuration (through profiles), history storage, etc. The default
99 configuration (through profiles), history storage, etc. The default
100 is usually $HOME/.ipython. This options can also be specified through
100 is usually $HOME/.ipython. This options can also be specified through
101 the environment variable IPYTHON_DIR.
101 the environment variable IPYTHON_DIR.
102 """
102 """
103 )
103 )
104
104
105 def list_profile_dirs(self):
105 def list_profile_dirs(self):
106 # Find the search paths
106 # Find the search paths
107 paths = [os.getcwdu(), self.ipython_dir]
107 paths = [os.getcwdu(), self.ipython_dir]
108
108
109 self.log.warn('Searching for IPython profiles in paths: %r' % paths)
109 self.log.warn('Searching for IPython profiles in paths: %r' % paths)
110 for path in paths:
110 for path in paths:
111 files = os.listdir(path)
111 files = os.listdir(path)
112 for f in files:
112 for f in files:
113 full_path = os.path.join(path, f)
113 full_path = os.path.join(path, f)
114 if os.path.isdir(full_path) and f.startswith('profile_'):
114 if os.path.isdir(full_path) and f.startswith('profile_'):
115 profile = f.split('_',1)[-1]
115 profile = f.split('_',1)[-1]
116 start_cmd = 'ipython profile=%s' % profile
116 start_cmd = 'ipython profile=%s' % profile
117 print start_cmd + " ==> " + full_path
117 print start_cmd + " ==> " + full_path
118
118
119 def start(self):
119 def start(self):
120 self.list_profile_dirs()
120 self.list_profile_dirs()
121
121
122
122
123 create_flags = {}
123 create_flags = {}
124 create_flags.update(base_flags)
124 create_flags.update(base_flags)
125 create_flags.update(boolean_flag('reset', 'ProfileCreate.overwrite',
125 create_flags.update(boolean_flag('reset', 'ProfileCreate.overwrite',
126 "reset config files to defaults", "leave existing config files"))
126 "reset config files to defaults", "leave existing config files"))
127 create_flags.update(boolean_flag('parallel', 'ProfileCreate.parallel',
127 create_flags.update(boolean_flag('parallel', 'ProfileCreate.parallel',
128 "Include parallel computing config files",
128 "Include parallel computing config files",
129 "Don't include parallel computing config files"))
129 "Don't include parallel computing config files"))
130
130
131 class ProfileCreate(BaseIPythonApplication):
131 class ProfileCreate(BaseIPythonApplication):
132 name = u'ipython-profile'
132 name = u'ipython-profile'
133 description = create_help
133 description = create_help
134 auto_create = Bool(True, config=False)
134 auto_create = Bool(True, config=False)
135
135
136 def _copy_config_files_default(self):
136 def _copy_config_files_default(self):
137 return True
137 return True
138
138
139 parallel = Bool(False, config=True,
139 parallel = Bool(False, config=True,
140 help="whether to include parallel computing config files")
140 help="whether to include parallel computing config files")
141 def _parallel_changed(self, name, old, new):
141 def _parallel_changed(self, name, old, new):
142 parallel_files = [ 'ipcontroller_config.py',
142 parallel_files = [ 'ipcontroller_config.py',
143 'ipengine_config.py',
143 'ipengine_config.py',
144 'ipcluster_config.py'
144 'ipcluster_config.py'
145 ]
145 ]
146 if new:
146 if new:
147 for cf in parallel_files:
147 for cf in parallel_files:
148 self.config_files.append(cf)
148 self.config_files.append(cf)
149 else:
149 else:
150 for cf in parallel_files:
150 for cf in parallel_files:
151 if cf in self.config_files:
151 if cf in self.config_files:
152 self.config_files.remove(cf)
152 self.config_files.remove(cf)
153
153
154 def parse_command_line(self, argv):
154 def parse_command_line(self, argv):
155 super(ProfileCreate, self).parse_command_line(argv)
155 super(ProfileCreate, self).parse_command_line(argv)
156 # accept positional arg as profile name
156 # accept positional arg as profile name
157 if self.extra_args:
157 if self.extra_args:
158 self.profile = self.extra_args[0]
158 self.profile = self.extra_args[0]
159
159
160 flags = Dict(create_flags)
160 flags = Dict(create_flags)
161
161
162 classes = [ProfileDir]
162 classes = [ProfileDir]
163
163
164 def init_config_files(self):
164 def init_config_files(self):
165 super(ProfileCreate, self).init_config_files()
165 super(ProfileCreate, self).init_config_files()
166 # use local imports, since these classes may import from here
166 # use local imports, since these classes may import from here
167 from IPython.frontend.terminal.ipapp import TerminalIPythonApp
167 from IPython.frontend.terminal.ipapp import TerminalIPythonApp
168 apps = [TerminalIPythonApp]
168 apps = [TerminalIPythonApp]
169 try:
169 try:
170 from IPython.frontend.qt.console.qtconsoleapp import IPythonQtConsoleApp
170 from IPython.frontend.qt.console.qtconsoleapp import IPythonQtConsoleApp
171 except Exception:
171 except Exception:
172 # this should be ImportError, but under weird circumstances
172 # this should be ImportError, but under weird circumstances
173 # this might be an AttributeError, or possibly others
173 # this might be an AttributeError, or possibly others
174 # in any case, nothing should cause the profile creation to crash.
174 # in any case, nothing should cause the profile creation to crash.
175 pass
175 pass
176 else:
176 else:
177 apps.append(IPythonQtConsoleApp)
177 apps.append(IPythonQtConsoleApp)
178 if self.parallel:
178 if self.parallel:
179 from IPython.parallel.apps.ipcontrollerapp import IPControllerApp
179 from IPython.parallel.apps.ipcontrollerapp import IPControllerApp
180 from IPython.parallel.apps.ipengineapp import IPEngineApp
180 from IPython.parallel.apps.ipengineapp import IPEngineApp
181 from IPython.parallel.apps.ipclusterapp import IPClusterStart
181 from IPython.parallel.apps.ipclusterapp import IPClusterStart
182 from IPython.parallel.apps.iploggerapp import IPLoggerApp
182 from IPython.parallel.apps.iploggerapp import IPLoggerApp
183 apps.extend([
183 apps.extend([
184 IPControllerApp,
184 IPControllerApp,
185 IPEngineApp,
185 IPEngineApp,
186 IPClusterStart,
186 IPClusterStart,
187 IPLoggerApp,
187 IPLoggerApp,
188 ])
188 ])
189 for App in apps:
189 for App in apps:
190 app = App()
190 app = App()
191 app.config.update(self.config)
191 app.config.update(self.config)
192 app.log = self.log
192 app.log = self.log
193 app.overwrite = self.overwrite
193 app.overwrite = self.overwrite
194 app.copy_config_files=True
194 app.copy_config_files=True
195 app.profile = self.profile
195 app.profile = self.profile
196 app.init_profile_dir()
196 app.init_profile_dir()
197 app.init_config_files()
197 app.init_config_files()
198
198
199 def stage_default_config_file(self):
199 def stage_default_config_file(self):
200 pass
200 pass
201
201
202 class ProfileApp(Application):
202 class ProfileApp(Application):
203 name = u'ipython-profile'
203 name = u'ipython-profile'
204 description = profile_help
204 description = profile_help
205
205
206 subcommands = Dict(dict(
206 subcommands = Dict(dict(
207 create = (ProfileCreate, "Create a new profile dir with default config files"),
207 create = (ProfileCreate, "Create a new profile dir with default config files"),
208 list = (ProfileList, "List existing profiles")
208 list = (ProfileList, "List existing profiles")
209 ))
209 ))
210
210
211 def start(self):
211 def start(self):
212 if self.subapp is None:
212 if self.subapp is None:
213 print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
213 print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
214 print
214 print
215 self.print_description()
215 self.print_description()
216 self.print_subcommands()
216 self.print_subcommands()
217 self.exit(1)
217 self.exit(1)
218 else:
218 else:
219 return self.subapp.start()
219 return self.subapp.start()
220
220
@@ -1,253 +1,253 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 A mixin for :class:`~IPython.core.application.Application` classes that
4 A mixin for :class:`~IPython.core.application.Application` classes that
5 launch InteractiveShell instances, load extensions, etc.
5 launch InteractiveShell instances, load extensions, etc.
6
6
7 Authors
7 Authors
8 -------
8 -------
9
9
10 * Min Ragan-Kelley
10 * Min Ragan-Kelley
11 """
11 """
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Copyright (C) 2008-2011 The IPython Development Team
14 # Copyright (C) 2008-2011 The IPython Development Team
15 #
15 #
16 # Distributed under the terms of the BSD License. The full license is in
16 # Distributed under the terms of the BSD License. The full license is in
17 # the file COPYING, distributed as part of this software.
17 # the file COPYING, distributed as part of this software.
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Imports
21 # Imports
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 from __future__ import absolute_import
24 from __future__ import absolute_import
25
25
26 import os
26 import os
27 import sys
27 import sys
28
28
29 from IPython.config.application import boolean_flag
29 from IPython.config.application import boolean_flag
30 from IPython.config.configurable import Configurable
30 from IPython.config.configurable import Configurable
31 from IPython.config.loader import Config
31 from IPython.config.loader import Config
32 from IPython.utils.path import filefind
32 from IPython.utils.path import filefind
33 from IPython.utils.traitlets import Unicode, Instance, List
33 from IPython.utils.traitlets import Unicode, Instance, List
34
34
35 #-----------------------------------------------------------------------------
35 #-----------------------------------------------------------------------------
36 # Aliases and Flags
36 # Aliases and Flags
37 #-----------------------------------------------------------------------------
37 #-----------------------------------------------------------------------------
38
38
39 shell_flags = {}
39 shell_flags = {}
40
40
41 addflag = lambda *args: shell_flags.update(boolean_flag(*args))
41 addflag = lambda *args: shell_flags.update(boolean_flag(*args))
42 addflag('autoindent', 'InteractiveShell.autoindent',
42 addflag('autoindent', 'InteractiveShell.autoindent',
43 'Turn on autoindenting.', 'Turn off autoindenting.'
43 'Turn on autoindenting.', 'Turn off autoindenting.'
44 )
44 )
45 addflag('automagic', 'InteractiveShell.automagic',
45 addflag('automagic', 'InteractiveShell.automagic',
46 """Turn on the auto calling of magic commands. Type %%magic at the
46 """Turn on the auto calling of magic commands. Type %%magic at the
47 IPython prompt for more information.""",
47 IPython prompt for more information.""",
48 'Turn off the auto calling of magic commands.'
48 'Turn off the auto calling of magic commands.'
49 )
49 )
50 addflag('pdb', 'InteractiveShell.pdb',
50 addflag('pdb', 'InteractiveShell.pdb',
51 "Enable auto calling the pdb debugger after every exception.",
51 "Enable auto calling the pdb debugger after every exception.",
52 "Disable auto calling the pdb debugger after every exception."
52 "Disable auto calling the pdb debugger after every exception."
53 )
53 )
54 addflag('pprint', 'PlainTextFormatter.pprint',
54 addflag('pprint', 'PlainTextFormatter.pprint',
55 "Enable auto pretty printing of results.",
55 "Enable auto pretty printing of results.",
56 "Disable auto auto pretty printing of results."
56 "Disable auto auto pretty printing of results."
57 )
57 )
58 addflag('color-info', 'InteractiveShell.color_info',
58 addflag('color-info', 'InteractiveShell.color_info',
59 """IPython can display information about objects via a set of func-
59 """IPython can display information about objects via a set of func-
60 tions, and optionally can use colors for this, syntax highlighting
60 tions, and optionally can use colors for this, syntax highlighting
61 source code and various other elements. However, because this
61 source code and various other elements. However, because this
62 information is passed through a pager (like 'less') and many pagers get
62 information is passed through a pager (like 'less') and many pagers get
63 confused with color codes, this option is off by default. You can test
63 confused with color codes, this option is off by default. You can test
64 it and turn it on permanently in your ipython_config.py file if it
64 it and turn it on permanently in your ipython_config.py file if it
65 works for you. Test it and turn it on permanently if it works with
65 works for you. Test it and turn it on permanently if it works with
66 your system. The magic function %%color_info allows you to toggle this
66 your system. The magic function %%color_info allows you to toggle this
67 interactively for testing.""",
67 interactively for testing.""",
68 "Disable using colors for info related things."
68 "Disable using colors for info related things."
69 )
69 )
70 addflag('deep-reload', 'InteractiveShell.deep_reload',
70 addflag('deep-reload', 'InteractiveShell.deep_reload',
71 """Enable deep (recursive) reloading by default. IPython can use the
71 """Enable deep (recursive) reloading by default. IPython can use the
72 deep_reload module which reloads changes in modules recursively (it
72 deep_reload module which reloads changes in modules recursively (it
73 replaces the reload() function, so you don't need to change anything to
73 replaces the reload() function, so you don't need to change anything to
74 use it). deep_reload() forces a full reload of modules whose code may
74 use it). deep_reload() forces a full reload of modules whose code may
75 have changed, which the default reload() function does not. When
75 have changed, which the default reload() function does not. When
76 deep_reload is off, IPython will use the normal reload(), but
76 deep_reload is off, IPython will use the normal reload(), but
77 deep_reload will still be available as dreload(). This feature is off
77 deep_reload will still be available as dreload(). This feature is off
78 by default [which means that you have both normal reload() and
78 by default [which means that you have both normal reload() and
79 dreload()].""",
79 dreload()].""",
80 "Disable deep (recursive) reloading by default."
80 "Disable deep (recursive) reloading by default."
81 )
81 )
82 nosep_config = Config()
82 nosep_config = Config()
83 nosep_config.InteractiveShell.separate_in = ''
83 nosep_config.InteractiveShell.separate_in = ''
84 nosep_config.InteractiveShell.separate_out = ''
84 nosep_config.InteractiveShell.separate_out = ''
85 nosep_config.InteractiveShell.separate_out2 = ''
85 nosep_config.InteractiveShell.separate_out2 = ''
86
86
87 shell_flags['nosep']=(nosep_config, "Eliminate all spacing between prompts.")
87 shell_flags['nosep']=(nosep_config, "Eliminate all spacing between prompts.")
88
88
89
89
90 # it's possible we don't want short aliases for *all* of these:
90 # it's possible we don't want short aliases for *all* of these:
91 shell_aliases = dict(
91 shell_aliases = dict(
92 autocall='InteractiveShell.autocall',
92 autocall='InteractiveShell.autocall',
93 cache_size='InteractiveShell.cache_size',
94 colors='InteractiveShell.colors',
93 colors='InteractiveShell.colors',
95 logfile='InteractiveShell.logfile',
94 logfile='InteractiveShell.logfile',
96 logappend='InteractiveShell.logappend',
95 logappend='InteractiveShell.logappend',
97 c='InteractiveShellApp.code_to_run',
96 c='InteractiveShellApp.code_to_run',
98 ext='InteractiveShellApp.extra_extension',
97 ext='InteractiveShellApp.extra_extension',
99 )
98 )
99 shell_aliases['cache-size'] = 'InteractiveShell.cache_size'
100
100
101 #-----------------------------------------------------------------------------
101 #-----------------------------------------------------------------------------
102 # Main classes and functions
102 # Main classes and functions
103 #-----------------------------------------------------------------------------
103 #-----------------------------------------------------------------------------
104
104
105 class InteractiveShellApp(Configurable):
105 class InteractiveShellApp(Configurable):
106 """A Mixin for applications that start InteractiveShell instances.
106 """A Mixin for applications that start InteractiveShell instances.
107
107
108 Provides configurables for loading extensions and executing files
108 Provides configurables for loading extensions and executing files
109 as part of configuring a Shell environment.
109 as part of configuring a Shell environment.
110
110
111 Provides init_extensions() and init_code() methods, to be called
111 Provides init_extensions() and init_code() methods, to be called
112 after init_shell(), which must be implemented by subclasses.
112 after init_shell(), which must be implemented by subclasses.
113 """
113 """
114 extensions = List(Unicode, config=True,
114 extensions = List(Unicode, config=True,
115 help="A list of dotted module names of IPython extensions to load."
115 help="A list of dotted module names of IPython extensions to load."
116 )
116 )
117 extra_extension = Unicode('', config=True,
117 extra_extension = Unicode('', config=True,
118 help="dotted module name of an IPython extension to load."
118 help="dotted module name of an IPython extension to load."
119 )
119 )
120 def _extra_extension_changed(self, name, old, new):
120 def _extra_extension_changed(self, name, old, new):
121 if new:
121 if new:
122 # add to self.extensions
122 # add to self.extensions
123 self.extensions.append(new)
123 self.extensions.append(new)
124
124
125 exec_files = List(Unicode, config=True,
125 exec_files = List(Unicode, config=True,
126 help="""List of files to run at IPython startup."""
126 help="""List of files to run at IPython startup."""
127 )
127 )
128 file_to_run = Unicode('', config=True,
128 file_to_run = Unicode('', config=True,
129 help="""A file to be run""")
129 help="""A file to be run""")
130
130
131 exec_lines = List(Unicode, config=True,
131 exec_lines = List(Unicode, config=True,
132 help="""lines of code to run at IPython startup."""
132 help="""lines of code to run at IPython startup."""
133 )
133 )
134 code_to_run = Unicode('', config=True,
134 code_to_run = Unicode('', config=True,
135 help="Execute the given command string."
135 help="Execute the given command string."
136 )
136 )
137 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
137 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
138
138
139 def init_shell(self):
139 def init_shell(self):
140 raise NotImplementedError("Override in subclasses")
140 raise NotImplementedError("Override in subclasses")
141
141
142 def init_extensions(self):
142 def init_extensions(self):
143 """Load all IPython extensions in IPythonApp.extensions.
143 """Load all IPython extensions in IPythonApp.extensions.
144
144
145 This uses the :meth:`ExtensionManager.load_extensions` to load all
145 This uses the :meth:`ExtensionManager.load_extensions` to load all
146 the extensions listed in ``self.extensions``.
146 the extensions listed in ``self.extensions``.
147 """
147 """
148 if not self.extensions:
148 if not self.extensions:
149 return
149 return
150 try:
150 try:
151 self.log.debug("Loading IPython extensions...")
151 self.log.debug("Loading IPython extensions...")
152 extensions = self.extensions
152 extensions = self.extensions
153 for ext in extensions:
153 for ext in extensions:
154 try:
154 try:
155 self.log.info("Loading IPython extension: %s" % ext)
155 self.log.info("Loading IPython extension: %s" % ext)
156 self.shell.extension_manager.load_extension(ext)
156 self.shell.extension_manager.load_extension(ext)
157 except:
157 except:
158 self.log.warn("Error in loading extension: %s" % ext)
158 self.log.warn("Error in loading extension: %s" % ext)
159 self.shell.showtraceback()
159 self.shell.showtraceback()
160 except:
160 except:
161 self.log.warn("Unknown error in loading extensions:")
161 self.log.warn("Unknown error in loading extensions:")
162 self.shell.showtraceback()
162 self.shell.showtraceback()
163
163
164 def init_code(self):
164 def init_code(self):
165 """run the pre-flight code, specified via exec_lines"""
165 """run the pre-flight code, specified via exec_lines"""
166 self._run_exec_lines()
166 self._run_exec_lines()
167 self._run_exec_files()
167 self._run_exec_files()
168 self._run_cmd_line_code()
168 self._run_cmd_line_code()
169
169
170 def _run_exec_lines(self):
170 def _run_exec_lines(self):
171 """Run lines of code in IPythonApp.exec_lines in the user's namespace."""
171 """Run lines of code in IPythonApp.exec_lines in the user's namespace."""
172 if not self.exec_lines:
172 if not self.exec_lines:
173 return
173 return
174 try:
174 try:
175 self.log.debug("Running code from IPythonApp.exec_lines...")
175 self.log.debug("Running code from IPythonApp.exec_lines...")
176 for line in self.exec_lines:
176 for line in self.exec_lines:
177 try:
177 try:
178 self.log.info("Running code in user namespace: %s" %
178 self.log.info("Running code in user namespace: %s" %
179 line)
179 line)
180 self.shell.run_cell(line, store_history=False)
180 self.shell.run_cell(line, store_history=False)
181 except:
181 except:
182 self.log.warn("Error in executing line in user "
182 self.log.warn("Error in executing line in user "
183 "namespace: %s" % line)
183 "namespace: %s" % line)
184 self.shell.showtraceback()
184 self.shell.showtraceback()
185 except:
185 except:
186 self.log.warn("Unknown error in handling IPythonApp.exec_lines:")
186 self.log.warn("Unknown error in handling IPythonApp.exec_lines:")
187 self.shell.showtraceback()
187 self.shell.showtraceback()
188
188
189 def _exec_file(self, fname):
189 def _exec_file(self, fname):
190 try:
190 try:
191 full_filename = filefind(fname, [u'.', self.ipython_dir])
191 full_filename = filefind(fname, [u'.', self.ipython_dir])
192 except IOError as e:
192 except IOError as e:
193 self.log.warn("File not found: %r"%fname)
193 self.log.warn("File not found: %r"%fname)
194 return
194 return
195 # Make sure that the running script gets a proper sys.argv as if it
195 # Make sure that the running script gets a proper sys.argv as if it
196 # were run from a system shell.
196 # were run from a system shell.
197 save_argv = sys.argv
197 save_argv = sys.argv
198 sys.argv = [full_filename] + self.extra_args[1:]
198 sys.argv = [full_filename] + self.extra_args[1:]
199 try:
199 try:
200 if os.path.isfile(full_filename):
200 if os.path.isfile(full_filename):
201 if full_filename.endswith('.ipy'):
201 if full_filename.endswith('.ipy'):
202 self.log.info("Running file in user namespace: %s" %
202 self.log.info("Running file in user namespace: %s" %
203 full_filename)
203 full_filename)
204 self.shell.safe_execfile_ipy(full_filename)
204 self.shell.safe_execfile_ipy(full_filename)
205 else:
205 else:
206 # default to python, even without extension
206 # default to python, even without extension
207 self.log.info("Running file in user namespace: %s" %
207 self.log.info("Running file in user namespace: %s" %
208 full_filename)
208 full_filename)
209 # Ensure that __file__ is always defined to match Python behavior
209 # Ensure that __file__ is always defined to match Python behavior
210 self.shell.user_ns['__file__'] = fname
210 self.shell.user_ns['__file__'] = fname
211 try:
211 try:
212 self.shell.safe_execfile(full_filename, self.shell.user_ns)
212 self.shell.safe_execfile(full_filename, self.shell.user_ns)
213 finally:
213 finally:
214 del self.shell.user_ns['__file__']
214 del self.shell.user_ns['__file__']
215 finally:
215 finally:
216 sys.argv = save_argv
216 sys.argv = save_argv
217
217
218 def _run_exec_files(self):
218 def _run_exec_files(self):
219 """Run files from IPythonApp.exec_files"""
219 """Run files from IPythonApp.exec_files"""
220 if not self.exec_files:
220 if not self.exec_files:
221 return
221 return
222
222
223 self.log.debug("Running files in IPythonApp.exec_files...")
223 self.log.debug("Running files in IPythonApp.exec_files...")
224 try:
224 try:
225 for fname in self.exec_files:
225 for fname in self.exec_files:
226 self._exec_file(fname)
226 self._exec_file(fname)
227 except:
227 except:
228 self.log.warn("Unknown error in handling IPythonApp.exec_files:")
228 self.log.warn("Unknown error in handling IPythonApp.exec_files:")
229 self.shell.showtraceback()
229 self.shell.showtraceback()
230
230
231 def _run_cmd_line_code(self):
231 def _run_cmd_line_code(self):
232 """Run code or file specified at the command-line"""
232 """Run code or file specified at the command-line"""
233 if self.code_to_run:
233 if self.code_to_run:
234 line = self.code_to_run
234 line = self.code_to_run
235 try:
235 try:
236 self.log.info("Running code given at command line (c=): %s" %
236 self.log.info("Running code given at command line (c=): %s" %
237 line)
237 line)
238 self.shell.run_cell(line, store_history=False)
238 self.shell.run_cell(line, store_history=False)
239 except:
239 except:
240 self.log.warn("Error in executing line in user namespace: %s" %
240 self.log.warn("Error in executing line in user namespace: %s" %
241 line)
241 line)
242 self.shell.showtraceback()
242 self.shell.showtraceback()
243
243
244 # Like Python itself, ignore the second if the first of these is present
244 # Like Python itself, ignore the second if the first of these is present
245 elif self.file_to_run:
245 elif self.file_to_run:
246 fname = self.file_to_run
246 fname = self.file_to_run
247 try:
247 try:
248 self._exec_file(fname)
248 self._exec_file(fname)
249 except:
249 except:
250 self.log.warn("Error in executing file in user namespace: %s" %
250 self.log.warn("Error in executing file in user namespace: %s" %
251 fname)
251 fname)
252 self.shell.showtraceback()
252 self.shell.showtraceback()
253
253
@@ -1,266 +1,266 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The Base Application class for IPython.parallel apps
4 The Base Application class for IPython.parallel apps
5
5
6 Authors:
6 Authors:
7
7
8 * Brian Granger
8 * Brian Granger
9 * Min RK
9 * Min RK
10
10
11 """
11 """
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Copyright (C) 2008-2011 The IPython Development Team
14 # Copyright (C) 2008-2011 The IPython Development Team
15 #
15 #
16 # Distributed under the terms of the BSD License. The full license is in
16 # Distributed under the terms of the BSD License. The full license is in
17 # the file COPYING, distributed as part of this software.
17 # the file COPYING, distributed as part of this software.
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Imports
21 # Imports
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 from __future__ import with_statement
24 from __future__ import with_statement
25
25
26 import os
26 import os
27 import logging
27 import logging
28 import re
28 import re
29 import sys
29 import sys
30
30
31 from subprocess import Popen, PIPE
31 from subprocess import Popen, PIPE
32
32
33 from IPython.core import release
33 from IPython.core import release
34 from IPython.core.crashhandler import CrashHandler
34 from IPython.core.crashhandler import CrashHandler
35 from IPython.core.application import (
35 from IPython.core.application import (
36 BaseIPythonApplication,
36 BaseIPythonApplication,
37 base_aliases as base_ip_aliases,
37 base_aliases as base_ip_aliases,
38 base_flags as base_ip_flags
38 base_flags as base_ip_flags
39 )
39 )
40 from IPython.utils.path import expand_path
40 from IPython.utils.path import expand_path
41
41
42 from IPython.utils.traitlets import Unicode, Bool, Instance, Dict, List
42 from IPython.utils.traitlets import Unicode, Bool, Instance, Dict, List
43
43
44 #-----------------------------------------------------------------------------
44 #-----------------------------------------------------------------------------
45 # Module errors
45 # Module errors
46 #-----------------------------------------------------------------------------
46 #-----------------------------------------------------------------------------
47
47
48 class PIDFileError(Exception):
48 class PIDFileError(Exception):
49 pass
49 pass
50
50
51
51
52 #-----------------------------------------------------------------------------
52 #-----------------------------------------------------------------------------
53 # Crash handler for this application
53 # Crash handler for this application
54 #-----------------------------------------------------------------------------
54 #-----------------------------------------------------------------------------
55
55
56
56
57 _message_template = """\
57 _message_template = """\
58 Oops, $self.app_name crashed. We do our best to make it stable, but...
58 Oops, $self.app_name crashed. We do our best to make it stable, but...
59
59
60 A crash report was automatically generated with the following information:
60 A crash report was automatically generated with the following information:
61 - A verbatim copy of the crash traceback.
61 - A verbatim copy of the crash traceback.
62 - Data on your current $self.app_name configuration.
62 - Data on your current $self.app_name configuration.
63
63
64 It was left in the file named:
64 It was left in the file named:
65 \t'$self.crash_report_fname'
65 \t'$self.crash_report_fname'
66 If you can email this file to the developers, the information in it will help
66 If you can email this file to the developers, the information in it will help
67 them in understanding and correcting the problem.
67 them in understanding and correcting the problem.
68
68
69 You can mail it to: $self.contact_name at $self.contact_email
69 You can mail it to: $self.contact_name at $self.contact_email
70 with the subject '$self.app_name Crash Report'.
70 with the subject '$self.app_name Crash Report'.
71
71
72 If you want to do it now, the following command will work (under Unix):
72 If you want to do it now, the following command will work (under Unix):
73 mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
73 mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
74
74
75 To ensure accurate tracking of this issue, please file a report about it at:
75 To ensure accurate tracking of this issue, please file a report about it at:
76 $self.bug_tracker
76 $self.bug_tracker
77 """
77 """
78
78
79 class ParallelCrashHandler(CrashHandler):
79 class ParallelCrashHandler(CrashHandler):
80 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
80 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
81
81
82 message_template = _message_template
82 message_template = _message_template
83
83
84 def __init__(self, app):
84 def __init__(self, app):
85 contact_name = release.authors['Min'][0]
85 contact_name = release.authors['Min'][0]
86 contact_email = release.authors['Min'][1]
86 contact_email = release.authors['Min'][1]
87 bug_tracker = 'http://github.com/ipython/ipython/issues'
87 bug_tracker = 'http://github.com/ipython/ipython/issues'
88 super(ParallelCrashHandler,self).__init__(
88 super(ParallelCrashHandler,self).__init__(
89 app, contact_name, contact_email, bug_tracker
89 app, contact_name, contact_email, bug_tracker
90 )
90 )
91
91
92
92
93 #-----------------------------------------------------------------------------
93 #-----------------------------------------------------------------------------
94 # Main application
94 # Main application
95 #-----------------------------------------------------------------------------
95 #-----------------------------------------------------------------------------
96 base_aliases = {}
96 base_aliases = {}
97 base_aliases.update(base_ip_aliases)
97 base_aliases.update(base_ip_aliases)
98 base_aliases.update({
98 base_aliases.update({
99 'profile_dir' : 'ProfileDir.location',
99 'profile-dir' : 'ProfileDir.location',
100 'work_dir' : 'BaseParallelApplication.work_dir',
100 'work-dir' : 'BaseParallelApplication.work_dir',
101 'log_to_file' : 'BaseParallelApplication.log_to_file',
101 'log-to-file' : 'BaseParallelApplication.log_to_file',
102 'clean_logs' : 'BaseParallelApplication.clean_logs',
102 'clean-logs' : 'BaseParallelApplication.clean_logs',
103 'log_url' : 'BaseParallelApplication.log_url',
103 'log-url' : 'BaseParallelApplication.log_url',
104 })
104 })
105
105
106 base_flags = {
106 base_flags = {
107 'log-to-file' : (
107 'log-to-file' : (
108 {'BaseParallelApplication' : {'log_to_file' : True}},
108 {'BaseParallelApplication' : {'log_to_file' : True}},
109 "send log output to a file"
109 "send log output to a file"
110 )
110 )
111 }
111 }
112 base_flags.update(base_ip_flags)
112 base_flags.update(base_ip_flags)
113
113
114 class BaseParallelApplication(BaseIPythonApplication):
114 class BaseParallelApplication(BaseIPythonApplication):
115 """The base Application for IPython.parallel apps
115 """The base Application for IPython.parallel apps
116
116
117 Principle extensions to BaseIPyythonApplication:
117 Principle extensions to BaseIPyythonApplication:
118
118
119 * work_dir
119 * work_dir
120 * remote logging via pyzmq
120 * remote logging via pyzmq
121 * IOLoop instance
121 * IOLoop instance
122 """
122 """
123
123
124 crash_handler_class = ParallelCrashHandler
124 crash_handler_class = ParallelCrashHandler
125
125
126 def _log_level_default(self):
126 def _log_level_default(self):
127 # temporarily override default_log_level to INFO
127 # temporarily override default_log_level to INFO
128 return logging.INFO
128 return logging.INFO
129
129
130 work_dir = Unicode(os.getcwdu(), config=True,
130 work_dir = Unicode(os.getcwdu(), config=True,
131 help='Set the working dir for the process.'
131 help='Set the working dir for the process.'
132 )
132 )
133 def _work_dir_changed(self, name, old, new):
133 def _work_dir_changed(self, name, old, new):
134 self.work_dir = unicode(expand_path(new))
134 self.work_dir = unicode(expand_path(new))
135
135
136 log_to_file = Bool(config=True,
136 log_to_file = Bool(config=True,
137 help="whether to log to a file")
137 help="whether to log to a file")
138
138
139 clean_logs = Bool(False, config=True,
139 clean_logs = Bool(False, config=True,
140 help="whether to cleanup old logfiles before starting")
140 help="whether to cleanup old logfiles before starting")
141
141
142 log_url = Unicode('', config=True,
142 log_url = Unicode('', config=True,
143 help="The ZMQ URL of the iplogger to aggregate logging.")
143 help="The ZMQ URL of the iplogger to aggregate logging.")
144
144
145 def _config_files_default(self):
145 def _config_files_default(self):
146 return ['ipcontroller_config.py', 'ipengine_config.py', 'ipcluster_config.py']
146 return ['ipcontroller_config.py', 'ipengine_config.py', 'ipcluster_config.py']
147
147
148 loop = Instance('zmq.eventloop.ioloop.IOLoop')
148 loop = Instance('zmq.eventloop.ioloop.IOLoop')
149 def _loop_default(self):
149 def _loop_default(self):
150 from zmq.eventloop.ioloop import IOLoop
150 from zmq.eventloop.ioloop import IOLoop
151 return IOLoop.instance()
151 return IOLoop.instance()
152
152
153 aliases = Dict(base_aliases)
153 aliases = Dict(base_aliases)
154 flags = Dict(base_flags)
154 flags = Dict(base_flags)
155
155
156 def initialize(self, argv=None):
156 def initialize(self, argv=None):
157 """initialize the app"""
157 """initialize the app"""
158 super(BaseParallelApplication, self).initialize(argv)
158 super(BaseParallelApplication, self).initialize(argv)
159 self.to_work_dir()
159 self.to_work_dir()
160 self.reinit_logging()
160 self.reinit_logging()
161
161
162 def to_work_dir(self):
162 def to_work_dir(self):
163 wd = self.work_dir
163 wd = self.work_dir
164 if unicode(wd) != os.getcwdu():
164 if unicode(wd) != os.getcwdu():
165 os.chdir(wd)
165 os.chdir(wd)
166 self.log.info("Changing to working dir: %s" % wd)
166 self.log.info("Changing to working dir: %s" % wd)
167 # This is the working dir by now.
167 # This is the working dir by now.
168 sys.path.insert(0, '')
168 sys.path.insert(0, '')
169
169
170 def reinit_logging(self):
170 def reinit_logging(self):
171 # Remove old log files
171 # Remove old log files
172 log_dir = self.profile_dir.log_dir
172 log_dir = self.profile_dir.log_dir
173 if self.clean_logs:
173 if self.clean_logs:
174 for f in os.listdir(log_dir):
174 for f in os.listdir(log_dir):
175 if re.match(r'%s-\d+\.(log|err|out)'%self.name,f):
175 if re.match(r'%s-\d+\.(log|err|out)'%self.name,f):
176 os.remove(os.path.join(log_dir, f))
176 os.remove(os.path.join(log_dir, f))
177 if self.log_to_file:
177 if self.log_to_file:
178 # Start logging to the new log file
178 # Start logging to the new log file
179 log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
179 log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
180 logfile = os.path.join(log_dir, log_filename)
180 logfile = os.path.join(log_dir, log_filename)
181 open_log_file = open(logfile, 'w')
181 open_log_file = open(logfile, 'w')
182 else:
182 else:
183 open_log_file = None
183 open_log_file = None
184 if open_log_file is not None:
184 if open_log_file is not None:
185 self.log.removeHandler(self._log_handler)
185 self.log.removeHandler(self._log_handler)
186 self._log_handler = logging.StreamHandler(open_log_file)
186 self._log_handler = logging.StreamHandler(open_log_file)
187 self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
187 self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
188 self._log_handler.setFormatter(self._log_formatter)
188 self._log_handler.setFormatter(self._log_formatter)
189 self.log.addHandler(self._log_handler)
189 self.log.addHandler(self._log_handler)
190
190
191 def write_pid_file(self, overwrite=False):
191 def write_pid_file(self, overwrite=False):
192 """Create a .pid file in the pid_dir with my pid.
192 """Create a .pid file in the pid_dir with my pid.
193
193
194 This must be called after pre_construct, which sets `self.pid_dir`.
194 This must be called after pre_construct, which sets `self.pid_dir`.
195 This raises :exc:`PIDFileError` if the pid file exists already.
195 This raises :exc:`PIDFileError` if the pid file exists already.
196 """
196 """
197 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
197 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
198 if os.path.isfile(pid_file):
198 if os.path.isfile(pid_file):
199 pid = self.get_pid_from_file()
199 pid = self.get_pid_from_file()
200 if not overwrite:
200 if not overwrite:
201 raise PIDFileError(
201 raise PIDFileError(
202 'The pid file [%s] already exists. \nThis could mean that this '
202 'The pid file [%s] already exists. \nThis could mean that this '
203 'server is already running with [pid=%s].' % (pid_file, pid)
203 'server is already running with [pid=%s].' % (pid_file, pid)
204 )
204 )
205 with open(pid_file, 'w') as f:
205 with open(pid_file, 'w') as f:
206 self.log.info("Creating pid file: %s" % pid_file)
206 self.log.info("Creating pid file: %s" % pid_file)
207 f.write(repr(os.getpid())+'\n')
207 f.write(repr(os.getpid())+'\n')
208
208
209 def remove_pid_file(self):
209 def remove_pid_file(self):
210 """Remove the pid file.
210 """Remove the pid file.
211
211
212 This should be called at shutdown by registering a callback with
212 This should be called at shutdown by registering a callback with
213 :func:`reactor.addSystemEventTrigger`. This needs to return
213 :func:`reactor.addSystemEventTrigger`. This needs to return
214 ``None``.
214 ``None``.
215 """
215 """
216 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
216 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
217 if os.path.isfile(pid_file):
217 if os.path.isfile(pid_file):
218 try:
218 try:
219 self.log.info("Removing pid file: %s" % pid_file)
219 self.log.info("Removing pid file: %s" % pid_file)
220 os.remove(pid_file)
220 os.remove(pid_file)
221 except:
221 except:
222 self.log.warn("Error removing the pid file: %s" % pid_file)
222 self.log.warn("Error removing the pid file: %s" % pid_file)
223
223
224 def get_pid_from_file(self):
224 def get_pid_from_file(self):
225 """Get the pid from the pid file.
225 """Get the pid from the pid file.
226
226
227 If the pid file doesn't exist a :exc:`PIDFileError` is raised.
227 If the pid file doesn't exist a :exc:`PIDFileError` is raised.
228 """
228 """
229 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
229 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
230 if os.path.isfile(pid_file):
230 if os.path.isfile(pid_file):
231 with open(pid_file, 'r') as f:
231 with open(pid_file, 'r') as f:
232 s = f.read().strip()
232 s = f.read().strip()
233 try:
233 try:
234 pid = int(s)
234 pid = int(s)
235 except:
235 except:
236 raise PIDFileError("invalid pid file: %s (contents: %r)"%(pid_file, s))
236 raise PIDFileError("invalid pid file: %s (contents: %r)"%(pid_file, s))
237 return pid
237 return pid
238 else:
238 else:
239 raise PIDFileError('pid file not found: %s' % pid_file)
239 raise PIDFileError('pid file not found: %s' % pid_file)
240
240
241 def check_pid(self, pid):
241 def check_pid(self, pid):
242 if os.name == 'nt':
242 if os.name == 'nt':
243 try:
243 try:
244 import ctypes
244 import ctypes
245 # returns 0 if no such process (of ours) exists
245 # returns 0 if no such process (of ours) exists
246 # positive int otherwise
246 # positive int otherwise
247 p = ctypes.windll.kernel32.OpenProcess(1,0,pid)
247 p = ctypes.windll.kernel32.OpenProcess(1,0,pid)
248 except Exception:
248 except Exception:
249 self.log.warn(
249 self.log.warn(
250 "Could not determine whether pid %i is running via `OpenProcess`. "
250 "Could not determine whether pid %i is running via `OpenProcess`. "
251 " Making the likely assumption that it is."%pid
251 " Making the likely assumption that it is."%pid
252 )
252 )
253 return True
253 return True
254 return bool(p)
254 return bool(p)
255 else:
255 else:
256 try:
256 try:
257 p = Popen(['ps','x'], stdout=PIPE, stderr=PIPE)
257 p = Popen(['ps','x'], stdout=PIPE, stderr=PIPE)
258 output,_ = p.communicate()
258 output,_ = p.communicate()
259 except OSError:
259 except OSError:
260 self.log.warn(
260 self.log.warn(
261 "Could not determine whether pid %i is running via `ps x`. "
261 "Could not determine whether pid %i is running via `ps x`. "
262 " Making the likely assumption that it is."%pid
262 " Making the likely assumption that it is."%pid
263 )
263 )
264 return True
264 return True
265 pids = map(int, re.findall(r'^\W*\d+', output, re.MULTILINE))
265 pids = map(int, re.findall(r'^\W*\d+', output, re.MULTILINE))
266 return pid in pids
266 return pid in pids
@@ -1,459 +1,459 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The ipcluster application.
4 The ipcluster application.
5
5
6 Authors:
6 Authors:
7
7
8 * Brian Granger
8 * Brian Granger
9 * MinRK
9 * MinRK
10
10
11 """
11 """
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Copyright (C) 2008-2011 The IPython Development Team
14 # Copyright (C) 2008-2011 The IPython Development Team
15 #
15 #
16 # Distributed under the terms of the BSD License. The full license is in
16 # Distributed under the terms of the BSD License. The full license is in
17 # the file COPYING, distributed as part of this software.
17 # the file COPYING, distributed as part of this software.
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Imports
21 # Imports
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 import errno
24 import errno
25 import logging
25 import logging
26 import os
26 import os
27 import re
27 import re
28 import signal
28 import signal
29
29
30 from subprocess import check_call, CalledProcessError, PIPE
30 from subprocess import check_call, CalledProcessError, PIPE
31 import zmq
31 import zmq
32 from zmq.eventloop import ioloop
32 from zmq.eventloop import ioloop
33
33
34 from IPython.config.application import Application, boolean_flag
34 from IPython.config.application import Application, boolean_flag
35 from IPython.config.loader import Config
35 from IPython.config.loader import Config
36 from IPython.core.application import BaseIPythonApplication
36 from IPython.core.application import BaseIPythonApplication
37 from IPython.core.profiledir import ProfileDir
37 from IPython.core.profiledir import ProfileDir
38 from IPython.utils.daemonize import daemonize
38 from IPython.utils.daemonize import daemonize
39 from IPython.utils.importstring import import_item
39 from IPython.utils.importstring import import_item
40 from IPython.utils.traitlets import (Int, Unicode, Bool, CFloat, Dict, List,
40 from IPython.utils.traitlets import (Int, Unicode, Bool, CFloat, Dict, List,
41 DottedObjectName)
41 DottedObjectName)
42
42
43 from IPython.parallel.apps.baseapp import (
43 from IPython.parallel.apps.baseapp import (
44 BaseParallelApplication,
44 BaseParallelApplication,
45 PIDFileError,
45 PIDFileError,
46 base_flags, base_aliases
46 base_flags, base_aliases
47 )
47 )
48
48
49
49
50 #-----------------------------------------------------------------------------
50 #-----------------------------------------------------------------------------
51 # Module level variables
51 # Module level variables
52 #-----------------------------------------------------------------------------
52 #-----------------------------------------------------------------------------
53
53
54
54
55 default_config_file_name = u'ipcluster_config.py'
55 default_config_file_name = u'ipcluster_config.py'
56
56
57
57
58 _description = """Start an IPython cluster for parallel computing.
58 _description = """Start an IPython cluster for parallel computing.
59
59
60 An IPython cluster consists of 1 controller and 1 or more engines.
60 An IPython cluster consists of 1 controller and 1 or more engines.
61 This command automates the startup of these processes using a wide
61 This command automates the startup of these processes using a wide
62 range of startup methods (SSH, local processes, PBS, mpiexec,
62 range of startup methods (SSH, local processes, PBS, mpiexec,
63 Windows HPC Server 2008). To start a cluster with 4 engines on your
63 Windows HPC Server 2008). To start a cluster with 4 engines on your
64 local host simply do 'ipcluster start n=4'. For more complex usage
64 local host simply do 'ipcluster start n=4'. For more complex usage
65 you will typically do 'ipcluster create profile=mycluster', then edit
65 you will typically do 'ipcluster create profile=mycluster', then edit
66 configuration files, followed by 'ipcluster start profile=mycluster n=4'.
66 configuration files, followed by 'ipcluster start profile=mycluster n=4'.
67 """
67 """
68
68
69
69
70 # Exit codes for ipcluster
70 # Exit codes for ipcluster
71
71
72 # This will be the exit code if the ipcluster appears to be running because
72 # This will be the exit code if the ipcluster appears to be running because
73 # a .pid file exists
73 # a .pid file exists
74 ALREADY_STARTED = 10
74 ALREADY_STARTED = 10
75
75
76
76
77 # This will be the exit code if ipcluster stop is run, but there is not .pid
77 # This will be the exit code if ipcluster stop is run, but there is not .pid
78 # file to be found.
78 # file to be found.
79 ALREADY_STOPPED = 11
79 ALREADY_STOPPED = 11
80
80
81 # This will be the exit code if ipcluster engines is run, but there is not .pid
81 # This will be the exit code if ipcluster engines is run, but there is not .pid
82 # file to be found.
82 # file to be found.
83 NO_CLUSTER = 12
83 NO_CLUSTER = 12
84
84
85
85
86 #-----------------------------------------------------------------------------
86 #-----------------------------------------------------------------------------
87 # Main application
87 # Main application
88 #-----------------------------------------------------------------------------
88 #-----------------------------------------------------------------------------
89 start_help = """Start an IPython cluster for parallel computing
89 start_help = """Start an IPython cluster for parallel computing
90
90
91 Start an ipython cluster by its profile name or cluster
91 Start an ipython cluster by its profile name or cluster
92 directory. Cluster directories contain configuration, log and
92 directory. Cluster directories contain configuration, log and
93 security related files and are named using the convention
93 security related files and are named using the convention
94 'profile_<name>' and should be creating using the 'start'
94 'profile_<name>' and should be creating using the 'start'
95 subcommand of 'ipcluster'. If your cluster directory is in
95 subcommand of 'ipcluster'. If your cluster directory is in
96 the cwd or the ipython directory, you can simply refer to it
96 the cwd or the ipython directory, you can simply refer to it
97 using its profile name, 'ipcluster start n=4 profile=<profile>`,
97 using its profile name, 'ipcluster start n=4 profile=<profile>`,
98 otherwise use the 'profile_dir' option.
98 otherwise use the 'profile_dir' option.
99 """
99 """
100 stop_help = """Stop a running IPython cluster
100 stop_help = """Stop a running IPython cluster
101
101
102 Stop a running ipython cluster by its profile name or cluster
102 Stop a running ipython cluster by its profile name or cluster
103 directory. Cluster directories are named using the convention
103 directory. Cluster directories are named using the convention
104 'profile_<name>'. If your cluster directory is in
104 'profile_<name>'. If your cluster directory is in
105 the cwd or the ipython directory, you can simply refer to it
105 the cwd or the ipython directory, you can simply refer to it
106 using its profile name, 'ipcluster stop profile=<profile>`, otherwise
106 using its profile name, 'ipcluster stop profile=<profile>`, otherwise
107 use the 'profile_dir' option.
107 use the 'profile_dir' option.
108 """
108 """
109 engines_help = """Start engines connected to an existing IPython cluster
109 engines_help = """Start engines connected to an existing IPython cluster
110
110
111 Start one or more engines to connect to an existing Cluster
111 Start one or more engines to connect to an existing Cluster
112 by profile name or cluster directory.
112 by profile name or cluster directory.
113 Cluster directories contain configuration, log and
113 Cluster directories contain configuration, log and
114 security related files and are named using the convention
114 security related files and are named using the convention
115 'profile_<name>' and should be creating using the 'start'
115 'profile_<name>' and should be creating using the 'start'
116 subcommand of 'ipcluster'. If your cluster directory is in
116 subcommand of 'ipcluster'. If your cluster directory is in
117 the cwd or the ipython directory, you can simply refer to it
117 the cwd or the ipython directory, you can simply refer to it
118 using its profile name, 'ipcluster engines n=4 profile=<profile>`,
118 using its profile name, 'ipcluster engines n=4 profile=<profile>`,
119 otherwise use the 'profile_dir' option.
119 otherwise use the 'profile_dir' option.
120 """
120 """
121 stop_aliases = dict(
121 stop_aliases = dict(
122 signal='IPClusterStop.signal',
122 signal='IPClusterStop.signal',
123 )
123 )
124 stop_aliases.update(base_aliases)
124 stop_aliases.update(base_aliases)
125
125
126 class IPClusterStop(BaseParallelApplication):
126 class IPClusterStop(BaseParallelApplication):
127 name = u'ipcluster'
127 name = u'ipcluster'
128 description = stop_help
128 description = stop_help
129 config_file_name = Unicode(default_config_file_name)
129 config_file_name = Unicode(default_config_file_name)
130
130
131 signal = Int(signal.SIGINT, config=True,
131 signal = Int(signal.SIGINT, config=True,
132 help="signal to use for stopping processes.")
132 help="signal to use for stopping processes.")
133
133
134 aliases = Dict(stop_aliases)
134 aliases = Dict(stop_aliases)
135
135
136 def start(self):
136 def start(self):
137 """Start the app for the stop subcommand."""
137 """Start the app for the stop subcommand."""
138 try:
138 try:
139 pid = self.get_pid_from_file()
139 pid = self.get_pid_from_file()
140 except PIDFileError:
140 except PIDFileError:
141 self.log.critical(
141 self.log.critical(
142 'Could not read pid file, cluster is probably not running.'
142 'Could not read pid file, cluster is probably not running.'
143 )
143 )
144 # Here I exit with a unusual exit status that other processes
144 # Here I exit with a unusual exit status that other processes
145 # can watch for to learn how I existed.
145 # can watch for to learn how I existed.
146 self.remove_pid_file()
146 self.remove_pid_file()
147 self.exit(ALREADY_STOPPED)
147 self.exit(ALREADY_STOPPED)
148
148
149 if not self.check_pid(pid):
149 if not self.check_pid(pid):
150 self.log.critical(
150 self.log.critical(
151 'Cluster [pid=%r] is not running.' % pid
151 'Cluster [pid=%r] is not running.' % pid
152 )
152 )
153 self.remove_pid_file()
153 self.remove_pid_file()
154 # Here I exit with a unusual exit status that other processes
154 # Here I exit with a unusual exit status that other processes
155 # can watch for to learn how I existed.
155 # can watch for to learn how I existed.
156 self.exit(ALREADY_STOPPED)
156 self.exit(ALREADY_STOPPED)
157
157
158 elif os.name=='posix':
158 elif os.name=='posix':
159 sig = self.signal
159 sig = self.signal
160 self.log.info(
160 self.log.info(
161 "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
161 "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
162 )
162 )
163 try:
163 try:
164 os.kill(pid, sig)
164 os.kill(pid, sig)
165 except OSError:
165 except OSError:
166 self.log.error("Stopping cluster failed, assuming already dead.",
166 self.log.error("Stopping cluster failed, assuming already dead.",
167 exc_info=True)
167 exc_info=True)
168 self.remove_pid_file()
168 self.remove_pid_file()
169 elif os.name=='nt':
169 elif os.name=='nt':
170 try:
170 try:
171 # kill the whole tree
171 # kill the whole tree
172 p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE)
172 p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE)
173 except (CalledProcessError, OSError):
173 except (CalledProcessError, OSError):
174 self.log.error("Stopping cluster failed, assuming already dead.",
174 self.log.error("Stopping cluster failed, assuming already dead.",
175 exc_info=True)
175 exc_info=True)
176 self.remove_pid_file()
176 self.remove_pid_file()
177
177
178 engine_aliases = {}
178 engine_aliases = {}
179 engine_aliases.update(base_aliases)
179 engine_aliases.update(base_aliases)
180 engine_aliases.update(dict(
180 engine_aliases.update(dict(
181 n='IPClusterEngines.n',
181 n='IPClusterEngines.n',
182 engines = 'IPClusterEngines.engine_launcher_class',
182 engines = 'IPClusterEngines.engine_launcher_class',
183 daemonize = 'IPClusterEngines.daemonize',
183 daemonize = 'IPClusterEngines.daemonize',
184 ))
184 ))
185 engine_flags = {}
185 engine_flags = {}
186 engine_flags.update(base_flags)
186 engine_flags.update(base_flags)
187
187
188 engine_flags.update(dict(
188 engine_flags.update(dict(
189 daemonize=(
189 daemonize=(
190 {'IPClusterEngines' : {'daemonize' : True}},
190 {'IPClusterEngines' : {'daemonize' : True}},
191 """run the cluster into the background (not available on Windows)""",
191 """run the cluster into the background (not available on Windows)""",
192 )
192 )
193 ))
193 ))
194 class IPClusterEngines(BaseParallelApplication):
194 class IPClusterEngines(BaseParallelApplication):
195
195
196 name = u'ipcluster'
196 name = u'ipcluster'
197 description = engines_help
197 description = engines_help
198 usage = None
198 usage = None
199 config_file_name = Unicode(default_config_file_name)
199 config_file_name = Unicode(default_config_file_name)
200 default_log_level = logging.INFO
200 default_log_level = logging.INFO
201 classes = List()
201 classes = List()
202 def _classes_default(self):
202 def _classes_default(self):
203 from IPython.parallel.apps import launcher
203 from IPython.parallel.apps import launcher
204 launchers = launcher.all_launchers
204 launchers = launcher.all_launchers
205 eslaunchers = [ l for l in launchers if 'EngineSet' in l.__name__]
205 eslaunchers = [ l for l in launchers if 'EngineSet' in l.__name__]
206 return [ProfileDir]+eslaunchers
206 return [ProfileDir]+eslaunchers
207
207
208 n = Int(2, config=True,
208 n = Int(2, config=True,
209 help="The number of engines to start.")
209 help="The number of engines to start.")
210
210
211 engine_launcher_class = DottedObjectName('LocalEngineSetLauncher',
211 engine_launcher_class = DottedObjectName('LocalEngineSetLauncher',
212 config=True,
212 config=True,
213 help="The class for launching a set of Engines."
213 help="The class for launching a set of Engines."
214 )
214 )
215 daemonize = Bool(False, config=True,
215 daemonize = Bool(False, config=True,
216 help="""Daemonize the ipcluster program. This implies --log-to-file.
216 help="""Daemonize the ipcluster program. This implies --log-to-file.
217 Not available on Windows.
217 Not available on Windows.
218 """)
218 """)
219
219
220 def _daemonize_changed(self, name, old, new):
220 def _daemonize_changed(self, name, old, new):
221 if new:
221 if new:
222 self.log_to_file = True
222 self.log_to_file = True
223
223
224 aliases = Dict(engine_aliases)
224 aliases = Dict(engine_aliases)
225 flags = Dict(engine_flags)
225 flags = Dict(engine_flags)
226 _stopping = False
226 _stopping = False
227
227
228 def initialize(self, argv=None):
228 def initialize(self, argv=None):
229 super(IPClusterEngines, self).initialize(argv)
229 super(IPClusterEngines, self).initialize(argv)
230 self.init_signal()
230 self.init_signal()
231 self.init_launchers()
231 self.init_launchers()
232
232
233 def init_launchers(self):
233 def init_launchers(self):
234 self.engine_launcher = self.build_launcher(self.engine_launcher_class)
234 self.engine_launcher = self.build_launcher(self.engine_launcher_class)
235 self.engine_launcher.on_stop(lambda r: self.loop.stop())
235 self.engine_launcher.on_stop(lambda r: self.loop.stop())
236
236
237 def init_signal(self):
237 def init_signal(self):
238 # Setup signals
238 # Setup signals
239 signal.signal(signal.SIGINT, self.sigint_handler)
239 signal.signal(signal.SIGINT, self.sigint_handler)
240
240
241 def build_launcher(self, clsname):
241 def build_launcher(self, clsname):
242 """import and instantiate a Launcher based on importstring"""
242 """import and instantiate a Launcher based on importstring"""
243 if '.' not in clsname:
243 if '.' not in clsname:
244 # not a module, presume it's the raw name in apps.launcher
244 # not a module, presume it's the raw name in apps.launcher
245 clsname = 'IPython.parallel.apps.launcher.'+clsname
245 clsname = 'IPython.parallel.apps.launcher.'+clsname
246 # print repr(clsname)
246 # print repr(clsname)
247 klass = import_item(clsname)
247 klass = import_item(clsname)
248
248
249 launcher = klass(
249 launcher = klass(
250 work_dir=self.profile_dir.location, config=self.config, log=self.log
250 work_dir=self.profile_dir.location, config=self.config, log=self.log
251 )
251 )
252 return launcher
252 return launcher
253
253
254 def start_engines(self):
254 def start_engines(self):
255 self.log.info("Starting %i engines"%self.n)
255 self.log.info("Starting %i engines"%self.n)
256 self.engine_launcher.start(
256 self.engine_launcher.start(
257 self.n,
257 self.n,
258 self.profile_dir.location
258 self.profile_dir.location
259 )
259 )
260
260
261 def stop_engines(self):
261 def stop_engines(self):
262 self.log.info("Stopping Engines...")
262 self.log.info("Stopping Engines...")
263 if self.engine_launcher.running:
263 if self.engine_launcher.running:
264 d = self.engine_launcher.stop()
264 d = self.engine_launcher.stop()
265 return d
265 return d
266 else:
266 else:
267 return None
267 return None
268
268
269 def stop_launchers(self, r=None):
269 def stop_launchers(self, r=None):
270 if not self._stopping:
270 if not self._stopping:
271 self._stopping = True
271 self._stopping = True
272 self.log.error("IPython cluster: stopping")
272 self.log.error("IPython cluster: stopping")
273 self.stop_engines()
273 self.stop_engines()
274 # Wait a few seconds to let things shut down.
274 # Wait a few seconds to let things shut down.
275 dc = ioloop.DelayedCallback(self.loop.stop, 4000, self.loop)
275 dc = ioloop.DelayedCallback(self.loop.stop, 4000, self.loop)
276 dc.start()
276 dc.start()
277
277
278 def sigint_handler(self, signum, frame):
278 def sigint_handler(self, signum, frame):
279 self.log.debug("SIGINT received, stopping launchers...")
279 self.log.debug("SIGINT received, stopping launchers...")
280 self.stop_launchers()
280 self.stop_launchers()
281
281
282 def start_logging(self):
282 def start_logging(self):
283 # Remove old log files of the controller and engine
283 # Remove old log files of the controller and engine
284 if self.clean_logs:
284 if self.clean_logs:
285 log_dir = self.profile_dir.log_dir
285 log_dir = self.profile_dir.log_dir
286 for f in os.listdir(log_dir):
286 for f in os.listdir(log_dir):
287 if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f):
287 if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f):
288 os.remove(os.path.join(log_dir, f))
288 os.remove(os.path.join(log_dir, f))
289 # This will remove old log files for ipcluster itself
289 # This will remove old log files for ipcluster itself
290 # super(IPBaseParallelApplication, self).start_logging()
290 # super(IPBaseParallelApplication, self).start_logging()
291
291
292 def start(self):
292 def start(self):
293 """Start the app for the engines subcommand."""
293 """Start the app for the engines subcommand."""
294 self.log.info("IPython cluster: started")
294 self.log.info("IPython cluster: started")
295 # First see if the cluster is already running
295 # First see if the cluster is already running
296
296
297 # Now log and daemonize
297 # Now log and daemonize
298 self.log.info(
298 self.log.info(
299 'Starting engines with [daemon=%r]' % self.daemonize
299 'Starting engines with [daemon=%r]' % self.daemonize
300 )
300 )
301 # TODO: Get daemonize working on Windows or as a Windows Server.
301 # TODO: Get daemonize working on Windows or as a Windows Server.
302 if self.daemonize:
302 if self.daemonize:
303 if os.name=='posix':
303 if os.name=='posix':
304 daemonize()
304 daemonize()
305
305
306 dc = ioloop.DelayedCallback(self.start_engines, 0, self.loop)
306 dc = ioloop.DelayedCallback(self.start_engines, 0, self.loop)
307 dc.start()
307 dc.start()
308 # Now write the new pid file AFTER our new forked pid is active.
308 # Now write the new pid file AFTER our new forked pid is active.
309 # self.write_pid_file()
309 # self.write_pid_file()
310 try:
310 try:
311 self.loop.start()
311 self.loop.start()
312 except KeyboardInterrupt:
312 except KeyboardInterrupt:
313 pass
313 pass
314 except zmq.ZMQError as e:
314 except zmq.ZMQError as e:
315 if e.errno == errno.EINTR:
315 if e.errno == errno.EINTR:
316 pass
316 pass
317 else:
317 else:
318 raise
318 raise
319
319
320 start_aliases = {}
320 start_aliases = {}
321 start_aliases.update(engine_aliases)
321 start_aliases.update(engine_aliases)
322 start_aliases.update(dict(
322 start_aliases.update(dict(
323 delay='IPClusterStart.delay',
323 delay='IPClusterStart.delay',
324 clean_logs='IPClusterStart.clean_logs',
325 controller = 'IPClusterStart.controller_launcher_class',
324 controller = 'IPClusterStart.controller_launcher_class',
326 ))
325 ))
326 start_aliases['clean-logs'] = 'IPClusterStart.clean_logs'
327
327
328 class IPClusterStart(IPClusterEngines):
328 class IPClusterStart(IPClusterEngines):
329
329
330 name = u'ipcluster'
330 name = u'ipcluster'
331 description = start_help
331 description = start_help
332 default_log_level = logging.INFO
332 default_log_level = logging.INFO
333 auto_create = Bool(True, config=True,
333 auto_create = Bool(True, config=True,
334 help="whether to create the profile_dir if it doesn't exist")
334 help="whether to create the profile_dir if it doesn't exist")
335 classes = List()
335 classes = List()
336 def _classes_default(self,):
336 def _classes_default(self,):
337 from IPython.parallel.apps import launcher
337 from IPython.parallel.apps import launcher
338 return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers
338 return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers
339
339
340 clean_logs = Bool(True, config=True,
340 clean_logs = Bool(True, config=True,
341 help="whether to cleanup old logs before starting")
341 help="whether to cleanup old logs before starting")
342
342
343 delay = CFloat(1., config=True,
343 delay = CFloat(1., config=True,
344 help="delay (in s) between starting the controller and the engines")
344 help="delay (in s) between starting the controller and the engines")
345
345
346 controller_launcher_class = DottedObjectName('LocalControllerLauncher',
346 controller_launcher_class = DottedObjectName('LocalControllerLauncher',
347 config=True,
347 config=True,
348 help="The class for launching a Controller."
348 help="The class for launching a Controller."
349 )
349 )
350 reset = Bool(False, config=True,
350 reset = Bool(False, config=True,
351 help="Whether to reset config files as part of '--create'."
351 help="Whether to reset config files as part of '--create'."
352 )
352 )
353
353
354 # flags = Dict(flags)
354 # flags = Dict(flags)
355 aliases = Dict(start_aliases)
355 aliases = Dict(start_aliases)
356
356
357 def init_launchers(self):
357 def init_launchers(self):
358 self.controller_launcher = self.build_launcher(self.controller_launcher_class)
358 self.controller_launcher = self.build_launcher(self.controller_launcher_class)
359 self.engine_launcher = self.build_launcher(self.engine_launcher_class)
359 self.engine_launcher = self.build_launcher(self.engine_launcher_class)
360 self.controller_launcher.on_stop(self.stop_launchers)
360 self.controller_launcher.on_stop(self.stop_launchers)
361
361
362 def start_controller(self):
362 def start_controller(self):
363 self.controller_launcher.start(
363 self.controller_launcher.start(
364 self.profile_dir.location
364 self.profile_dir.location
365 )
365 )
366
366
367 def stop_controller(self):
367 def stop_controller(self):
368 # self.log.info("In stop_controller")
368 # self.log.info("In stop_controller")
369 if self.controller_launcher and self.controller_launcher.running:
369 if self.controller_launcher and self.controller_launcher.running:
370 return self.controller_launcher.stop()
370 return self.controller_launcher.stop()
371
371
372 def stop_launchers(self, r=None):
372 def stop_launchers(self, r=None):
373 if not self._stopping:
373 if not self._stopping:
374 self.stop_controller()
374 self.stop_controller()
375 super(IPClusterStart, self).stop_launchers()
375 super(IPClusterStart, self).stop_launchers()
376
376
377 def start(self):
377 def start(self):
378 """Start the app for the start subcommand."""
378 """Start the app for the start subcommand."""
379 # First see if the cluster is already running
379 # First see if the cluster is already running
380 try:
380 try:
381 pid = self.get_pid_from_file()
381 pid = self.get_pid_from_file()
382 except PIDFileError:
382 except PIDFileError:
383 pass
383 pass
384 else:
384 else:
385 if self.check_pid(pid):
385 if self.check_pid(pid):
386 self.log.critical(
386 self.log.critical(
387 'Cluster is already running with [pid=%s]. '
387 'Cluster is already running with [pid=%s]. '
388 'use "ipcluster stop" to stop the cluster.' % pid
388 'use "ipcluster stop" to stop the cluster.' % pid
389 )
389 )
390 # Here I exit with a unusual exit status that other processes
390 # Here I exit with a unusual exit status that other processes
391 # can watch for to learn how I existed.
391 # can watch for to learn how I existed.
392 self.exit(ALREADY_STARTED)
392 self.exit(ALREADY_STARTED)
393 else:
393 else:
394 self.remove_pid_file()
394 self.remove_pid_file()
395
395
396
396
397 # Now log and daemonize
397 # Now log and daemonize
398 self.log.info(
398 self.log.info(
399 'Starting ipcluster with [daemon=%r]' % self.daemonize
399 'Starting ipcluster with [daemon=%r]' % self.daemonize
400 )
400 )
401 # TODO: Get daemonize working on Windows or as a Windows Server.
401 # TODO: Get daemonize working on Windows or as a Windows Server.
402 if self.daemonize:
402 if self.daemonize:
403 if os.name=='posix':
403 if os.name=='posix':
404 daemonize()
404 daemonize()
405
405
406 dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop)
406 dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop)
407 dc.start()
407 dc.start()
408 dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop)
408 dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop)
409 dc.start()
409 dc.start()
410 # Now write the new pid file AFTER our new forked pid is active.
410 # Now write the new pid file AFTER our new forked pid is active.
411 self.write_pid_file()
411 self.write_pid_file()
412 try:
412 try:
413 self.loop.start()
413 self.loop.start()
414 except KeyboardInterrupt:
414 except KeyboardInterrupt:
415 pass
415 pass
416 except zmq.ZMQError as e:
416 except zmq.ZMQError as e:
417 if e.errno == errno.EINTR:
417 if e.errno == errno.EINTR:
418 pass
418 pass
419 else:
419 else:
420 raise
420 raise
421 finally:
421 finally:
422 self.remove_pid_file()
422 self.remove_pid_file()
423
423
424 base='IPython.parallel.apps.ipclusterapp.IPCluster'
424 base='IPython.parallel.apps.ipclusterapp.IPCluster'
425
425
426 class IPClusterApp(Application):
426 class IPClusterApp(Application):
427 name = u'ipcluster'
427 name = u'ipcluster'
428 description = _description
428 description = _description
429
429
430 subcommands = {
430 subcommands = {
431 'start' : (base+'Start', start_help),
431 'start' : (base+'Start', start_help),
432 'stop' : (base+'Stop', stop_help),
432 'stop' : (base+'Stop', stop_help),
433 'engines' : (base+'Engines', engines_help),
433 'engines' : (base+'Engines', engines_help),
434 }
434 }
435
435
436 # no aliases or flags for parent App
436 # no aliases or flags for parent App
437 aliases = Dict()
437 aliases = Dict()
438 flags = Dict()
438 flags = Dict()
439
439
440 def start(self):
440 def start(self):
441 if self.subapp is None:
441 if self.subapp is None:
442 print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
442 print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
443 print
443 print
444 self.print_description()
444 self.print_description()
445 self.print_subcommands()
445 self.print_subcommands()
446 self.exit(1)
446 self.exit(1)
447 else:
447 else:
448 return self.subapp.start()
448 return self.subapp.start()
449
449
450 def launch_new_instance():
450 def launch_new_instance():
451 """Create and run the IPython cluster."""
451 """Create and run the IPython cluster."""
452 app = IPClusterApp.instance()
452 app = IPClusterApp.instance()
453 app.initialize()
453 app.initialize()
454 app.start()
454 app.start()
455
455
456
456
457 if __name__ == '__main__':
457 if __name__ == '__main__':
458 launch_new_instance()
458 launch_new_instance()
459
459
@@ -1,422 +1,420 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The IPython controller application.
4 The IPython controller application.
5
5
6 Authors:
6 Authors:
7
7
8 * Brian Granger
8 * Brian Granger
9 * MinRK
9 * MinRK
10
10
11 """
11 """
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Copyright (C) 2008-2011 The IPython Development Team
14 # Copyright (C) 2008-2011 The IPython Development Team
15 #
15 #
16 # Distributed under the terms of the BSD License. The full license is in
16 # Distributed under the terms of the BSD License. The full license is in
17 # the file COPYING, distributed as part of this software.
17 # the file COPYING, distributed as part of this software.
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Imports
21 # Imports
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 from __future__ import with_statement
24 from __future__ import with_statement
25
25
26 import os
26 import os
27 import socket
27 import socket
28 import stat
28 import stat
29 import sys
29 import sys
30 import uuid
30 import uuid
31
31
32 from multiprocessing import Process
32 from multiprocessing import Process
33
33
34 import zmq
34 import zmq
35 from zmq.devices import ProcessMonitoredQueue
35 from zmq.devices import ProcessMonitoredQueue
36 from zmq.log.handlers import PUBHandler
36 from zmq.log.handlers import PUBHandler
37 from zmq.utils import jsonapi as json
37 from zmq.utils import jsonapi as json
38
38
39 from IPython.config.application import boolean_flag
39 from IPython.config.application import boolean_flag
40 from IPython.core.profiledir import ProfileDir
40 from IPython.core.profiledir import ProfileDir
41
41
42 from IPython.parallel.apps.baseapp import (
42 from IPython.parallel.apps.baseapp import (
43 BaseParallelApplication,
43 BaseParallelApplication,
44 base_aliases,
44 base_aliases,
45 base_flags,
45 base_flags,
46 )
46 )
47 from IPython.utils.importstring import import_item
47 from IPython.utils.importstring import import_item
48 from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict
48 from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict
49
49
50 # from IPython.parallel.controller.controller import ControllerFactory
50 # from IPython.parallel.controller.controller import ControllerFactory
51 from IPython.zmq.session import Session
51 from IPython.zmq.session import Session
52 from IPython.parallel.controller.heartmonitor import HeartMonitor
52 from IPython.parallel.controller.heartmonitor import HeartMonitor
53 from IPython.parallel.controller.hub import HubFactory
53 from IPython.parallel.controller.hub import HubFactory
54 from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
54 from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
55 from IPython.parallel.controller.sqlitedb import SQLiteDB
55 from IPython.parallel.controller.sqlitedb import SQLiteDB
56
56
57 from IPython.parallel.util import signal_children, split_url, asbytes
57 from IPython.parallel.util import signal_children, split_url, asbytes
58
58
59 # conditional import of MongoDB backend class
59 # conditional import of MongoDB backend class
60
60
61 try:
61 try:
62 from IPython.parallel.controller.mongodb import MongoDB
62 from IPython.parallel.controller.mongodb import MongoDB
63 except ImportError:
63 except ImportError:
64 maybe_mongo = []
64 maybe_mongo = []
65 else:
65 else:
66 maybe_mongo = [MongoDB]
66 maybe_mongo = [MongoDB]
67
67
68
68
69 #-----------------------------------------------------------------------------
69 #-----------------------------------------------------------------------------
70 # Module level variables
70 # Module level variables
71 #-----------------------------------------------------------------------------
71 #-----------------------------------------------------------------------------
72
72
73
73
74 #: The default config file name for this application
74 #: The default config file name for this application
75 default_config_file_name = u'ipcontroller_config.py'
75 default_config_file_name = u'ipcontroller_config.py'
76
76
77
77
78 _description = """Start the IPython controller for parallel computing.
78 _description = """Start the IPython controller for parallel computing.
79
79
80 The IPython controller provides a gateway between the IPython engines and
80 The IPython controller provides a gateway between the IPython engines and
81 clients. The controller needs to be started before the engines and can be
81 clients. The controller needs to be started before the engines and can be
82 configured using command line options or using a cluster directory. Cluster
82 configured using command line options or using a cluster directory. Cluster
83 directories contain config, log and security files and are usually located in
83 directories contain config, log and security files and are usually located in
84 your ipython directory and named as "profile_name". See the `profile`
84 your ipython directory and named as "profile_name". See the `profile`
85 and `profile_dir` options for details.
85 and `profile_dir` options for details.
86 """
86 """
87
87
88
88
89
89
90
90
91 #-----------------------------------------------------------------------------
91 #-----------------------------------------------------------------------------
92 # The main application
92 # The main application
93 #-----------------------------------------------------------------------------
93 #-----------------------------------------------------------------------------
94 flags = {}
94 flags = {}
95 flags.update(base_flags)
95 flags.update(base_flags)
96 flags.update({
96 flags.update({
97 'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
97 'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
98 'Use threads instead of processes for the schedulers'),
98 'Use threads instead of processes for the schedulers'),
99 'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
99 'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
100 'use the SQLiteDB backend'),
100 'use the SQLiteDB backend'),
101 'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
101 'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
102 'use the MongoDB backend'),
102 'use the MongoDB backend'),
103 'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
103 'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
104 'use the in-memory DictDB backend'),
104 'use the in-memory DictDB backend'),
105 'reuse' : ({'IPControllerApp' : {'reuse_files' : True}},
105 'reuse' : ({'IPControllerApp' : {'reuse_files' : True}},
106 'reuse existing json connection files')
106 'reuse existing json connection files')
107 })
107 })
108
108
109 flags.update(boolean_flag('secure', 'IPControllerApp.secure',
109 flags.update(boolean_flag('secure', 'IPControllerApp.secure',
110 "Use HMAC digests for authentication of messages.",
110 "Use HMAC digests for authentication of messages.",
111 "Don't authenticate messages."
111 "Don't authenticate messages."
112 ))
112 ))
113 aliases = dict(
113 aliases = dict(
114 reuse_files = 'IPControllerApp.reuse_files',
115 secure = 'IPControllerApp.secure',
114 secure = 'IPControllerApp.secure',
116 ssh = 'IPControllerApp.ssh_server',
115 ssh = 'IPControllerApp.ssh_server',
117 use_threads = 'IPControllerApp.use_threads',
118 location = 'IPControllerApp.location',
116 location = 'IPControllerApp.location',
119
117
120 ident = 'Session.session',
118 ident = 'Session.session',
121 user = 'Session.username',
119 user = 'Session.username',
122 exec_key = 'Session.keyfile',
120 keyfile = 'Session.keyfile',
123
121
124 url = 'HubFactory.url',
122 url = 'HubFactory.url',
125 ip = 'HubFactory.ip',
123 ip = 'HubFactory.ip',
126 transport = 'HubFactory.transport',
124 transport = 'HubFactory.transport',
127 port = 'HubFactory.regport',
125 port = 'HubFactory.regport',
128
126
129 ping = 'HeartMonitor.period',
127 ping = 'HeartMonitor.period',
130
128
131 scheme = 'TaskScheduler.scheme_name',
129 scheme = 'TaskScheduler.scheme_name',
132 hwm = 'TaskScheduler.hwm',
130 hwm = 'TaskScheduler.hwm',
133 )
131 )
134 aliases.update(base_aliases)
132 aliases.update(base_aliases)
135
133
136 class IPControllerApp(BaseParallelApplication):
134 class IPControllerApp(BaseParallelApplication):
137
135
138 name = u'ipcontroller'
136 name = u'ipcontroller'
139 description = _description
137 description = _description
140 config_file_name = Unicode(default_config_file_name)
138 config_file_name = Unicode(default_config_file_name)
141 classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo
139 classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo
142
140
143 # change default to True
141 # change default to True
144 auto_create = Bool(True, config=True,
142 auto_create = Bool(True, config=True,
145 help="""Whether to create profile dir if it doesn't exist.""")
143 help="""Whether to create profile dir if it doesn't exist.""")
146
144
147 reuse_files = Bool(False, config=True,
145 reuse_files = Bool(False, config=True,
148 help='Whether to reuse existing json connection files.'
146 help='Whether to reuse existing json connection files.'
149 )
147 )
150 secure = Bool(True, config=True,
148 secure = Bool(True, config=True,
151 help='Whether to use HMAC digests for extra message authentication.'
149 help='Whether to use HMAC digests for extra message authentication.'
152 )
150 )
153 ssh_server = Unicode(u'', config=True,
151 ssh_server = Unicode(u'', config=True,
154 help="""ssh url for clients to use when connecting to the Controller
152 help="""ssh url for clients to use when connecting to the Controller
155 processes. It should be of the form: [user@]server[:port]. The
153 processes. It should be of the form: [user@]server[:port]. The
156 Controller's listening addresses must be accessible from the ssh server""",
154 Controller's listening addresses must be accessible from the ssh server""",
157 )
155 )
158 location = Unicode(u'', config=True,
156 location = Unicode(u'', config=True,
159 help="""The external IP or domain name of the Controller, used for disambiguating
157 help="""The external IP or domain name of the Controller, used for disambiguating
160 engine and client connections.""",
158 engine and client connections.""",
161 )
159 )
162 import_statements = List([], config=True,
160 import_statements = List([], config=True,
163 help="import statements to be run at startup. Necessary in some environments"
161 help="import statements to be run at startup. Necessary in some environments"
164 )
162 )
165
163
166 use_threads = Bool(False, config=True,
164 use_threads = Bool(False, config=True,
167 help='Use threads instead of processes for the schedulers',
165 help='Use threads instead of processes for the schedulers',
168 )
166 )
169
167
170 # internal
168 # internal
171 children = List()
169 children = List()
172 mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')
170 mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')
173
171
174 def _use_threads_changed(self, name, old, new):
172 def _use_threads_changed(self, name, old, new):
175 self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
173 self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
176
174
177 aliases = Dict(aliases)
175 aliases = Dict(aliases)
178 flags = Dict(flags)
176 flags = Dict(flags)
179
177
180
178
181 def save_connection_dict(self, fname, cdict):
179 def save_connection_dict(self, fname, cdict):
182 """save a connection dict to json file."""
180 """save a connection dict to json file."""
183 c = self.config
181 c = self.config
184 url = cdict['url']
182 url = cdict['url']
185 location = cdict['location']
183 location = cdict['location']
186 if not location:
184 if not location:
187 try:
185 try:
188 proto,ip,port = split_url(url)
186 proto,ip,port = split_url(url)
189 except AssertionError:
187 except AssertionError:
190 pass
188 pass
191 else:
189 else:
192 location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
190 location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
193 cdict['location'] = location
191 cdict['location'] = location
194 fname = os.path.join(self.profile_dir.security_dir, fname)
192 fname = os.path.join(self.profile_dir.security_dir, fname)
195 with open(fname, 'wb') as f:
193 with open(fname, 'wb') as f:
196 f.write(json.dumps(cdict, indent=2))
194 f.write(json.dumps(cdict, indent=2))
197 os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
195 os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
198
196
199 def load_config_from_json(self):
197 def load_config_from_json(self):
200 """load config from existing json connector files."""
198 """load config from existing json connector files."""
201 c = self.config
199 c = self.config
202 # load from engine config
200 # load from engine config
203 with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-engine.json')) as f:
201 with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-engine.json')) as f:
204 cfg = json.loads(f.read())
202 cfg = json.loads(f.read())
205 key = c.Session.key = asbytes(cfg['exec_key'])
203 key = c.Session.key = asbytes(cfg['exec_key'])
206 xport,addr = cfg['url'].split('://')
204 xport,addr = cfg['url'].split('://')
207 c.HubFactory.engine_transport = xport
205 c.HubFactory.engine_transport = xport
208 ip,ports = addr.split(':')
206 ip,ports = addr.split(':')
209 c.HubFactory.engine_ip = ip
207 c.HubFactory.engine_ip = ip
210 c.HubFactory.regport = int(ports)
208 c.HubFactory.regport = int(ports)
211 self.location = cfg['location']
209 self.location = cfg['location']
212 # load client config
210 # load client config
213 with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-client.json')) as f:
211 with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-client.json')) as f:
214 cfg = json.loads(f.read())
212 cfg = json.loads(f.read())
215 assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
213 assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
216 xport,addr = cfg['url'].split('://')
214 xport,addr = cfg['url'].split('://')
217 c.HubFactory.client_transport = xport
215 c.HubFactory.client_transport = xport
218 ip,ports = addr.split(':')
216 ip,ports = addr.split(':')
219 c.HubFactory.client_ip = ip
217 c.HubFactory.client_ip = ip
220 self.ssh_server = cfg['ssh']
218 self.ssh_server = cfg['ssh']
221 assert int(ports) == c.HubFactory.regport, "regport mismatch"
219 assert int(ports) == c.HubFactory.regport, "regport mismatch"
222
220
223 def init_hub(self):
221 def init_hub(self):
224 c = self.config
222 c = self.config
225
223
226 self.do_import_statements()
224 self.do_import_statements()
227 reusing = self.reuse_files
225 reusing = self.reuse_files
228 if reusing:
226 if reusing:
229 try:
227 try:
230 self.load_config_from_json()
228 self.load_config_from_json()
231 except (AssertionError,IOError):
229 except (AssertionError,IOError):
232 reusing=False
230 reusing=False
233 # check again, because reusing may have failed:
231 # check again, because reusing may have failed:
234 if reusing:
232 if reusing:
235 pass
233 pass
236 elif self.secure:
234 elif self.secure:
237 key = str(uuid.uuid4())
235 key = str(uuid.uuid4())
238 # keyfile = os.path.join(self.profile_dir.security_dir, self.exec_key)
236 # keyfile = os.path.join(self.profile_dir.security_dir, self.exec_key)
239 # with open(keyfile, 'w') as f:
237 # with open(keyfile, 'w') as f:
240 # f.write(key)
238 # f.write(key)
241 # os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
239 # os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
242 c.Session.key = asbytes(key)
240 c.Session.key = asbytes(key)
243 else:
241 else:
244 key = c.Session.key = b''
242 key = c.Session.key = b''
245
243
246 try:
244 try:
247 self.factory = HubFactory(config=c, log=self.log)
245 self.factory = HubFactory(config=c, log=self.log)
248 # self.start_logging()
246 # self.start_logging()
249 self.factory.init_hub()
247 self.factory.init_hub()
250 except:
248 except:
251 self.log.error("Couldn't construct the Controller", exc_info=True)
249 self.log.error("Couldn't construct the Controller", exc_info=True)
252 self.exit(1)
250 self.exit(1)
253
251
254 if not reusing:
252 if not reusing:
255 # save to new json config files
253 # save to new json config files
256 f = self.factory
254 f = self.factory
257 cdict = {'exec_key' : key,
255 cdict = {'exec_key' : key,
258 'ssh' : self.ssh_server,
256 'ssh' : self.ssh_server,
259 'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
257 'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
260 'location' : self.location
258 'location' : self.location
261 }
259 }
262 self.save_connection_dict('ipcontroller-client.json', cdict)
260 self.save_connection_dict('ipcontroller-client.json', cdict)
263 edict = cdict
261 edict = cdict
264 edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
262 edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
265 self.save_connection_dict('ipcontroller-engine.json', edict)
263 self.save_connection_dict('ipcontroller-engine.json', edict)
266
264
267 #
265 #
268 def init_schedulers(self):
266 def init_schedulers(self):
269 children = self.children
267 children = self.children
270 mq = import_item(str(self.mq_class))
268 mq = import_item(str(self.mq_class))
271
269
272 hub = self.factory
270 hub = self.factory
273 # maybe_inproc = 'inproc://monitor' if self.use_threads else self.monitor_url
271 # maybe_inproc = 'inproc://monitor' if self.use_threads else self.monitor_url
274 # IOPub relay (in a Process)
272 # IOPub relay (in a Process)
275 q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub')
273 q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub')
276 q.bind_in(hub.client_info['iopub'])
274 q.bind_in(hub.client_info['iopub'])
277 q.bind_out(hub.engine_info['iopub'])
275 q.bind_out(hub.engine_info['iopub'])
278 q.setsockopt_out(zmq.SUBSCRIBE, b'')
276 q.setsockopt_out(zmq.SUBSCRIBE, b'')
279 q.connect_mon(hub.monitor_url)
277 q.connect_mon(hub.monitor_url)
280 q.daemon=True
278 q.daemon=True
281 children.append(q)
279 children.append(q)
282
280
283 # Multiplexer Queue (in a Process)
281 # Multiplexer Queue (in a Process)
284 q = mq(zmq.XREP, zmq.XREP, zmq.PUB, b'in', b'out')
282 q = mq(zmq.XREP, zmq.XREP, zmq.PUB, b'in', b'out')
285 q.bind_in(hub.client_info['mux'])
283 q.bind_in(hub.client_info['mux'])
286 q.setsockopt_in(zmq.IDENTITY, b'mux')
284 q.setsockopt_in(zmq.IDENTITY, b'mux')
287 q.bind_out(hub.engine_info['mux'])
285 q.bind_out(hub.engine_info['mux'])
288 q.connect_mon(hub.monitor_url)
286 q.connect_mon(hub.monitor_url)
289 q.daemon=True
287 q.daemon=True
290 children.append(q)
288 children.append(q)
291
289
292 # Control Queue (in a Process)
290 # Control Queue (in a Process)
293 q = mq(zmq.XREP, zmq.XREP, zmq.PUB, b'incontrol', b'outcontrol')
291 q = mq(zmq.XREP, zmq.XREP, zmq.PUB, b'incontrol', b'outcontrol')
294 q.bind_in(hub.client_info['control'])
292 q.bind_in(hub.client_info['control'])
295 q.setsockopt_in(zmq.IDENTITY, b'control')
293 q.setsockopt_in(zmq.IDENTITY, b'control')
296 q.bind_out(hub.engine_info['control'])
294 q.bind_out(hub.engine_info['control'])
297 q.connect_mon(hub.monitor_url)
295 q.connect_mon(hub.monitor_url)
298 q.daemon=True
296 q.daemon=True
299 children.append(q)
297 children.append(q)
300 try:
298 try:
301 scheme = self.config.TaskScheduler.scheme_name
299 scheme = self.config.TaskScheduler.scheme_name
302 except AttributeError:
300 except AttributeError:
303 scheme = TaskScheduler.scheme_name.get_default_value()
301 scheme = TaskScheduler.scheme_name.get_default_value()
304 # Task Queue (in a Process)
302 # Task Queue (in a Process)
305 if scheme == 'pure':
303 if scheme == 'pure':
306 self.log.warn("task::using pure XREQ Task scheduler")
304 self.log.warn("task::using pure XREQ Task scheduler")
307 q = mq(zmq.XREP, zmq.XREQ, zmq.PUB, b'intask', b'outtask')
305 q = mq(zmq.XREP, zmq.XREQ, zmq.PUB, b'intask', b'outtask')
308 # q.setsockopt_out(zmq.HWM, hub.hwm)
306 # q.setsockopt_out(zmq.HWM, hub.hwm)
309 q.bind_in(hub.client_info['task'][1])
307 q.bind_in(hub.client_info['task'][1])
310 q.setsockopt_in(zmq.IDENTITY, b'task')
308 q.setsockopt_in(zmq.IDENTITY, b'task')
311 q.bind_out(hub.engine_info['task'])
309 q.bind_out(hub.engine_info['task'])
312 q.connect_mon(hub.monitor_url)
310 q.connect_mon(hub.monitor_url)
313 q.daemon=True
311 q.daemon=True
314 children.append(q)
312 children.append(q)
315 elif scheme == 'none':
313 elif scheme == 'none':
316 self.log.warn("task::using no Task scheduler")
314 self.log.warn("task::using no Task scheduler")
317
315
318 else:
316 else:
319 self.log.info("task::using Python %s Task scheduler"%scheme)
317 self.log.info("task::using Python %s Task scheduler"%scheme)
320 sargs = (hub.client_info['task'][1], hub.engine_info['task'],
318 sargs = (hub.client_info['task'][1], hub.engine_info['task'],
321 hub.monitor_url, hub.client_info['notification'])
319 hub.monitor_url, hub.client_info['notification'])
322 kwargs = dict(logname='scheduler', loglevel=self.log_level,
320 kwargs = dict(logname='scheduler', loglevel=self.log_level,
323 log_url = self.log_url, config=dict(self.config))
321 log_url = self.log_url, config=dict(self.config))
324 if 'Process' in self.mq_class:
322 if 'Process' in self.mq_class:
325 # run the Python scheduler in a Process
323 # run the Python scheduler in a Process
326 q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
324 q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
327 q.daemon=True
325 q.daemon=True
328 children.append(q)
326 children.append(q)
329 else:
327 else:
330 # single-threaded Controller
328 # single-threaded Controller
331 kwargs['in_thread'] = True
329 kwargs['in_thread'] = True
332 launch_scheduler(*sargs, **kwargs)
330 launch_scheduler(*sargs, **kwargs)
333
331
334
332
335 def save_urls(self):
333 def save_urls(self):
336 """save the registration urls to files."""
334 """save the registration urls to files."""
337 c = self.config
335 c = self.config
338
336
339 sec_dir = self.profile_dir.security_dir
337 sec_dir = self.profile_dir.security_dir
340 cf = self.factory
338 cf = self.factory
341
339
342 with open(os.path.join(sec_dir, 'ipcontroller-engine.url'), 'w') as f:
340 with open(os.path.join(sec_dir, 'ipcontroller-engine.url'), 'w') as f:
343 f.write("%s://%s:%s"%(cf.engine_transport, cf.engine_ip, cf.regport))
341 f.write("%s://%s:%s"%(cf.engine_transport, cf.engine_ip, cf.regport))
344
342
345 with open(os.path.join(sec_dir, 'ipcontroller-client.url'), 'w') as f:
343 with open(os.path.join(sec_dir, 'ipcontroller-client.url'), 'w') as f:
346 f.write("%s://%s:%s"%(cf.client_transport, cf.client_ip, cf.regport))
344 f.write("%s://%s:%s"%(cf.client_transport, cf.client_ip, cf.regport))
347
345
348
346
349 def do_import_statements(self):
347 def do_import_statements(self):
350 statements = self.import_statements
348 statements = self.import_statements
351 for s in statements:
349 for s in statements:
352 try:
350 try:
353 self.log.msg("Executing statement: '%s'" % s)
351 self.log.msg("Executing statement: '%s'" % s)
354 exec s in globals(), locals()
352 exec s in globals(), locals()
355 except:
353 except:
356 self.log.msg("Error running statement: %s" % s)
354 self.log.msg("Error running statement: %s" % s)
357
355
358 def forward_logging(self):
356 def forward_logging(self):
359 if self.log_url:
357 if self.log_url:
360 self.log.info("Forwarding logging to %s"%self.log_url)
358 self.log.info("Forwarding logging to %s"%self.log_url)
361 context = zmq.Context.instance()
359 context = zmq.Context.instance()
362 lsock = context.socket(zmq.PUB)
360 lsock = context.socket(zmq.PUB)
363 lsock.connect(self.log_url)
361 lsock.connect(self.log_url)
364 handler = PUBHandler(lsock)
362 handler = PUBHandler(lsock)
365 self.log.removeHandler(self._log_handler)
363 self.log.removeHandler(self._log_handler)
366 handler.root_topic = 'controller'
364 handler.root_topic = 'controller'
367 handler.setLevel(self.log_level)
365 handler.setLevel(self.log_level)
368 self.log.addHandler(handler)
366 self.log.addHandler(handler)
369 self._log_handler = handler
367 self._log_handler = handler
370 # #
368 # #
371
369
372 def initialize(self, argv=None):
370 def initialize(self, argv=None):
373 super(IPControllerApp, self).initialize(argv)
371 super(IPControllerApp, self).initialize(argv)
374 self.forward_logging()
372 self.forward_logging()
375 self.init_hub()
373 self.init_hub()
376 self.init_schedulers()
374 self.init_schedulers()
377
375
378 def start(self):
376 def start(self):
379 # Start the subprocesses:
377 # Start the subprocesses:
380 self.factory.start()
378 self.factory.start()
381 child_procs = []
379 child_procs = []
382 for child in self.children:
380 for child in self.children:
383 child.start()
381 child.start()
384 if isinstance(child, ProcessMonitoredQueue):
382 if isinstance(child, ProcessMonitoredQueue):
385 child_procs.append(child.launcher)
383 child_procs.append(child.launcher)
386 elif isinstance(child, Process):
384 elif isinstance(child, Process):
387 child_procs.append(child)
385 child_procs.append(child)
388 if child_procs:
386 if child_procs:
389 signal_children(child_procs)
387 signal_children(child_procs)
390
388
391 self.write_pid_file(overwrite=True)
389 self.write_pid_file(overwrite=True)
392
390
393 try:
391 try:
394 self.factory.loop.start()
392 self.factory.loop.start()
395 except KeyboardInterrupt:
393 except KeyboardInterrupt:
396 self.log.critical("Interrupted, Exiting...\n")
394 self.log.critical("Interrupted, Exiting...\n")
397
395
398
396
399
397
400 def launch_new_instance():
398 def launch_new_instance():
401 """Create and run the IPython controller"""
399 """Create and run the IPython controller"""
402 if sys.platform == 'win32':
400 if sys.platform == 'win32':
403 # make sure we don't get called from a multiprocessing subprocess
401 # make sure we don't get called from a multiprocessing subprocess
404 # this can result in infinite Controllers being started on Windows
402 # this can result in infinite Controllers being started on Windows
405 # which doesn't have a proper fork, so multiprocessing is wonky
403 # which doesn't have a proper fork, so multiprocessing is wonky
406
404
407 # this only comes up when IPython has been installed using vanilla
405 # this only comes up when IPython has been installed using vanilla
408 # setuptools, and *not* distribute.
406 # setuptools, and *not* distribute.
409 import multiprocessing
407 import multiprocessing
410 p = multiprocessing.current_process()
408 p = multiprocessing.current_process()
411 # the main process has name 'MainProcess'
409 # the main process has name 'MainProcess'
412 # subprocesses will have names like 'Process-1'
410 # subprocesses will have names like 'Process-1'
413 if p.name != 'MainProcess':
411 if p.name != 'MainProcess':
414 # we are a subprocess, don't start another Controller!
412 # we are a subprocess, don't start another Controller!
415 return
413 return
416 app = IPControllerApp.instance()
414 app = IPControllerApp.instance()
417 app.initialize()
415 app.initialize()
418 app.start()
416 app.start()
419
417
420
418
421 if __name__ == '__main__':
419 if __name__ == '__main__':
422 launch_new_instance()
420 launch_new_instance()
@@ -1,301 +1,301 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The IPython engine application
4 The IPython engine application
5
5
6 Authors:
6 Authors:
7
7
8 * Brian Granger
8 * Brian Granger
9 * MinRK
9 * MinRK
10
10
11 """
11 """
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Copyright (C) 2008-2011 The IPython Development Team
14 # Copyright (C) 2008-2011 The IPython Development Team
15 #
15 #
16 # Distributed under the terms of the BSD License. The full license is in
16 # Distributed under the terms of the BSD License. The full license is in
17 # the file COPYING, distributed as part of this software.
17 # the file COPYING, distributed as part of this software.
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Imports
21 # Imports
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 import json
24 import json
25 import os
25 import os
26 import sys
26 import sys
27 import time
27 import time
28
28
29 import zmq
29 import zmq
30 from zmq.eventloop import ioloop
30 from zmq.eventloop import ioloop
31
31
32 from IPython.core.profiledir import ProfileDir
32 from IPython.core.profiledir import ProfileDir
33 from IPython.parallel.apps.baseapp import (
33 from IPython.parallel.apps.baseapp import (
34 BaseParallelApplication,
34 BaseParallelApplication,
35 base_aliases,
35 base_aliases,
36 base_flags,
36 base_flags,
37 )
37 )
38 from IPython.zmq.log import EnginePUBHandler
38 from IPython.zmq.log import EnginePUBHandler
39
39
40 from IPython.config.configurable import Configurable
40 from IPython.config.configurable import Configurable
41 from IPython.zmq.session import Session
41 from IPython.zmq.session import Session
42 from IPython.parallel.engine.engine import EngineFactory
42 from IPython.parallel.engine.engine import EngineFactory
43 from IPython.parallel.engine.streamkernel import Kernel
43 from IPython.parallel.engine.streamkernel import Kernel
44 from IPython.parallel.util import disambiguate_url, asbytes
44 from IPython.parallel.util import disambiguate_url, asbytes
45
45
46 from IPython.utils.importstring import import_item
46 from IPython.utils.importstring import import_item
47 from IPython.utils.traitlets import Bool, Unicode, Dict, List, Float
47 from IPython.utils.traitlets import Bool, Unicode, Dict, List, Float
48
48
49
49
50 #-----------------------------------------------------------------------------
50 #-----------------------------------------------------------------------------
51 # Module level variables
51 # Module level variables
52 #-----------------------------------------------------------------------------
52 #-----------------------------------------------------------------------------
53
53
54 #: The default config file name for this application
54 #: The default config file name for this application
55 default_config_file_name = u'ipengine_config.py'
55 default_config_file_name = u'ipengine_config.py'
56
56
57 _description = """Start an IPython engine for parallel computing.
57 _description = """Start an IPython engine for parallel computing.
58
58
59 IPython engines run in parallel and perform computations on behalf of a client
59 IPython engines run in parallel and perform computations on behalf of a client
60 and controller. A controller needs to be started before the engines. The
60 and controller. A controller needs to be started before the engines. The
61 engine can be configured using command line options or using a cluster
61 engine can be configured using command line options or using a cluster
62 directory. Cluster directories contain config, log and security files and are
62 directory. Cluster directories contain config, log and security files and are
63 usually located in your ipython directory and named as "profile_name".
63 usually located in your ipython directory and named as "profile_name".
64 See the `profile` and `profile_dir` options for details.
64 See the `profile` and `profile_dir` options for details.
65 """
65 """
66
66
67
67
68 #-----------------------------------------------------------------------------
68 #-----------------------------------------------------------------------------
69 # MPI configuration
69 # MPI configuration
70 #-----------------------------------------------------------------------------
70 #-----------------------------------------------------------------------------
71
71
72 mpi4py_init = """from mpi4py import MPI as mpi
72 mpi4py_init = """from mpi4py import MPI as mpi
73 mpi.size = mpi.COMM_WORLD.Get_size()
73 mpi.size = mpi.COMM_WORLD.Get_size()
74 mpi.rank = mpi.COMM_WORLD.Get_rank()
74 mpi.rank = mpi.COMM_WORLD.Get_rank()
75 """
75 """
76
76
77
77
78 pytrilinos_init = """from PyTrilinos import Epetra
78 pytrilinos_init = """from PyTrilinos import Epetra
79 class SimpleStruct:
79 class SimpleStruct:
80 pass
80 pass
81 mpi = SimpleStruct()
81 mpi = SimpleStruct()
82 mpi.rank = 0
82 mpi.rank = 0
83 mpi.size = 0
83 mpi.size = 0
84 """
84 """
85
85
86 class MPI(Configurable):
86 class MPI(Configurable):
87 """Configurable for MPI initialization"""
87 """Configurable for MPI initialization"""
88 use = Unicode('', config=True,
88 use = Unicode('', config=True,
89 help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).'
89 help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).'
90 )
90 )
91
91
92 def _on_use_changed(self, old, new):
92 def _on_use_changed(self, old, new):
93 # load default init script if it's not set
93 # load default init script if it's not set
94 if not self.init_script:
94 if not self.init_script:
95 self.init_script = self.default_inits.get(new, '')
95 self.init_script = self.default_inits.get(new, '')
96
96
97 init_script = Unicode('', config=True,
97 init_script = Unicode('', config=True,
98 help="Initialization code for MPI")
98 help="Initialization code for MPI")
99
99
100 default_inits = Dict({'mpi4py' : mpi4py_init, 'pytrilinos':pytrilinos_init},
100 default_inits = Dict({'mpi4py' : mpi4py_init, 'pytrilinos':pytrilinos_init},
101 config=True)
101 config=True)
102
102
103
103
104 #-----------------------------------------------------------------------------
104 #-----------------------------------------------------------------------------
105 # Main application
105 # Main application
106 #-----------------------------------------------------------------------------
106 #-----------------------------------------------------------------------------
107 aliases = dict(
107 aliases = dict(
108 file = 'IPEngineApp.url_file',
108 file = 'IPEngineApp.url_file',
109 c = 'IPEngineApp.startup_command',
109 c = 'IPEngineApp.startup_command',
110 s = 'IPEngineApp.startup_script',
110 s = 'IPEngineApp.startup_script',
111
111
112 ident = 'Session.session',
112 ident = 'Session.session',
113 user = 'Session.username',
113 user = 'Session.username',
114 exec_key = 'Session.keyfile',
114 keyfile = 'Session.keyfile',
115
115
116 url = 'EngineFactory.url',
116 url = 'EngineFactory.url',
117 ip = 'EngineFactory.ip',
117 ip = 'EngineFactory.ip',
118 transport = 'EngineFactory.transport',
118 transport = 'EngineFactory.transport',
119 port = 'EngineFactory.regport',
119 port = 'EngineFactory.regport',
120 location = 'EngineFactory.location',
120 location = 'EngineFactory.location',
121
121
122 timeout = 'EngineFactory.timeout',
122 timeout = 'EngineFactory.timeout',
123
123
124 mpi = 'MPI.use',
124 mpi = 'MPI.use',
125
125
126 )
126 )
127 aliases.update(base_aliases)
127 aliases.update(base_aliases)
128
128
129 class IPEngineApp(BaseParallelApplication):
129 class IPEngineApp(BaseParallelApplication):
130
130
131 name = Unicode(u'ipengine')
131 name = Unicode(u'ipengine')
132 description = Unicode(_description)
132 description = Unicode(_description)
133 config_file_name = Unicode(default_config_file_name)
133 config_file_name = Unicode(default_config_file_name)
134 classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI])
134 classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI])
135
135
136 startup_script = Unicode(u'', config=True,
136 startup_script = Unicode(u'', config=True,
137 help='specify a script to be run at startup')
137 help='specify a script to be run at startup')
138 startup_command = Unicode('', config=True,
138 startup_command = Unicode('', config=True,
139 help='specify a command to be run at startup')
139 help='specify a command to be run at startup')
140
140
141 url_file = Unicode(u'', config=True,
141 url_file = Unicode(u'', config=True,
142 help="""The full location of the file containing the connection information for
142 help="""The full location of the file containing the connection information for
143 the controller. If this is not given, the file must be in the
143 the controller. If this is not given, the file must be in the
144 security directory of the cluster directory. This location is
144 security directory of the cluster directory. This location is
145 resolved using the `profile` or `profile_dir` options.""",
145 resolved using the `profile` or `profile_dir` options.""",
146 )
146 )
147 wait_for_url_file = Float(5, config=True,
147 wait_for_url_file = Float(5, config=True,
148 help="""The maximum number of seconds to wait for url_file to exist.
148 help="""The maximum number of seconds to wait for url_file to exist.
149 This is useful for batch-systems and shared-filesystems where the
149 This is useful for batch-systems and shared-filesystems where the
150 controller and engine are started at the same time and it
150 controller and engine are started at the same time and it
151 may take a moment for the controller to write the connector files.""")
151 may take a moment for the controller to write the connector files.""")
152
152
153 url_file_name = Unicode(u'ipcontroller-engine.json')
153 url_file_name = Unicode(u'ipcontroller-engine.json')
154 log_url = Unicode('', config=True,
154 log_url = Unicode('', config=True,
155 help="""The URL for the iploggerapp instance, for forwarding
155 help="""The URL for the iploggerapp instance, for forwarding
156 logging to a central location.""")
156 logging to a central location.""")
157
157
158 aliases = Dict(aliases)
158 aliases = Dict(aliases)
159
159
160 # def find_key_file(self):
160 # def find_key_file(self):
161 # """Set the key file.
161 # """Set the key file.
162 #
162 #
163 # Here we don't try to actually see if it exists for is valid as that
163 # Here we don't try to actually see if it exists for is valid as that
164 # is hadled by the connection logic.
164 # is hadled by the connection logic.
165 # """
165 # """
166 # config = self.master_config
166 # config = self.master_config
167 # # Find the actual controller key file
167 # # Find the actual controller key file
168 # if not config.Global.key_file:
168 # if not config.Global.key_file:
169 # try_this = os.path.join(
169 # try_this = os.path.join(
170 # config.Global.profile_dir,
170 # config.Global.profile_dir,
171 # config.Global.security_dir,
171 # config.Global.security_dir,
172 # config.Global.key_file_name
172 # config.Global.key_file_name
173 # )
173 # )
174 # config.Global.key_file = try_this
174 # config.Global.key_file = try_this
175
175
176 def find_url_file(self):
176 def find_url_file(self):
177 """Set the url file.
177 """Set the url file.
178
178
179 Here we don't try to actually see if it exists for is valid as that
179 Here we don't try to actually see if it exists for is valid as that
180 is hadled by the connection logic.
180 is hadled by the connection logic.
181 """
181 """
182 config = self.config
182 config = self.config
183 # Find the actual controller key file
183 # Find the actual controller key file
184 if not self.url_file:
184 if not self.url_file:
185 self.url_file = os.path.join(
185 self.url_file = os.path.join(
186 self.profile_dir.security_dir,
186 self.profile_dir.security_dir,
187 self.url_file_name
187 self.url_file_name
188 )
188 )
189 def init_engine(self):
189 def init_engine(self):
190 # This is the working dir by now.
190 # This is the working dir by now.
191 sys.path.insert(0, '')
191 sys.path.insert(0, '')
192 config = self.config
192 config = self.config
193 # print config
193 # print config
194 self.find_url_file()
194 self.find_url_file()
195
195
196 # was the url manually specified?
196 # was the url manually specified?
197 keys = set(self.config.EngineFactory.keys())
197 keys = set(self.config.EngineFactory.keys())
198 keys = keys.union(set(self.config.RegistrationFactory.keys()))
198 keys = keys.union(set(self.config.RegistrationFactory.keys()))
199
199
200 if keys.intersection(set(['ip', 'url', 'port'])):
200 if keys.intersection(set(['ip', 'url', 'port'])):
201 # Connection info was specified, don't wait for the file
201 # Connection info was specified, don't wait for the file
202 url_specified = True
202 url_specified = True
203 self.wait_for_url_file = 0
203 self.wait_for_url_file = 0
204 else:
204 else:
205 url_specified = False
205 url_specified = False
206
206
207 if self.wait_for_url_file and not os.path.exists(self.url_file):
207 if self.wait_for_url_file and not os.path.exists(self.url_file):
208 self.log.warn("url_file %r not found"%self.url_file)
208 self.log.warn("url_file %r not found"%self.url_file)
209 self.log.warn("Waiting up to %.1f seconds for it to arrive."%self.wait_for_url_file)
209 self.log.warn("Waiting up to %.1f seconds for it to arrive."%self.wait_for_url_file)
210 tic = time.time()
210 tic = time.time()
211 while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file):
211 while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file):
212 # wait for url_file to exist, for up to 10 seconds
212 # wait for url_file to exist, for up to 10 seconds
213 time.sleep(0.1)
213 time.sleep(0.1)
214
214
215 if os.path.exists(self.url_file):
215 if os.path.exists(self.url_file):
216 self.log.info("Loading url_file %r"%self.url_file)
216 self.log.info("Loading url_file %r"%self.url_file)
217 with open(self.url_file) as f:
217 with open(self.url_file) as f:
218 d = json.loads(f.read())
218 d = json.loads(f.read())
219 if d['exec_key']:
219 if d['exec_key']:
220 config.Session.key = asbytes(d['exec_key'])
220 config.Session.key = asbytes(d['exec_key'])
221 d['url'] = disambiguate_url(d['url'], d['location'])
221 d['url'] = disambiguate_url(d['url'], d['location'])
222 config.EngineFactory.url = d['url']
222 config.EngineFactory.url = d['url']
223 config.EngineFactory.location = d['location']
223 config.EngineFactory.location = d['location']
224 elif not url_specified:
224 elif not url_specified:
225 self.log.critical("Fatal: url file never arrived: %s"%self.url_file)
225 self.log.critical("Fatal: url file never arrived: %s"%self.url_file)
226 self.exit(1)
226 self.exit(1)
227
227
228
228
229 try:
229 try:
230 exec_lines = config.Kernel.exec_lines
230 exec_lines = config.Kernel.exec_lines
231 except AttributeError:
231 except AttributeError:
232 config.Kernel.exec_lines = []
232 config.Kernel.exec_lines = []
233 exec_lines = config.Kernel.exec_lines
233 exec_lines = config.Kernel.exec_lines
234
234
235 if self.startup_script:
235 if self.startup_script:
236 enc = sys.getfilesystemencoding() or 'utf8'
236 enc = sys.getfilesystemencoding() or 'utf8'
237 cmd="execfile(%r)"%self.startup_script.encode(enc)
237 cmd="execfile(%r)"%self.startup_script.encode(enc)
238 exec_lines.append(cmd)
238 exec_lines.append(cmd)
239 if self.startup_command:
239 if self.startup_command:
240 exec_lines.append(self.startup_command)
240 exec_lines.append(self.startup_command)
241
241
242 # Create the underlying shell class and Engine
242 # Create the underlying shell class and Engine
243 # shell_class = import_item(self.master_config.Global.shell_class)
243 # shell_class = import_item(self.master_config.Global.shell_class)
244 # print self.config
244 # print self.config
245 try:
245 try:
246 self.engine = EngineFactory(config=config, log=self.log)
246 self.engine = EngineFactory(config=config, log=self.log)
247 except:
247 except:
248 self.log.error("Couldn't start the Engine", exc_info=True)
248 self.log.error("Couldn't start the Engine", exc_info=True)
249 self.exit(1)
249 self.exit(1)
250
250
251 def forward_logging(self):
251 def forward_logging(self):
252 if self.log_url:
252 if self.log_url:
253 self.log.info("Forwarding logging to %s"%self.log_url)
253 self.log.info("Forwarding logging to %s"%self.log_url)
254 context = self.engine.context
254 context = self.engine.context
255 lsock = context.socket(zmq.PUB)
255 lsock = context.socket(zmq.PUB)
256 lsock.connect(self.log_url)
256 lsock.connect(self.log_url)
257 self.log.removeHandler(self._log_handler)
257 self.log.removeHandler(self._log_handler)
258 handler = EnginePUBHandler(self.engine, lsock)
258 handler = EnginePUBHandler(self.engine, lsock)
259 handler.setLevel(self.log_level)
259 handler.setLevel(self.log_level)
260 self.log.addHandler(handler)
260 self.log.addHandler(handler)
261 self._log_handler = handler
261 self._log_handler = handler
262 #
262 #
263 def init_mpi(self):
263 def init_mpi(self):
264 global mpi
264 global mpi
265 self.mpi = MPI(config=self.config)
265 self.mpi = MPI(config=self.config)
266
266
267 mpi_import_statement = self.mpi.init_script
267 mpi_import_statement = self.mpi.init_script
268 if mpi_import_statement:
268 if mpi_import_statement:
269 try:
269 try:
270 self.log.info("Initializing MPI:")
270 self.log.info("Initializing MPI:")
271 self.log.info(mpi_import_statement)
271 self.log.info(mpi_import_statement)
272 exec mpi_import_statement in globals()
272 exec mpi_import_statement in globals()
273 except:
273 except:
274 mpi = None
274 mpi = None
275 else:
275 else:
276 mpi = None
276 mpi = None
277
277
278 def initialize(self, argv=None):
278 def initialize(self, argv=None):
279 super(IPEngineApp, self).initialize(argv)
279 super(IPEngineApp, self).initialize(argv)
280 self.init_mpi()
280 self.init_mpi()
281 self.init_engine()
281 self.init_engine()
282 self.forward_logging()
282 self.forward_logging()
283
283
284 def start(self):
284 def start(self):
285 self.engine.start()
285 self.engine.start()
286 try:
286 try:
287 self.engine.loop.start()
287 self.engine.loop.start()
288 except KeyboardInterrupt:
288 except KeyboardInterrupt:
289 self.log.critical("Engine Interrupted, shutting down...\n")
289 self.log.critical("Engine Interrupted, shutting down...\n")
290
290
291
291
292 def launch_new_instance():
292 def launch_new_instance():
293 """Create and run the IPython engine"""
293 """Create and run the IPython engine"""
294 app = IPEngineApp.instance()
294 app = IPEngineApp.instance()
295 app.initialize()
295 app.initialize()
296 app.start()
296 app.start()
297
297
298
298
299 if __name__ == '__main__':
299 if __name__ == '__main__':
300 launch_new_instance()
300 launch_new_instance()
301
301
@@ -1,1065 +1,1065 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 Facilities for launching IPython processes asynchronously.
4 Facilities for launching IPython processes asynchronously.
5
5
6 Authors:
6 Authors:
7
7
8 * Brian Granger
8 * Brian Granger
9 * MinRK
9 * MinRK
10 """
10 """
11
11
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13 # Copyright (C) 2008-2011 The IPython Development Team
13 # Copyright (C) 2008-2011 The IPython Development Team
14 #
14 #
15 # Distributed under the terms of the BSD License. The full license is in
15 # Distributed under the terms of the BSD License. The full license is in
16 # the file COPYING, distributed as part of this software.
16 # the file COPYING, distributed as part of this software.
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20 # Imports
20 # Imports
21 #-----------------------------------------------------------------------------
21 #-----------------------------------------------------------------------------
22
22
23 import copy
23 import copy
24 import logging
24 import logging
25 import os
25 import os
26 import re
26 import re
27 import stat
27 import stat
28
28
29 # signal imports, handling various platforms, versions
29 # signal imports, handling various platforms, versions
30
30
31 from signal import SIGINT, SIGTERM
31 from signal import SIGINT, SIGTERM
32 try:
32 try:
33 from signal import SIGKILL
33 from signal import SIGKILL
34 except ImportError:
34 except ImportError:
35 # Windows
35 # Windows
36 SIGKILL=SIGTERM
36 SIGKILL=SIGTERM
37
37
38 try:
38 try:
39 # Windows >= 2.7, 3.2
39 # Windows >= 2.7, 3.2
40 from signal import CTRL_C_EVENT as SIGINT
40 from signal import CTRL_C_EVENT as SIGINT
41 except ImportError:
41 except ImportError:
42 pass
42 pass
43
43
44 from subprocess import Popen, PIPE, STDOUT
44 from subprocess import Popen, PIPE, STDOUT
45 try:
45 try:
46 from subprocess import check_output
46 from subprocess import check_output
47 except ImportError:
47 except ImportError:
48 # pre-2.7, define check_output with Popen
48 # pre-2.7, define check_output with Popen
49 def check_output(*args, **kwargs):
49 def check_output(*args, **kwargs):
50 kwargs.update(dict(stdout=PIPE))
50 kwargs.update(dict(stdout=PIPE))
51 p = Popen(*args, **kwargs)
51 p = Popen(*args, **kwargs)
52 out,err = p.communicate()
52 out,err = p.communicate()
53 return out
53 return out
54
54
55 from zmq.eventloop import ioloop
55 from zmq.eventloop import ioloop
56
56
57 from IPython.config.application import Application
57 from IPython.config.application import Application
58 from IPython.config.configurable import LoggingConfigurable
58 from IPython.config.configurable import LoggingConfigurable
59 from IPython.utils.text import EvalFormatter
59 from IPython.utils.text import EvalFormatter
60 from IPython.utils.traitlets import Any, Int, List, Unicode, Dict, Instance
60 from IPython.utils.traitlets import Any, Int, List, Unicode, Dict, Instance
61 from IPython.utils.path import get_ipython_module_path
61 from IPython.utils.path import get_ipython_module_path
62 from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError
62 from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError
63
63
64 from .win32support import forward_read_events
64 from .win32support import forward_read_events
65
65
66 from .winhpcjob import IPControllerTask, IPEngineTask, IPControllerJob, IPEngineSetJob
66 from .winhpcjob import IPControllerTask, IPEngineTask, IPControllerJob, IPEngineSetJob
67
67
68 WINDOWS = os.name == 'nt'
68 WINDOWS = os.name == 'nt'
69
69
70 #-----------------------------------------------------------------------------
70 #-----------------------------------------------------------------------------
71 # Paths to the kernel apps
71 # Paths to the kernel apps
72 #-----------------------------------------------------------------------------
72 #-----------------------------------------------------------------------------
73
73
74
74
75 ipcluster_cmd_argv = pycmd2argv(get_ipython_module_path(
75 ipcluster_cmd_argv = pycmd2argv(get_ipython_module_path(
76 'IPython.parallel.apps.ipclusterapp'
76 'IPython.parallel.apps.ipclusterapp'
77 ))
77 ))
78
78
79 ipengine_cmd_argv = pycmd2argv(get_ipython_module_path(
79 ipengine_cmd_argv = pycmd2argv(get_ipython_module_path(
80 'IPython.parallel.apps.ipengineapp'
80 'IPython.parallel.apps.ipengineapp'
81 ))
81 ))
82
82
83 ipcontroller_cmd_argv = pycmd2argv(get_ipython_module_path(
83 ipcontroller_cmd_argv = pycmd2argv(get_ipython_module_path(
84 'IPython.parallel.apps.ipcontrollerapp'
84 'IPython.parallel.apps.ipcontrollerapp'
85 ))
85 ))
86
86
87 #-----------------------------------------------------------------------------
87 #-----------------------------------------------------------------------------
88 # Base launchers and errors
88 # Base launchers and errors
89 #-----------------------------------------------------------------------------
89 #-----------------------------------------------------------------------------
90
90
91
91
92 class LauncherError(Exception):
92 class LauncherError(Exception):
93 pass
93 pass
94
94
95
95
96 class ProcessStateError(LauncherError):
96 class ProcessStateError(LauncherError):
97 pass
97 pass
98
98
99
99
100 class UnknownStatus(LauncherError):
100 class UnknownStatus(LauncherError):
101 pass
101 pass
102
102
103
103
104 class BaseLauncher(LoggingConfigurable):
104 class BaseLauncher(LoggingConfigurable):
105 """An asbtraction for starting, stopping and signaling a process."""
105 """An asbtraction for starting, stopping and signaling a process."""
106
106
107 # In all of the launchers, the work_dir is where child processes will be
107 # In all of the launchers, the work_dir is where child processes will be
108 # run. This will usually be the profile_dir, but may not be. any work_dir
108 # run. This will usually be the profile_dir, but may not be. any work_dir
109 # passed into the __init__ method will override the config value.
109 # passed into the __init__ method will override the config value.
110 # This should not be used to set the work_dir for the actual engine
110 # This should not be used to set the work_dir for the actual engine
111 # and controller. Instead, use their own config files or the
111 # and controller. Instead, use their own config files or the
112 # controller_args, engine_args attributes of the launchers to add
112 # controller_args, engine_args attributes of the launchers to add
113 # the work_dir option.
113 # the work_dir option.
114 work_dir = Unicode(u'.')
114 work_dir = Unicode(u'.')
115 loop = Instance('zmq.eventloop.ioloop.IOLoop')
115 loop = Instance('zmq.eventloop.ioloop.IOLoop')
116
116
117 start_data = Any()
117 start_data = Any()
118 stop_data = Any()
118 stop_data = Any()
119
119
120 def _loop_default(self):
120 def _loop_default(self):
121 return ioloop.IOLoop.instance()
121 return ioloop.IOLoop.instance()
122
122
123 def __init__(self, work_dir=u'.', config=None, **kwargs):
123 def __init__(self, work_dir=u'.', config=None, **kwargs):
124 super(BaseLauncher, self).__init__(work_dir=work_dir, config=config, **kwargs)
124 super(BaseLauncher, self).__init__(work_dir=work_dir, config=config, **kwargs)
125 self.state = 'before' # can be before, running, after
125 self.state = 'before' # can be before, running, after
126 self.stop_callbacks = []
126 self.stop_callbacks = []
127 self.start_data = None
127 self.start_data = None
128 self.stop_data = None
128 self.stop_data = None
129
129
130 @property
130 @property
131 def args(self):
131 def args(self):
132 """A list of cmd and args that will be used to start the process.
132 """A list of cmd and args that will be used to start the process.
133
133
134 This is what is passed to :func:`spawnProcess` and the first element
134 This is what is passed to :func:`spawnProcess` and the first element
135 will be the process name.
135 will be the process name.
136 """
136 """
137 return self.find_args()
137 return self.find_args()
138
138
139 def find_args(self):
139 def find_args(self):
140 """The ``.args`` property calls this to find the args list.
140 """The ``.args`` property calls this to find the args list.
141
141
142 Subcommand should implement this to construct the cmd and args.
142 Subcommand should implement this to construct the cmd and args.
143 """
143 """
144 raise NotImplementedError('find_args must be implemented in a subclass')
144 raise NotImplementedError('find_args must be implemented in a subclass')
145
145
146 @property
146 @property
147 def arg_str(self):
147 def arg_str(self):
148 """The string form of the program arguments."""
148 """The string form of the program arguments."""
149 return ' '.join(self.args)
149 return ' '.join(self.args)
150
150
151 @property
151 @property
152 def running(self):
152 def running(self):
153 """Am I running."""
153 """Am I running."""
154 if self.state == 'running':
154 if self.state == 'running':
155 return True
155 return True
156 else:
156 else:
157 return False
157 return False
158
158
159 def start(self):
159 def start(self):
160 """Start the process."""
160 """Start the process."""
161 raise NotImplementedError('start must be implemented in a subclass')
161 raise NotImplementedError('start must be implemented in a subclass')
162
162
163 def stop(self):
163 def stop(self):
164 """Stop the process and notify observers of stopping.
164 """Stop the process and notify observers of stopping.
165
165
166 This method will return None immediately.
166 This method will return None immediately.
167 To observe the actual process stopping, see :meth:`on_stop`.
167 To observe the actual process stopping, see :meth:`on_stop`.
168 """
168 """
169 raise NotImplementedError('stop must be implemented in a subclass')
169 raise NotImplementedError('stop must be implemented in a subclass')
170
170
171 def on_stop(self, f):
171 def on_stop(self, f):
172 """Register a callback to be called with this Launcher's stop_data
172 """Register a callback to be called with this Launcher's stop_data
173 when the process actually finishes.
173 when the process actually finishes.
174 """
174 """
175 if self.state=='after':
175 if self.state=='after':
176 return f(self.stop_data)
176 return f(self.stop_data)
177 else:
177 else:
178 self.stop_callbacks.append(f)
178 self.stop_callbacks.append(f)
179
179
180 def notify_start(self, data):
180 def notify_start(self, data):
181 """Call this to trigger startup actions.
181 """Call this to trigger startup actions.
182
182
183 This logs the process startup and sets the state to 'running'. It is
183 This logs the process startup and sets the state to 'running'. It is
184 a pass-through so it can be used as a callback.
184 a pass-through so it can be used as a callback.
185 """
185 """
186
186
187 self.log.info('Process %r started: %r' % (self.args[0], data))
187 self.log.info('Process %r started: %r' % (self.args[0], data))
188 self.start_data = data
188 self.start_data = data
189 self.state = 'running'
189 self.state = 'running'
190 return data
190 return data
191
191
192 def notify_stop(self, data):
192 def notify_stop(self, data):
193 """Call this to trigger process stop actions.
193 """Call this to trigger process stop actions.
194
194
195 This logs the process stopping and sets the state to 'after'. Call
195 This logs the process stopping and sets the state to 'after'. Call
196 this to trigger callbacks registered via :meth:`on_stop`."""
196 this to trigger callbacks registered via :meth:`on_stop`."""
197
197
198 self.log.info('Process %r stopped: %r' % (self.args[0], data))
198 self.log.info('Process %r stopped: %r' % (self.args[0], data))
199 self.stop_data = data
199 self.stop_data = data
200 self.state = 'after'
200 self.state = 'after'
201 for i in range(len(self.stop_callbacks)):
201 for i in range(len(self.stop_callbacks)):
202 d = self.stop_callbacks.pop()
202 d = self.stop_callbacks.pop()
203 d(data)
203 d(data)
204 return data
204 return data
205
205
206 def signal(self, sig):
206 def signal(self, sig):
207 """Signal the process.
207 """Signal the process.
208
208
209 Parameters
209 Parameters
210 ----------
210 ----------
211 sig : str or int
211 sig : str or int
212 'KILL', 'INT', etc., or any signal number
212 'KILL', 'INT', etc., or any signal number
213 """
213 """
214 raise NotImplementedError('signal must be implemented in a subclass')
214 raise NotImplementedError('signal must be implemented in a subclass')
215
215
216
216
217 #-----------------------------------------------------------------------------
217 #-----------------------------------------------------------------------------
218 # Local process launchers
218 # Local process launchers
219 #-----------------------------------------------------------------------------
219 #-----------------------------------------------------------------------------
220
220
221
221
222 class LocalProcessLauncher(BaseLauncher):
222 class LocalProcessLauncher(BaseLauncher):
223 """Start and stop an external process in an asynchronous manner.
223 """Start and stop an external process in an asynchronous manner.
224
224
225 This will launch the external process with a working directory of
225 This will launch the external process with a working directory of
226 ``self.work_dir``.
226 ``self.work_dir``.
227 """
227 """
228
228
229 # This is used to to construct self.args, which is passed to
229 # This is used to to construct self.args, which is passed to
230 # spawnProcess.
230 # spawnProcess.
231 cmd_and_args = List([])
231 cmd_and_args = List([])
232 poll_frequency = Int(100) # in ms
232 poll_frequency = Int(100) # in ms
233
233
234 def __init__(self, work_dir=u'.', config=None, **kwargs):
234 def __init__(self, work_dir=u'.', config=None, **kwargs):
235 super(LocalProcessLauncher, self).__init__(
235 super(LocalProcessLauncher, self).__init__(
236 work_dir=work_dir, config=config, **kwargs
236 work_dir=work_dir, config=config, **kwargs
237 )
237 )
238 self.process = None
238 self.process = None
239 self.poller = None
239 self.poller = None
240
240
241 def find_args(self):
241 def find_args(self):
242 return self.cmd_and_args
242 return self.cmd_and_args
243
243
244 def start(self):
244 def start(self):
245 if self.state == 'before':
245 if self.state == 'before':
246 self.process = Popen(self.args,
246 self.process = Popen(self.args,
247 stdout=PIPE,stderr=PIPE,stdin=PIPE,
247 stdout=PIPE,stderr=PIPE,stdin=PIPE,
248 env=os.environ,
248 env=os.environ,
249 cwd=self.work_dir
249 cwd=self.work_dir
250 )
250 )
251 if WINDOWS:
251 if WINDOWS:
252 self.stdout = forward_read_events(self.process.stdout)
252 self.stdout = forward_read_events(self.process.stdout)
253 self.stderr = forward_read_events(self.process.stderr)
253 self.stderr = forward_read_events(self.process.stderr)
254 else:
254 else:
255 self.stdout = self.process.stdout.fileno()
255 self.stdout = self.process.stdout.fileno()
256 self.stderr = self.process.stderr.fileno()
256 self.stderr = self.process.stderr.fileno()
257 self.loop.add_handler(self.stdout, self.handle_stdout, self.loop.READ)
257 self.loop.add_handler(self.stdout, self.handle_stdout, self.loop.READ)
258 self.loop.add_handler(self.stderr, self.handle_stderr, self.loop.READ)
258 self.loop.add_handler(self.stderr, self.handle_stderr, self.loop.READ)
259 self.poller = ioloop.PeriodicCallback(self.poll, self.poll_frequency, self.loop)
259 self.poller = ioloop.PeriodicCallback(self.poll, self.poll_frequency, self.loop)
260 self.poller.start()
260 self.poller.start()
261 self.notify_start(self.process.pid)
261 self.notify_start(self.process.pid)
262 else:
262 else:
263 s = 'The process was already started and has state: %r' % self.state
263 s = 'The process was already started and has state: %r' % self.state
264 raise ProcessStateError(s)
264 raise ProcessStateError(s)
265
265
266 def stop(self):
266 def stop(self):
267 return self.interrupt_then_kill()
267 return self.interrupt_then_kill()
268
268
269 def signal(self, sig):
269 def signal(self, sig):
270 if self.state == 'running':
270 if self.state == 'running':
271 if WINDOWS and sig != SIGINT:
271 if WINDOWS and sig != SIGINT:
272 # use Windows tree-kill for better child cleanup
272 # use Windows tree-kill for better child cleanup
273 check_output(['taskkill', '-pid', str(self.process.pid), '-t', '-f'])
273 check_output(['taskkill', '-pid', str(self.process.pid), '-t', '-f'])
274 else:
274 else:
275 self.process.send_signal(sig)
275 self.process.send_signal(sig)
276
276
277 def interrupt_then_kill(self, delay=2.0):
277 def interrupt_then_kill(self, delay=2.0):
278 """Send INT, wait a delay and then send KILL."""
278 """Send INT, wait a delay and then send KILL."""
279 try:
279 try:
280 self.signal(SIGINT)
280 self.signal(SIGINT)
281 except Exception:
281 except Exception:
282 self.log.debug("interrupt failed")
282 self.log.debug("interrupt failed")
283 pass
283 pass
284 self.killer = ioloop.DelayedCallback(lambda : self.signal(SIGKILL), delay*1000, self.loop)
284 self.killer = ioloop.DelayedCallback(lambda : self.signal(SIGKILL), delay*1000, self.loop)
285 self.killer.start()
285 self.killer.start()
286
286
287 # callbacks, etc:
287 # callbacks, etc:
288
288
289 def handle_stdout(self, fd, events):
289 def handle_stdout(self, fd, events):
290 if WINDOWS:
290 if WINDOWS:
291 line = self.stdout.recv()
291 line = self.stdout.recv()
292 else:
292 else:
293 line = self.process.stdout.readline()
293 line = self.process.stdout.readline()
294 # a stopped process will be readable but return empty strings
294 # a stopped process will be readable but return empty strings
295 if line:
295 if line:
296 self.log.info(line[:-1])
296 self.log.info(line[:-1])
297 else:
297 else:
298 self.poll()
298 self.poll()
299
299
300 def handle_stderr(self, fd, events):
300 def handle_stderr(self, fd, events):
301 if WINDOWS:
301 if WINDOWS:
302 line = self.stderr.recv()
302 line = self.stderr.recv()
303 else:
303 else:
304 line = self.process.stderr.readline()
304 line = self.process.stderr.readline()
305 # a stopped process will be readable but return empty strings
305 # a stopped process will be readable but return empty strings
306 if line:
306 if line:
307 self.log.error(line[:-1])
307 self.log.error(line[:-1])
308 else:
308 else:
309 self.poll()
309 self.poll()
310
310
311 def poll(self):
311 def poll(self):
312 status = self.process.poll()
312 status = self.process.poll()
313 if status is not None:
313 if status is not None:
314 self.poller.stop()
314 self.poller.stop()
315 self.loop.remove_handler(self.stdout)
315 self.loop.remove_handler(self.stdout)
316 self.loop.remove_handler(self.stderr)
316 self.loop.remove_handler(self.stderr)
317 self.notify_stop(dict(exit_code=status, pid=self.process.pid))
317 self.notify_stop(dict(exit_code=status, pid=self.process.pid))
318 return status
318 return status
319
319
320 class LocalControllerLauncher(LocalProcessLauncher):
320 class LocalControllerLauncher(LocalProcessLauncher):
321 """Launch a controller as a regular external process."""
321 """Launch a controller as a regular external process."""
322
322
323 controller_cmd = List(ipcontroller_cmd_argv, config=True,
323 controller_cmd = List(ipcontroller_cmd_argv, config=True,
324 help="""Popen command to launch ipcontroller.""")
324 help="""Popen command to launch ipcontroller.""")
325 # Command line arguments to ipcontroller.
325 # Command line arguments to ipcontroller.
326 controller_args = List(['--log-to-file','--log_level=%i'%logging.INFO], config=True,
326 controller_args = List(['--log-to-file','--log-level=%i'%logging.INFO], config=True,
327 help="""command-line args to pass to ipcontroller""")
327 help="""command-line args to pass to ipcontroller""")
328
328
329 def find_args(self):
329 def find_args(self):
330 return self.controller_cmd + self.controller_args
330 return self.controller_cmd + self.controller_args
331
331
332 def start(self, profile_dir):
332 def start(self, profile_dir):
333 """Start the controller by profile_dir."""
333 """Start the controller by profile_dir."""
334 self.controller_args.extend(['--profile_dir=%s'%profile_dir])
334 self.controller_args.extend(['--profile-dir=%s'%profile_dir])
335 self.profile_dir = unicode(profile_dir)
335 self.profile_dir = unicode(profile_dir)
336 self.log.info("Starting LocalControllerLauncher: %r" % self.args)
336 self.log.info("Starting LocalControllerLauncher: %r" % self.args)
337 return super(LocalControllerLauncher, self).start()
337 return super(LocalControllerLauncher, self).start()
338
338
339
339
340 class LocalEngineLauncher(LocalProcessLauncher):
340 class LocalEngineLauncher(LocalProcessLauncher):
341 """Launch a single engine as a regular externall process."""
341 """Launch a single engine as a regular externall process."""
342
342
343 engine_cmd = List(ipengine_cmd_argv, config=True,
343 engine_cmd = List(ipengine_cmd_argv, config=True,
344 help="""command to launch the Engine.""")
344 help="""command to launch the Engine.""")
345 # Command line arguments for ipengine.
345 # Command line arguments for ipengine.
346 engine_args = List(['--log-to-file','--log_level=%i'%logging.INFO], config=True,
346 engine_args = List(['--log-to-file','--log-level=%i'%logging.INFO], config=True,
347 help="command-line arguments to pass to ipengine"
347 help="command-line arguments to pass to ipengine"
348 )
348 )
349
349
350 def find_args(self):
350 def find_args(self):
351 return self.engine_cmd + self.engine_args
351 return self.engine_cmd + self.engine_args
352
352
353 def start(self, profile_dir):
353 def start(self, profile_dir):
354 """Start the engine by profile_dir."""
354 """Start the engine by profile_dir."""
355 self.engine_args.extend(['--profile_dir=%s'%profile_dir])
355 self.engine_args.extend(['--profile-dir=%s'%profile_dir])
356 self.profile_dir = unicode(profile_dir)
356 self.profile_dir = unicode(profile_dir)
357 return super(LocalEngineLauncher, self).start()
357 return super(LocalEngineLauncher, self).start()
358
358
359
359
360 class LocalEngineSetLauncher(BaseLauncher):
360 class LocalEngineSetLauncher(BaseLauncher):
361 """Launch a set of engines as regular external processes."""
361 """Launch a set of engines as regular external processes."""
362
362
363 # Command line arguments for ipengine.
363 # Command line arguments for ipengine.
364 engine_args = List(
364 engine_args = List(
365 ['--log-to-file','--log_level=%i'%logging.INFO], config=True,
365 ['--log-to-file','--log-level=%i'%logging.INFO], config=True,
366 help="command-line arguments to pass to ipengine"
366 help="command-line arguments to pass to ipengine"
367 )
367 )
368 # launcher class
368 # launcher class
369 launcher_class = LocalEngineLauncher
369 launcher_class = LocalEngineLauncher
370
370
371 launchers = Dict()
371 launchers = Dict()
372 stop_data = Dict()
372 stop_data = Dict()
373
373
374 def __init__(self, work_dir=u'.', config=None, **kwargs):
374 def __init__(self, work_dir=u'.', config=None, **kwargs):
375 super(LocalEngineSetLauncher, self).__init__(
375 super(LocalEngineSetLauncher, self).__init__(
376 work_dir=work_dir, config=config, **kwargs
376 work_dir=work_dir, config=config, **kwargs
377 )
377 )
378 self.stop_data = {}
378 self.stop_data = {}
379
379
380 def start(self, n, profile_dir):
380 def start(self, n, profile_dir):
381 """Start n engines by profile or profile_dir."""
381 """Start n engines by profile or profile_dir."""
382 self.profile_dir = unicode(profile_dir)
382 self.profile_dir = unicode(profile_dir)
383 dlist = []
383 dlist = []
384 for i in range(n):
384 for i in range(n):
385 el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log)
385 el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log)
386 # Copy the engine args over to each engine launcher.
386 # Copy the engine args over to each engine launcher.
387 el.engine_args = copy.deepcopy(self.engine_args)
387 el.engine_args = copy.deepcopy(self.engine_args)
388 el.on_stop(self._notice_engine_stopped)
388 el.on_stop(self._notice_engine_stopped)
389 d = el.start(profile_dir)
389 d = el.start(profile_dir)
390 if i==0:
390 if i==0:
391 self.log.info("Starting LocalEngineSetLauncher: %r" % el.args)
391 self.log.info("Starting LocalEngineSetLauncher: %r" % el.args)
392 self.launchers[i] = el
392 self.launchers[i] = el
393 dlist.append(d)
393 dlist.append(d)
394 self.notify_start(dlist)
394 self.notify_start(dlist)
395 # The consumeErrors here could be dangerous
395 # The consumeErrors here could be dangerous
396 # dfinal = gatherBoth(dlist, consumeErrors=True)
396 # dfinal = gatherBoth(dlist, consumeErrors=True)
397 # dfinal.addCallback(self.notify_start)
397 # dfinal.addCallback(self.notify_start)
398 return dlist
398 return dlist
399
399
400 def find_args(self):
400 def find_args(self):
401 return ['engine set']
401 return ['engine set']
402
402
403 def signal(self, sig):
403 def signal(self, sig):
404 dlist = []
404 dlist = []
405 for el in self.launchers.itervalues():
405 for el in self.launchers.itervalues():
406 d = el.signal(sig)
406 d = el.signal(sig)
407 dlist.append(d)
407 dlist.append(d)
408 # dfinal = gatherBoth(dlist, consumeErrors=True)
408 # dfinal = gatherBoth(dlist, consumeErrors=True)
409 return dlist
409 return dlist
410
410
411 def interrupt_then_kill(self, delay=1.0):
411 def interrupt_then_kill(self, delay=1.0):
412 dlist = []
412 dlist = []
413 for el in self.launchers.itervalues():
413 for el in self.launchers.itervalues():
414 d = el.interrupt_then_kill(delay)
414 d = el.interrupt_then_kill(delay)
415 dlist.append(d)
415 dlist.append(d)
416 # dfinal = gatherBoth(dlist, consumeErrors=True)
416 # dfinal = gatherBoth(dlist, consumeErrors=True)
417 return dlist
417 return dlist
418
418
419 def stop(self):
419 def stop(self):
420 return self.interrupt_then_kill()
420 return self.interrupt_then_kill()
421
421
422 def _notice_engine_stopped(self, data):
422 def _notice_engine_stopped(self, data):
423 pid = data['pid']
423 pid = data['pid']
424 for idx,el in self.launchers.iteritems():
424 for idx,el in self.launchers.iteritems():
425 if el.process.pid == pid:
425 if el.process.pid == pid:
426 break
426 break
427 self.launchers.pop(idx)
427 self.launchers.pop(idx)
428 self.stop_data[idx] = data
428 self.stop_data[idx] = data
429 if not self.launchers:
429 if not self.launchers:
430 self.notify_stop(self.stop_data)
430 self.notify_stop(self.stop_data)
431
431
432
432
433 #-----------------------------------------------------------------------------
433 #-----------------------------------------------------------------------------
434 # MPIExec launchers
434 # MPIExec launchers
435 #-----------------------------------------------------------------------------
435 #-----------------------------------------------------------------------------
436
436
437
437
438 class MPIExecLauncher(LocalProcessLauncher):
438 class MPIExecLauncher(LocalProcessLauncher):
439 """Launch an external process using mpiexec."""
439 """Launch an external process using mpiexec."""
440
440
441 mpi_cmd = List(['mpiexec'], config=True,
441 mpi_cmd = List(['mpiexec'], config=True,
442 help="The mpiexec command to use in starting the process."
442 help="The mpiexec command to use in starting the process."
443 )
443 )
444 mpi_args = List([], config=True,
444 mpi_args = List([], config=True,
445 help="The command line arguments to pass to mpiexec."
445 help="The command line arguments to pass to mpiexec."
446 )
446 )
447 program = List(['date'], config=True,
447 program = List(['date'], config=True,
448 help="The program to start via mpiexec.")
448 help="The program to start via mpiexec.")
449 program_args = List([], config=True,
449 program_args = List([], config=True,
450 help="The command line argument to the program."
450 help="The command line argument to the program."
451 )
451 )
452 n = Int(1)
452 n = Int(1)
453
453
454 def find_args(self):
454 def find_args(self):
455 """Build self.args using all the fields."""
455 """Build self.args using all the fields."""
456 return self.mpi_cmd + ['-n', str(self.n)] + self.mpi_args + \
456 return self.mpi_cmd + ['-n', str(self.n)] + self.mpi_args + \
457 self.program + self.program_args
457 self.program + self.program_args
458
458
459 def start(self, n):
459 def start(self, n):
460 """Start n instances of the program using mpiexec."""
460 """Start n instances of the program using mpiexec."""
461 self.n = n
461 self.n = n
462 return super(MPIExecLauncher, self).start()
462 return super(MPIExecLauncher, self).start()
463
463
464
464
465 class MPIExecControllerLauncher(MPIExecLauncher):
465 class MPIExecControllerLauncher(MPIExecLauncher):
466 """Launch a controller using mpiexec."""
466 """Launch a controller using mpiexec."""
467
467
468 controller_cmd = List(ipcontroller_cmd_argv, config=True,
468 controller_cmd = List(ipcontroller_cmd_argv, config=True,
469 help="Popen command to launch the Contropper"
469 help="Popen command to launch the Contropper"
470 )
470 )
471 controller_args = List(['--log-to-file','--log_level=%i'%logging.INFO], config=True,
471 controller_args = List(['--log-to-file','--log-level=%i'%logging.INFO], config=True,
472 help="Command line arguments to pass to ipcontroller."
472 help="Command line arguments to pass to ipcontroller."
473 )
473 )
474 n = Int(1)
474 n = Int(1)
475
475
476 def start(self, profile_dir):
476 def start(self, profile_dir):
477 """Start the controller by profile_dir."""
477 """Start the controller by profile_dir."""
478 self.controller_args.extend(['--profile_dir=%s'%profile_dir])
478 self.controller_args.extend(['--profile-dir=%s'%profile_dir])
479 self.profile_dir = unicode(profile_dir)
479 self.profile_dir = unicode(profile_dir)
480 self.log.info("Starting MPIExecControllerLauncher: %r" % self.args)
480 self.log.info("Starting MPIExecControllerLauncher: %r" % self.args)
481 return super(MPIExecControllerLauncher, self).start(1)
481 return super(MPIExecControllerLauncher, self).start(1)
482
482
483 def find_args(self):
483 def find_args(self):
484 return self.mpi_cmd + ['-n', str(self.n)] + self.mpi_args + \
484 return self.mpi_cmd + ['-n', str(self.n)] + self.mpi_args + \
485 self.controller_cmd + self.controller_args
485 self.controller_cmd + self.controller_args
486
486
487
487
488 class MPIExecEngineSetLauncher(MPIExecLauncher):
488 class MPIExecEngineSetLauncher(MPIExecLauncher):
489
489
490 program = List(ipengine_cmd_argv, config=True,
490 program = List(ipengine_cmd_argv, config=True,
491 help="Popen command for ipengine"
491 help="Popen command for ipengine"
492 )
492 )
493 program_args = List(
493 program_args = List(
494 ['--log-to-file','--log_level=%i'%logging.INFO], config=True,
494 ['--log-to-file','--log-level=%i'%logging.INFO], config=True,
495 help="Command line arguments for ipengine."
495 help="Command line arguments for ipengine."
496 )
496 )
497 n = Int(1)
497 n = Int(1)
498
498
499 def start(self, n, profile_dir):
499 def start(self, n, profile_dir):
500 """Start n engines by profile or profile_dir."""
500 """Start n engines by profile or profile_dir."""
501 self.program_args.extend(['--profile_dir=%s'%profile_dir])
501 self.program_args.extend(['--profile-dir=%s'%profile_dir])
502 self.profile_dir = unicode(profile_dir)
502 self.profile_dir = unicode(profile_dir)
503 self.n = n
503 self.n = n
504 self.log.info('Starting MPIExecEngineSetLauncher: %r' % self.args)
504 self.log.info('Starting MPIExecEngineSetLauncher: %r' % self.args)
505 return super(MPIExecEngineSetLauncher, self).start(n)
505 return super(MPIExecEngineSetLauncher, self).start(n)
506
506
507 #-----------------------------------------------------------------------------
507 #-----------------------------------------------------------------------------
508 # SSH launchers
508 # SSH launchers
509 #-----------------------------------------------------------------------------
509 #-----------------------------------------------------------------------------
510
510
511 # TODO: Get SSH Launcher back to level of sshx in 0.10.2
511 # TODO: Get SSH Launcher back to level of sshx in 0.10.2
512
512
513 class SSHLauncher(LocalProcessLauncher):
513 class SSHLauncher(LocalProcessLauncher):
514 """A minimal launcher for ssh.
514 """A minimal launcher for ssh.
515
515
516 To be useful this will probably have to be extended to use the ``sshx``
516 To be useful this will probably have to be extended to use the ``sshx``
517 idea for environment variables. There could be other things this needs
517 idea for environment variables. There could be other things this needs
518 as well.
518 as well.
519 """
519 """
520
520
521 ssh_cmd = List(['ssh'], config=True,
521 ssh_cmd = List(['ssh'], config=True,
522 help="command for starting ssh")
522 help="command for starting ssh")
523 ssh_args = List(['-tt'], config=True,
523 ssh_args = List(['-tt'], config=True,
524 help="args to pass to ssh")
524 help="args to pass to ssh")
525 program = List(['date'], config=True,
525 program = List(['date'], config=True,
526 help="Program to launch via ssh")
526 help="Program to launch via ssh")
527 program_args = List([], config=True,
527 program_args = List([], config=True,
528 help="args to pass to remote program")
528 help="args to pass to remote program")
529 hostname = Unicode('', config=True,
529 hostname = Unicode('', config=True,
530 help="hostname on which to launch the program")
530 help="hostname on which to launch the program")
531 user = Unicode('', config=True,
531 user = Unicode('', config=True,
532 help="username for ssh")
532 help="username for ssh")
533 location = Unicode('', config=True,
533 location = Unicode('', config=True,
534 help="user@hostname location for ssh in one setting")
534 help="user@hostname location for ssh in one setting")
535
535
536 def _hostname_changed(self, name, old, new):
536 def _hostname_changed(self, name, old, new):
537 if self.user:
537 if self.user:
538 self.location = u'%s@%s' % (self.user, new)
538 self.location = u'%s@%s' % (self.user, new)
539 else:
539 else:
540 self.location = new
540 self.location = new
541
541
542 def _user_changed(self, name, old, new):
542 def _user_changed(self, name, old, new):
543 self.location = u'%s@%s' % (new, self.hostname)
543 self.location = u'%s@%s' % (new, self.hostname)
544
544
545 def find_args(self):
545 def find_args(self):
546 return self.ssh_cmd + self.ssh_args + [self.location] + \
546 return self.ssh_cmd + self.ssh_args + [self.location] + \
547 self.program + self.program_args
547 self.program + self.program_args
548
548
549 def start(self, profile_dir, hostname=None, user=None):
549 def start(self, profile_dir, hostname=None, user=None):
550 self.profile_dir = unicode(profile_dir)
550 self.profile_dir = unicode(profile_dir)
551 if hostname is not None:
551 if hostname is not None:
552 self.hostname = hostname
552 self.hostname = hostname
553 if user is not None:
553 if user is not None:
554 self.user = user
554 self.user = user
555
555
556 return super(SSHLauncher, self).start()
556 return super(SSHLauncher, self).start()
557
557
558 def signal(self, sig):
558 def signal(self, sig):
559 if self.state == 'running':
559 if self.state == 'running':
560 # send escaped ssh connection-closer
560 # send escaped ssh connection-closer
561 self.process.stdin.write('~.')
561 self.process.stdin.write('~.')
562 self.process.stdin.flush()
562 self.process.stdin.flush()
563
563
564
564
565
565
566 class SSHControllerLauncher(SSHLauncher):
566 class SSHControllerLauncher(SSHLauncher):
567
567
568 program = List(ipcontroller_cmd_argv, config=True,
568 program = List(ipcontroller_cmd_argv, config=True,
569 help="remote ipcontroller command.")
569 help="remote ipcontroller command.")
570 program_args = List(['--reuse-files', '--log-to-file','--log_level=%i'%logging.INFO], config=True,
570 program_args = List(['--reuse-files', '--log-to-file','--log-level=%i'%logging.INFO], config=True,
571 help="Command line arguments to ipcontroller.")
571 help="Command line arguments to ipcontroller.")
572
572
573
573
574 class SSHEngineLauncher(SSHLauncher):
574 class SSHEngineLauncher(SSHLauncher):
575 program = List(ipengine_cmd_argv, config=True,
575 program = List(ipengine_cmd_argv, config=True,
576 help="remote ipengine command.")
576 help="remote ipengine command.")
577 # Command line arguments for ipengine.
577 # Command line arguments for ipengine.
578 program_args = List(
578 program_args = List(
579 ['--log-to-file','log_level=%i'%logging.INFO], config=True,
579 ['--log-to-file','log_level=%i'%logging.INFO], config=True,
580 help="Command line arguments to ipengine."
580 help="Command line arguments to ipengine."
581 )
581 )
582
582
583 class SSHEngineSetLauncher(LocalEngineSetLauncher):
583 class SSHEngineSetLauncher(LocalEngineSetLauncher):
584 launcher_class = SSHEngineLauncher
584 launcher_class = SSHEngineLauncher
585 engines = Dict(config=True,
585 engines = Dict(config=True,
586 help="""dict of engines to launch. This is a dict by hostname of ints,
586 help="""dict of engines to launch. This is a dict by hostname of ints,
587 corresponding to the number of engines to start on that host.""")
587 corresponding to the number of engines to start on that host.""")
588
588
589 def start(self, n, profile_dir):
589 def start(self, n, profile_dir):
590 """Start engines by profile or profile_dir.
590 """Start engines by profile or profile_dir.
591 `n` is ignored, and the `engines` config property is used instead.
591 `n` is ignored, and the `engines` config property is used instead.
592 """
592 """
593
593
594 self.profile_dir = unicode(profile_dir)
594 self.profile_dir = unicode(profile_dir)
595 dlist = []
595 dlist = []
596 for host, n in self.engines.iteritems():
596 for host, n in self.engines.iteritems():
597 if isinstance(n, (tuple, list)):
597 if isinstance(n, (tuple, list)):
598 n, args = n
598 n, args = n
599 else:
599 else:
600 args = copy.deepcopy(self.engine_args)
600 args = copy.deepcopy(self.engine_args)
601
601
602 if '@' in host:
602 if '@' in host:
603 user,host = host.split('@',1)
603 user,host = host.split('@',1)
604 else:
604 else:
605 user=None
605 user=None
606 for i in range(n):
606 for i in range(n):
607 el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log)
607 el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log)
608
608
609 # Copy the engine args over to each engine launcher.
609 # Copy the engine args over to each engine launcher.
610 i
610 i
611 el.program_args = args
611 el.program_args = args
612 el.on_stop(self._notice_engine_stopped)
612 el.on_stop(self._notice_engine_stopped)
613 d = el.start(profile_dir, user=user, hostname=host)
613 d = el.start(profile_dir, user=user, hostname=host)
614 if i==0:
614 if i==0:
615 self.log.info("Starting SSHEngineSetLauncher: %r" % el.args)
615 self.log.info("Starting SSHEngineSetLauncher: %r" % el.args)
616 self.launchers[host+str(i)] = el
616 self.launchers[host+str(i)] = el
617 dlist.append(d)
617 dlist.append(d)
618 self.notify_start(dlist)
618 self.notify_start(dlist)
619 return dlist
619 return dlist
620
620
621
621
622
622
623 #-----------------------------------------------------------------------------
623 #-----------------------------------------------------------------------------
624 # Windows HPC Server 2008 scheduler launchers
624 # Windows HPC Server 2008 scheduler launchers
625 #-----------------------------------------------------------------------------
625 #-----------------------------------------------------------------------------
626
626
627
627
628 # This is only used on Windows.
628 # This is only used on Windows.
629 def find_job_cmd():
629 def find_job_cmd():
630 if WINDOWS:
630 if WINDOWS:
631 try:
631 try:
632 return find_cmd('job')
632 return find_cmd('job')
633 except (FindCmdError, ImportError):
633 except (FindCmdError, ImportError):
634 # ImportError will be raised if win32api is not installed
634 # ImportError will be raised if win32api is not installed
635 return 'job'
635 return 'job'
636 else:
636 else:
637 return 'job'
637 return 'job'
638
638
639
639
640 class WindowsHPCLauncher(BaseLauncher):
640 class WindowsHPCLauncher(BaseLauncher):
641
641
642 job_id_regexp = Unicode(r'\d+', config=True,
642 job_id_regexp = Unicode(r'\d+', config=True,
643 help="""A regular expression used to get the job id from the output of the
643 help="""A regular expression used to get the job id from the output of the
644 submit_command. """
644 submit_command. """
645 )
645 )
646 job_file_name = Unicode(u'ipython_job.xml', config=True,
646 job_file_name = Unicode(u'ipython_job.xml', config=True,
647 help="The filename of the instantiated job script.")
647 help="The filename of the instantiated job script.")
648 # The full path to the instantiated job script. This gets made dynamically
648 # The full path to the instantiated job script. This gets made dynamically
649 # by combining the work_dir with the job_file_name.
649 # by combining the work_dir with the job_file_name.
650 job_file = Unicode(u'')
650 job_file = Unicode(u'')
651 scheduler = Unicode('', config=True,
651 scheduler = Unicode('', config=True,
652 help="The hostname of the scheduler to submit the job to.")
652 help="The hostname of the scheduler to submit the job to.")
653 job_cmd = Unicode(find_job_cmd(), config=True,
653 job_cmd = Unicode(find_job_cmd(), config=True,
654 help="The command for submitting jobs.")
654 help="The command for submitting jobs.")
655
655
656 def __init__(self, work_dir=u'.', config=None, **kwargs):
656 def __init__(self, work_dir=u'.', config=None, **kwargs):
657 super(WindowsHPCLauncher, self).__init__(
657 super(WindowsHPCLauncher, self).__init__(
658 work_dir=work_dir, config=config, **kwargs
658 work_dir=work_dir, config=config, **kwargs
659 )
659 )
660
660
661 @property
661 @property
662 def job_file(self):
662 def job_file(self):
663 return os.path.join(self.work_dir, self.job_file_name)
663 return os.path.join(self.work_dir, self.job_file_name)
664
664
665 def write_job_file(self, n):
665 def write_job_file(self, n):
666 raise NotImplementedError("Implement write_job_file in a subclass.")
666 raise NotImplementedError("Implement write_job_file in a subclass.")
667
667
668 def find_args(self):
668 def find_args(self):
669 return [u'job.exe']
669 return [u'job.exe']
670
670
671 def parse_job_id(self, output):
671 def parse_job_id(self, output):
672 """Take the output of the submit command and return the job id."""
672 """Take the output of the submit command and return the job id."""
673 m = re.search(self.job_id_regexp, output)
673 m = re.search(self.job_id_regexp, output)
674 if m is not None:
674 if m is not None:
675 job_id = m.group()
675 job_id = m.group()
676 else:
676 else:
677 raise LauncherError("Job id couldn't be determined: %s" % output)
677 raise LauncherError("Job id couldn't be determined: %s" % output)
678 self.job_id = job_id
678 self.job_id = job_id
679 self.log.info('Job started with job id: %r' % job_id)
679 self.log.info('Job started with job id: %r' % job_id)
680 return job_id
680 return job_id
681
681
682 def start(self, n):
682 def start(self, n):
683 """Start n copies of the process using the Win HPC job scheduler."""
683 """Start n copies of the process using the Win HPC job scheduler."""
684 self.write_job_file(n)
684 self.write_job_file(n)
685 args = [
685 args = [
686 'submit',
686 'submit',
687 '/jobfile:%s' % self.job_file,
687 '/jobfile:%s' % self.job_file,
688 '/scheduler:%s' % self.scheduler
688 '/scheduler:%s' % self.scheduler
689 ]
689 ]
690 self.log.info("Starting Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
690 self.log.info("Starting Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
691
691
692 output = check_output([self.job_cmd]+args,
692 output = check_output([self.job_cmd]+args,
693 env=os.environ,
693 env=os.environ,
694 cwd=self.work_dir,
694 cwd=self.work_dir,
695 stderr=STDOUT
695 stderr=STDOUT
696 )
696 )
697 job_id = self.parse_job_id(output)
697 job_id = self.parse_job_id(output)
698 self.notify_start(job_id)
698 self.notify_start(job_id)
699 return job_id
699 return job_id
700
700
701 def stop(self):
701 def stop(self):
702 args = [
702 args = [
703 'cancel',
703 'cancel',
704 self.job_id,
704 self.job_id,
705 '/scheduler:%s' % self.scheduler
705 '/scheduler:%s' % self.scheduler
706 ]
706 ]
707 self.log.info("Stopping Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
707 self.log.info("Stopping Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
708 try:
708 try:
709 output = check_output([self.job_cmd]+args,
709 output = check_output([self.job_cmd]+args,
710 env=os.environ,
710 env=os.environ,
711 cwd=self.work_dir,
711 cwd=self.work_dir,
712 stderr=STDOUT
712 stderr=STDOUT
713 )
713 )
714 except:
714 except:
715 output = 'The job already appears to be stoppped: %r' % self.job_id
715 output = 'The job already appears to be stoppped: %r' % self.job_id
716 self.notify_stop(dict(job_id=self.job_id, output=output)) # Pass the output of the kill cmd
716 self.notify_stop(dict(job_id=self.job_id, output=output)) # Pass the output of the kill cmd
717 return output
717 return output
718
718
719
719
720 class WindowsHPCControllerLauncher(WindowsHPCLauncher):
720 class WindowsHPCControllerLauncher(WindowsHPCLauncher):
721
721
722 job_file_name = Unicode(u'ipcontroller_job.xml', config=True,
722 job_file_name = Unicode(u'ipcontroller_job.xml', config=True,
723 help="WinHPC xml job file.")
723 help="WinHPC xml job file.")
724 extra_args = List([], config=False,
724 extra_args = List([], config=False,
725 help="extra args to pass to ipcontroller")
725 help="extra args to pass to ipcontroller")
726
726
727 def write_job_file(self, n):
727 def write_job_file(self, n):
728 job = IPControllerJob(config=self.config)
728 job = IPControllerJob(config=self.config)
729
729
730 t = IPControllerTask(config=self.config)
730 t = IPControllerTask(config=self.config)
731 # The tasks work directory is *not* the actual work directory of
731 # The tasks work directory is *not* the actual work directory of
732 # the controller. It is used as the base path for the stdout/stderr
732 # the controller. It is used as the base path for the stdout/stderr
733 # files that the scheduler redirects to.
733 # files that the scheduler redirects to.
734 t.work_directory = self.profile_dir
734 t.work_directory = self.profile_dir
735 # Add the profile_dir and from self.start().
735 # Add the profile_dir and from self.start().
736 t.controller_args.extend(self.extra_args)
736 t.controller_args.extend(self.extra_args)
737 job.add_task(t)
737 job.add_task(t)
738
738
739 self.log.info("Writing job description file: %s" % self.job_file)
739 self.log.info("Writing job description file: %s" % self.job_file)
740 job.write(self.job_file)
740 job.write(self.job_file)
741
741
742 @property
742 @property
743 def job_file(self):
743 def job_file(self):
744 return os.path.join(self.profile_dir, self.job_file_name)
744 return os.path.join(self.profile_dir, self.job_file_name)
745
745
746 def start(self, profile_dir):
746 def start(self, profile_dir):
747 """Start the controller by profile_dir."""
747 """Start the controller by profile_dir."""
748 self.extra_args = ['--profile_dir=%s'%profile_dir]
748 self.extra_args = ['--profile-dir=%s'%profile_dir]
749 self.profile_dir = unicode(profile_dir)
749 self.profile_dir = unicode(profile_dir)
750 return super(WindowsHPCControllerLauncher, self).start(1)
750 return super(WindowsHPCControllerLauncher, self).start(1)
751
751
752
752
753 class WindowsHPCEngineSetLauncher(WindowsHPCLauncher):
753 class WindowsHPCEngineSetLauncher(WindowsHPCLauncher):
754
754
755 job_file_name = Unicode(u'ipengineset_job.xml', config=True,
755 job_file_name = Unicode(u'ipengineset_job.xml', config=True,
756 help="jobfile for ipengines job")
756 help="jobfile for ipengines job")
757 extra_args = List([], config=False,
757 extra_args = List([], config=False,
758 help="extra args to pas to ipengine")
758 help="extra args to pas to ipengine")
759
759
760 def write_job_file(self, n):
760 def write_job_file(self, n):
761 job = IPEngineSetJob(config=self.config)
761 job = IPEngineSetJob(config=self.config)
762
762
763 for i in range(n):
763 for i in range(n):
764 t = IPEngineTask(config=self.config)
764 t = IPEngineTask(config=self.config)
765 # The tasks work directory is *not* the actual work directory of
765 # The tasks work directory is *not* the actual work directory of
766 # the engine. It is used as the base path for the stdout/stderr
766 # the engine. It is used as the base path for the stdout/stderr
767 # files that the scheduler redirects to.
767 # files that the scheduler redirects to.
768 t.work_directory = self.profile_dir
768 t.work_directory = self.profile_dir
769 # Add the profile_dir and from self.start().
769 # Add the profile_dir and from self.start().
770 t.engine_args.extend(self.extra_args)
770 t.engine_args.extend(self.extra_args)
771 job.add_task(t)
771 job.add_task(t)
772
772
773 self.log.info("Writing job description file: %s" % self.job_file)
773 self.log.info("Writing job description file: %s" % self.job_file)
774 job.write(self.job_file)
774 job.write(self.job_file)
775
775
776 @property
776 @property
777 def job_file(self):
777 def job_file(self):
778 return os.path.join(self.profile_dir, self.job_file_name)
778 return os.path.join(self.profile_dir, self.job_file_name)
779
779
780 def start(self, n, profile_dir):
780 def start(self, n, profile_dir):
781 """Start the controller by profile_dir."""
781 """Start the controller by profile_dir."""
782 self.extra_args = ['--profile_dir=%s'%profile_dir]
782 self.extra_args = ['--profile-dir=%s'%profile_dir]
783 self.profile_dir = unicode(profile_dir)
783 self.profile_dir = unicode(profile_dir)
784 return super(WindowsHPCEngineSetLauncher, self).start(n)
784 return super(WindowsHPCEngineSetLauncher, self).start(n)
785
785
786
786
787 #-----------------------------------------------------------------------------
787 #-----------------------------------------------------------------------------
788 # Batch (PBS) system launchers
788 # Batch (PBS) system launchers
789 #-----------------------------------------------------------------------------
789 #-----------------------------------------------------------------------------
790
790
791 class BatchSystemLauncher(BaseLauncher):
791 class BatchSystemLauncher(BaseLauncher):
792 """Launch an external process using a batch system.
792 """Launch an external process using a batch system.
793
793
794 This class is designed to work with UNIX batch systems like PBS, LSF,
794 This class is designed to work with UNIX batch systems like PBS, LSF,
795 GridEngine, etc. The overall model is that there are different commands
795 GridEngine, etc. The overall model is that there are different commands
796 like qsub, qdel, etc. that handle the starting and stopping of the process.
796 like qsub, qdel, etc. that handle the starting and stopping of the process.
797
797
798 This class also has the notion of a batch script. The ``batch_template``
798 This class also has the notion of a batch script. The ``batch_template``
799 attribute can be set to a string that is a template for the batch script.
799 attribute can be set to a string that is a template for the batch script.
800 This template is instantiated using string formatting. Thus the template can
800 This template is instantiated using string formatting. Thus the template can
801 use {n} fot the number of instances. Subclasses can add additional variables
801 use {n} fot the number of instances. Subclasses can add additional variables
802 to the template dict.
802 to the template dict.
803 """
803 """
804
804
805 # Subclasses must fill these in. See PBSEngineSet
805 # Subclasses must fill these in. See PBSEngineSet
806 submit_command = List([''], config=True,
806 submit_command = List([''], config=True,
807 help="The name of the command line program used to submit jobs.")
807 help="The name of the command line program used to submit jobs.")
808 delete_command = List([''], config=True,
808 delete_command = List([''], config=True,
809 help="The name of the command line program used to delete jobs.")
809 help="The name of the command line program used to delete jobs.")
810 job_id_regexp = Unicode('', config=True,
810 job_id_regexp = Unicode('', config=True,
811 help="""A regular expression used to get the job id from the output of the
811 help="""A regular expression used to get the job id from the output of the
812 submit_command.""")
812 submit_command.""")
813 batch_template = Unicode('', config=True,
813 batch_template = Unicode('', config=True,
814 help="The string that is the batch script template itself.")
814 help="The string that is the batch script template itself.")
815 batch_template_file = Unicode(u'', config=True,
815 batch_template_file = Unicode(u'', config=True,
816 help="The file that contains the batch template.")
816 help="The file that contains the batch template.")
817 batch_file_name = Unicode(u'batch_script', config=True,
817 batch_file_name = Unicode(u'batch_script', config=True,
818 help="The filename of the instantiated batch script.")
818 help="The filename of the instantiated batch script.")
819 queue = Unicode(u'', config=True,
819 queue = Unicode(u'', config=True,
820 help="The PBS Queue.")
820 help="The PBS Queue.")
821
821
822 # not configurable, override in subclasses
822 # not configurable, override in subclasses
823 # PBS Job Array regex
823 # PBS Job Array regex
824 job_array_regexp = Unicode('')
824 job_array_regexp = Unicode('')
825 job_array_template = Unicode('')
825 job_array_template = Unicode('')
826 # PBS Queue regex
826 # PBS Queue regex
827 queue_regexp = Unicode('')
827 queue_regexp = Unicode('')
828 queue_template = Unicode('')
828 queue_template = Unicode('')
829 # The default batch template, override in subclasses
829 # The default batch template, override in subclasses
830 default_template = Unicode('')
830 default_template = Unicode('')
831 # The full path to the instantiated batch script.
831 # The full path to the instantiated batch script.
832 batch_file = Unicode(u'')
832 batch_file = Unicode(u'')
833 # the format dict used with batch_template:
833 # the format dict used with batch_template:
834 context = Dict()
834 context = Dict()
835 # the Formatter instance for rendering the templates:
835 # the Formatter instance for rendering the templates:
836 formatter = Instance(EvalFormatter, (), {})
836 formatter = Instance(EvalFormatter, (), {})
837
837
838
838
839 def find_args(self):
839 def find_args(self):
840 return self.submit_command + [self.batch_file]
840 return self.submit_command + [self.batch_file]
841
841
842 def __init__(self, work_dir=u'.', config=None, **kwargs):
842 def __init__(self, work_dir=u'.', config=None, **kwargs):
843 super(BatchSystemLauncher, self).__init__(
843 super(BatchSystemLauncher, self).__init__(
844 work_dir=work_dir, config=config, **kwargs
844 work_dir=work_dir, config=config, **kwargs
845 )
845 )
846 self.batch_file = os.path.join(self.work_dir, self.batch_file_name)
846 self.batch_file = os.path.join(self.work_dir, self.batch_file_name)
847
847
848 def parse_job_id(self, output):
848 def parse_job_id(self, output):
849 """Take the output of the submit command and return the job id."""
849 """Take the output of the submit command and return the job id."""
850 m = re.search(self.job_id_regexp, output)
850 m = re.search(self.job_id_regexp, output)
851 if m is not None:
851 if m is not None:
852 job_id = m.group()
852 job_id = m.group()
853 else:
853 else:
854 raise LauncherError("Job id couldn't be determined: %s" % output)
854 raise LauncherError("Job id couldn't be determined: %s" % output)
855 self.job_id = job_id
855 self.job_id = job_id
856 self.log.info('Job submitted with job id: %r' % job_id)
856 self.log.info('Job submitted with job id: %r' % job_id)
857 return job_id
857 return job_id
858
858
859 def write_batch_script(self, n):
859 def write_batch_script(self, n):
860 """Instantiate and write the batch script to the work_dir."""
860 """Instantiate and write the batch script to the work_dir."""
861 self.context['n'] = n
861 self.context['n'] = n
862 self.context['queue'] = self.queue
862 self.context['queue'] = self.queue
863 # first priority is batch_template if set
863 # first priority is batch_template if set
864 if self.batch_template_file and not self.batch_template:
864 if self.batch_template_file and not self.batch_template:
865 # second priority is batch_template_file
865 # second priority is batch_template_file
866 with open(self.batch_template_file) as f:
866 with open(self.batch_template_file) as f:
867 self.batch_template = f.read()
867 self.batch_template = f.read()
868 if not self.batch_template:
868 if not self.batch_template:
869 # third (last) priority is default_template
869 # third (last) priority is default_template
870 self.batch_template = self.default_template
870 self.batch_template = self.default_template
871
871
872 # add jobarray or queue lines to user-specified template
872 # add jobarray or queue lines to user-specified template
873 # note that this is *only* when user did not specify a template.
873 # note that this is *only* when user did not specify a template.
874 regex = re.compile(self.job_array_regexp)
874 regex = re.compile(self.job_array_regexp)
875 # print regex.search(self.batch_template)
875 # print regex.search(self.batch_template)
876 if not regex.search(self.batch_template):
876 if not regex.search(self.batch_template):
877 self.log.info("adding job array settings to batch script")
877 self.log.info("adding job array settings to batch script")
878 firstline, rest = self.batch_template.split('\n',1)
878 firstline, rest = self.batch_template.split('\n',1)
879 self.batch_template = u'\n'.join([firstline, self.job_array_template, rest])
879 self.batch_template = u'\n'.join([firstline, self.job_array_template, rest])
880
880
881 regex = re.compile(self.queue_regexp)
881 regex = re.compile(self.queue_regexp)
882 # print regex.search(self.batch_template)
882 # print regex.search(self.batch_template)
883 if self.queue and not regex.search(self.batch_template):
883 if self.queue and not regex.search(self.batch_template):
884 self.log.info("adding PBS queue settings to batch script")
884 self.log.info("adding PBS queue settings to batch script")
885 firstline, rest = self.batch_template.split('\n',1)
885 firstline, rest = self.batch_template.split('\n',1)
886 self.batch_template = u'\n'.join([firstline, self.queue_template, rest])
886 self.batch_template = u'\n'.join([firstline, self.queue_template, rest])
887
887
888 script_as_string = self.formatter.format(self.batch_template, **self.context)
888 script_as_string = self.formatter.format(self.batch_template, **self.context)
889 self.log.info('Writing instantiated batch script: %s' % self.batch_file)
889 self.log.info('Writing instantiated batch script: %s' % self.batch_file)
890
890
891 with open(self.batch_file, 'w') as f:
891 with open(self.batch_file, 'w') as f:
892 f.write(script_as_string)
892 f.write(script_as_string)
893 os.chmod(self.batch_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
893 os.chmod(self.batch_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
894
894
895 def start(self, n, profile_dir):
895 def start(self, n, profile_dir):
896 """Start n copies of the process using a batch system."""
896 """Start n copies of the process using a batch system."""
897 # Here we save profile_dir in the context so they
897 # Here we save profile_dir in the context so they
898 # can be used in the batch script template as {profile_dir}
898 # can be used in the batch script template as {profile_dir}
899 self.context['profile_dir'] = profile_dir
899 self.context['profile_dir'] = profile_dir
900 self.profile_dir = unicode(profile_dir)
900 self.profile_dir = unicode(profile_dir)
901 self.write_batch_script(n)
901 self.write_batch_script(n)
902 output = check_output(self.args, env=os.environ)
902 output = check_output(self.args, env=os.environ)
903
903
904 job_id = self.parse_job_id(output)
904 job_id = self.parse_job_id(output)
905 self.notify_start(job_id)
905 self.notify_start(job_id)
906 return job_id
906 return job_id
907
907
908 def stop(self):
908 def stop(self):
909 output = check_output(self.delete_command+[self.job_id], env=os.environ)
909 output = check_output(self.delete_command+[self.job_id], env=os.environ)
910 self.notify_stop(dict(job_id=self.job_id, output=output)) # Pass the output of the kill cmd
910 self.notify_stop(dict(job_id=self.job_id, output=output)) # Pass the output of the kill cmd
911 return output
911 return output
912
912
913
913
914 class PBSLauncher(BatchSystemLauncher):
914 class PBSLauncher(BatchSystemLauncher):
915 """A BatchSystemLauncher subclass for PBS."""
915 """A BatchSystemLauncher subclass for PBS."""
916
916
917 submit_command = List(['qsub'], config=True,
917 submit_command = List(['qsub'], config=True,
918 help="The PBS submit command ['qsub']")
918 help="The PBS submit command ['qsub']")
919 delete_command = List(['qdel'], config=True,
919 delete_command = List(['qdel'], config=True,
920 help="The PBS delete command ['qsub']")
920 help="The PBS delete command ['qsub']")
921 job_id_regexp = Unicode(r'\d+', config=True,
921 job_id_regexp = Unicode(r'\d+', config=True,
922 help="Regular expresion for identifying the job ID [r'\d+']")
922 help="Regular expresion for identifying the job ID [r'\d+']")
923
923
924 batch_file = Unicode(u'')
924 batch_file = Unicode(u'')
925 job_array_regexp = Unicode('#PBS\W+-t\W+[\w\d\-\$]+')
925 job_array_regexp = Unicode('#PBS\W+-t\W+[\w\d\-\$]+')
926 job_array_template = Unicode('#PBS -t 1-{n}')
926 job_array_template = Unicode('#PBS -t 1-{n}')
927 queue_regexp = Unicode('#PBS\W+-q\W+\$?\w+')
927 queue_regexp = Unicode('#PBS\W+-q\W+\$?\w+')
928 queue_template = Unicode('#PBS -q {queue}')
928 queue_template = Unicode('#PBS -q {queue}')
929
929
930
930
931 class PBSControllerLauncher(PBSLauncher):
931 class PBSControllerLauncher(PBSLauncher):
932 """Launch a controller using PBS."""
932 """Launch a controller using PBS."""
933
933
934 batch_file_name = Unicode(u'pbs_controller', config=True,
934 batch_file_name = Unicode(u'pbs_controller', config=True,
935 help="batch file name for the controller job.")
935 help="batch file name for the controller job.")
936 default_template= Unicode("""#!/bin/sh
936 default_template= Unicode("""#!/bin/sh
937 #PBS -V
937 #PBS -V
938 #PBS -N ipcontroller
938 #PBS -N ipcontroller
939 %s --log-to-file --profile_dir={profile_dir}
939 %s --log-to-file --profile-dir={profile_dir}
940 """%(' '.join(ipcontroller_cmd_argv)))
940 """%(' '.join(ipcontroller_cmd_argv)))
941
941
942 def start(self, profile_dir):
942 def start(self, profile_dir):
943 """Start the controller by profile or profile_dir."""
943 """Start the controller by profile or profile_dir."""
944 self.log.info("Starting PBSControllerLauncher: %r" % self.args)
944 self.log.info("Starting PBSControllerLauncher: %r" % self.args)
945 return super(PBSControllerLauncher, self).start(1, profile_dir)
945 return super(PBSControllerLauncher, self).start(1, profile_dir)
946
946
947
947
948 class PBSEngineSetLauncher(PBSLauncher):
948 class PBSEngineSetLauncher(PBSLauncher):
949 """Launch Engines using PBS"""
949 """Launch Engines using PBS"""
950 batch_file_name = Unicode(u'pbs_engines', config=True,
950 batch_file_name = Unicode(u'pbs_engines', config=True,
951 help="batch file name for the engine(s) job.")
951 help="batch file name for the engine(s) job.")
952 default_template= Unicode(u"""#!/bin/sh
952 default_template= Unicode(u"""#!/bin/sh
953 #PBS -V
953 #PBS -V
954 #PBS -N ipengine
954 #PBS -N ipengine
955 %s --profile_dir={profile_dir}
955 %s --profile-dir={profile_dir}
956 """%(' '.join(ipengine_cmd_argv)))
956 """%(' '.join(ipengine_cmd_argv)))
957
957
958 def start(self, n, profile_dir):
958 def start(self, n, profile_dir):
959 """Start n engines by profile or profile_dir."""
959 """Start n engines by profile or profile_dir."""
960 self.log.info('Starting %i engines with PBSEngineSetLauncher: %r' % (n, self.args))
960 self.log.info('Starting %i engines with PBSEngineSetLauncher: %r' % (n, self.args))
961 return super(PBSEngineSetLauncher, self).start(n, profile_dir)
961 return super(PBSEngineSetLauncher, self).start(n, profile_dir)
962
962
963 #SGE is very similar to PBS
963 #SGE is very similar to PBS
964
964
965 class SGELauncher(PBSLauncher):
965 class SGELauncher(PBSLauncher):
966 """Sun GridEngine is a PBS clone with slightly different syntax"""
966 """Sun GridEngine is a PBS clone with slightly different syntax"""
967 job_array_regexp = Unicode('#\$\W+\-t')
967 job_array_regexp = Unicode('#\$\W+\-t')
968 job_array_template = Unicode('#$ -t 1-{n}')
968 job_array_template = Unicode('#$ -t 1-{n}')
969 queue_regexp = Unicode('#\$\W+-q\W+\$?\w+')
969 queue_regexp = Unicode('#\$\W+-q\W+\$?\w+')
970 queue_template = Unicode('#$ -q {queue}')
970 queue_template = Unicode('#$ -q {queue}')
971
971
972 class SGEControllerLauncher(SGELauncher):
972 class SGEControllerLauncher(SGELauncher):
973 """Launch a controller using SGE."""
973 """Launch a controller using SGE."""
974
974
975 batch_file_name = Unicode(u'sge_controller', config=True,
975 batch_file_name = Unicode(u'sge_controller', config=True,
976 help="batch file name for the ipontroller job.")
976 help="batch file name for the ipontroller job.")
977 default_template= Unicode(u"""#$ -V
977 default_template= Unicode(u"""#$ -V
978 #$ -S /bin/sh
978 #$ -S /bin/sh
979 #$ -N ipcontroller
979 #$ -N ipcontroller
980 %s --log-to-file --profile_dir={profile_dir}
980 %s --log-to-file --profile-dir={profile_dir}
981 """%(' '.join(ipcontroller_cmd_argv)))
981 """%(' '.join(ipcontroller_cmd_argv)))
982
982
983 def start(self, profile_dir):
983 def start(self, profile_dir):
984 """Start the controller by profile or profile_dir."""
984 """Start the controller by profile or profile_dir."""
985 self.log.info("Starting PBSControllerLauncher: %r" % self.args)
985 self.log.info("Starting PBSControllerLauncher: %r" % self.args)
986 return super(SGEControllerLauncher, self).start(1, profile_dir)
986 return super(SGEControllerLauncher, self).start(1, profile_dir)
987
987
988 class SGEEngineSetLauncher(SGELauncher):
988 class SGEEngineSetLauncher(SGELauncher):
989 """Launch Engines with SGE"""
989 """Launch Engines with SGE"""
990 batch_file_name = Unicode(u'sge_engines', config=True,
990 batch_file_name = Unicode(u'sge_engines', config=True,
991 help="batch file name for the engine(s) job.")
991 help="batch file name for the engine(s) job.")
992 default_template = Unicode("""#$ -V
992 default_template = Unicode("""#$ -V
993 #$ -S /bin/sh
993 #$ -S /bin/sh
994 #$ -N ipengine
994 #$ -N ipengine
995 %s --profile_dir={profile_dir}
995 %s --profile-dir={profile_dir}
996 """%(' '.join(ipengine_cmd_argv)))
996 """%(' '.join(ipengine_cmd_argv)))
997
997
998 def start(self, n, profile_dir):
998 def start(self, n, profile_dir):
999 """Start n engines by profile or profile_dir."""
999 """Start n engines by profile or profile_dir."""
1000 self.log.info('Starting %i engines with SGEEngineSetLauncher: %r' % (n, self.args))
1000 self.log.info('Starting %i engines with SGEEngineSetLauncher: %r' % (n, self.args))
1001 return super(SGEEngineSetLauncher, self).start(n, profile_dir)
1001 return super(SGEEngineSetLauncher, self).start(n, profile_dir)
1002
1002
1003
1003
1004 #-----------------------------------------------------------------------------
1004 #-----------------------------------------------------------------------------
1005 # A launcher for ipcluster itself!
1005 # A launcher for ipcluster itself!
1006 #-----------------------------------------------------------------------------
1006 #-----------------------------------------------------------------------------
1007
1007
1008
1008
1009 class IPClusterLauncher(LocalProcessLauncher):
1009 class IPClusterLauncher(LocalProcessLauncher):
1010 """Launch the ipcluster program in an external process."""
1010 """Launch the ipcluster program in an external process."""
1011
1011
1012 ipcluster_cmd = List(ipcluster_cmd_argv, config=True,
1012 ipcluster_cmd = List(ipcluster_cmd_argv, config=True,
1013 help="Popen command for ipcluster")
1013 help="Popen command for ipcluster")
1014 ipcluster_args = List(
1014 ipcluster_args = List(
1015 ['--clean-logs', '--log-to-file', '--log_level=%i'%logging.INFO], config=True,
1015 ['--clean-logs', '--log-to-file', '--log-level=%i'%logging.INFO], config=True,
1016 help="Command line arguments to pass to ipcluster.")
1016 help="Command line arguments to pass to ipcluster.")
1017 ipcluster_subcommand = Unicode('start')
1017 ipcluster_subcommand = Unicode('start')
1018 ipcluster_n = Int(2)
1018 ipcluster_n = Int(2)
1019
1019
1020 def find_args(self):
1020 def find_args(self):
1021 return self.ipcluster_cmd + [self.ipcluster_subcommand] + \
1021 return self.ipcluster_cmd + [self.ipcluster_subcommand] + \
1022 ['--n=%i'%self.ipcluster_n] + self.ipcluster_args
1022 ['--n=%i'%self.ipcluster_n] + self.ipcluster_args
1023
1023
1024 def start(self):
1024 def start(self):
1025 self.log.info("Starting ipcluster: %r" % self.args)
1025 self.log.info("Starting ipcluster: %r" % self.args)
1026 return super(IPClusterLauncher, self).start()
1026 return super(IPClusterLauncher, self).start()
1027
1027
1028 #-----------------------------------------------------------------------------
1028 #-----------------------------------------------------------------------------
1029 # Collections of launchers
1029 # Collections of launchers
1030 #-----------------------------------------------------------------------------
1030 #-----------------------------------------------------------------------------
1031
1031
1032 local_launchers = [
1032 local_launchers = [
1033 LocalControllerLauncher,
1033 LocalControllerLauncher,
1034 LocalEngineLauncher,
1034 LocalEngineLauncher,
1035 LocalEngineSetLauncher,
1035 LocalEngineSetLauncher,
1036 ]
1036 ]
1037 mpi_launchers = [
1037 mpi_launchers = [
1038 MPIExecLauncher,
1038 MPIExecLauncher,
1039 MPIExecControllerLauncher,
1039 MPIExecControllerLauncher,
1040 MPIExecEngineSetLauncher,
1040 MPIExecEngineSetLauncher,
1041 ]
1041 ]
1042 ssh_launchers = [
1042 ssh_launchers = [
1043 SSHLauncher,
1043 SSHLauncher,
1044 SSHControllerLauncher,
1044 SSHControllerLauncher,
1045 SSHEngineLauncher,
1045 SSHEngineLauncher,
1046 SSHEngineSetLauncher,
1046 SSHEngineSetLauncher,
1047 ]
1047 ]
1048 winhpc_launchers = [
1048 winhpc_launchers = [
1049 WindowsHPCLauncher,
1049 WindowsHPCLauncher,
1050 WindowsHPCControllerLauncher,
1050 WindowsHPCControllerLauncher,
1051 WindowsHPCEngineSetLauncher,
1051 WindowsHPCEngineSetLauncher,
1052 ]
1052 ]
1053 pbs_launchers = [
1053 pbs_launchers = [
1054 PBSLauncher,
1054 PBSLauncher,
1055 PBSControllerLauncher,
1055 PBSControllerLauncher,
1056 PBSEngineSetLauncher,
1056 PBSEngineSetLauncher,
1057 ]
1057 ]
1058 sge_launchers = [
1058 sge_launchers = [
1059 SGELauncher,
1059 SGELauncher,
1060 SGEControllerLauncher,
1060 SGEControllerLauncher,
1061 SGEEngineSetLauncher,
1061 SGEEngineSetLauncher,
1062 ]
1062 ]
1063 all_launchers = local_launchers + mpi_launchers + ssh_launchers + winhpc_launchers\
1063 all_launchers = local_launchers + mpi_launchers + ssh_launchers + winhpc_launchers\
1064 + pbs_launchers + sge_launchers
1064 + pbs_launchers + sge_launchers
1065
1065
@@ -1,111 +1,111 b''
1 """toplevel setup/teardown for parallel tests."""
1 """toplevel setup/teardown for parallel tests."""
2
2
3 #-------------------------------------------------------------------------------
3 #-------------------------------------------------------------------------------
4 # Copyright (C) 2011 The IPython Development Team
4 # Copyright (C) 2011 The IPython Development Team
5 #
5 #
6 # Distributed under the terms of the BSD License. The full license is in
6 # Distributed under the terms of the BSD License. The full license is in
7 # the file COPYING, distributed as part of this software.
7 # the file COPYING, distributed as part of this software.
8 #-------------------------------------------------------------------------------
8 #-------------------------------------------------------------------------------
9
9
10 #-------------------------------------------------------------------------------
10 #-------------------------------------------------------------------------------
11 # Imports
11 # Imports
12 #-------------------------------------------------------------------------------
12 #-------------------------------------------------------------------------------
13
13
14 import os
14 import os
15 import tempfile
15 import tempfile
16 import time
16 import time
17 from subprocess import Popen
17 from subprocess import Popen
18
18
19 from IPython.utils.path import get_ipython_dir
19 from IPython.utils.path import get_ipython_dir
20 from IPython.parallel import Client
20 from IPython.parallel import Client
21 from IPython.parallel.apps.launcher import (LocalProcessLauncher,
21 from IPython.parallel.apps.launcher import (LocalProcessLauncher,
22 ipengine_cmd_argv,
22 ipengine_cmd_argv,
23 ipcontroller_cmd_argv,
23 ipcontroller_cmd_argv,
24 SIGKILL)
24 SIGKILL)
25
25
26 # globals
26 # globals
27 launchers = []
27 launchers = []
28 blackhole = open(os.devnull, 'w')
28 blackhole = open(os.devnull, 'w')
29
29
30 # Launcher class
30 # Launcher class
31 class TestProcessLauncher(LocalProcessLauncher):
31 class TestProcessLauncher(LocalProcessLauncher):
32 """subclass LocalProcessLauncher, to prevent extra sockets and threads being created on Windows"""
32 """subclass LocalProcessLauncher, to prevent extra sockets and threads being created on Windows"""
33 def start(self):
33 def start(self):
34 if self.state == 'before':
34 if self.state == 'before':
35 self.process = Popen(self.args,
35 self.process = Popen(self.args,
36 stdout=blackhole, stderr=blackhole,
36 stdout=blackhole, stderr=blackhole,
37 env=os.environ,
37 env=os.environ,
38 cwd=self.work_dir
38 cwd=self.work_dir
39 )
39 )
40 self.notify_start(self.process.pid)
40 self.notify_start(self.process.pid)
41 self.poll = self.process.poll
41 self.poll = self.process.poll
42 else:
42 else:
43 s = 'The process was already started and has state: %r' % self.state
43 s = 'The process was already started and has state: %r' % self.state
44 raise ProcessStateError(s)
44 raise ProcessStateError(s)
45
45
46 # nose setup/teardown
46 # nose setup/teardown
47
47
48 def setup():
48 def setup():
49 cluster_dir = os.path.join(get_ipython_dir(), 'profile_iptest')
49 cluster_dir = os.path.join(get_ipython_dir(), 'profile_iptest')
50 engine_json = os.path.join(cluster_dir, 'security', 'ipcontroller-engine.json')
50 engine_json = os.path.join(cluster_dir, 'security', 'ipcontroller-engine.json')
51 client_json = os.path.join(cluster_dir, 'security', 'ipcontroller-client.json')
51 client_json = os.path.join(cluster_dir, 'security', 'ipcontroller-client.json')
52 for json in (engine_json, client_json):
52 for json in (engine_json, client_json):
53 if os.path.exists(json):
53 if os.path.exists(json):
54 os.remove(json)
54 os.remove(json)
55
55
56 cp = TestProcessLauncher()
56 cp = TestProcessLauncher()
57 cp.cmd_and_args = ipcontroller_cmd_argv + \
57 cp.cmd_and_args = ipcontroller_cmd_argv + \
58 ['--profile=iptest', '--log_level=50']
58 ['--profile=iptest', '--log-level=50']
59 cp.start()
59 cp.start()
60 launchers.append(cp)
60 launchers.append(cp)
61 tic = time.time()
61 tic = time.time()
62 while not os.path.exists(engine_json) or not os.path.exists(client_json):
62 while not os.path.exists(engine_json) or not os.path.exists(client_json):
63 if cp.poll() is not None:
63 if cp.poll() is not None:
64 print cp.poll()
64 print cp.poll()
65 raise RuntimeError("The test controller failed to start.")
65 raise RuntimeError("The test controller failed to start.")
66 elif time.time()-tic > 10:
66 elif time.time()-tic > 10:
67 raise RuntimeError("Timeout waiting for the test controller to start.")
67 raise RuntimeError("Timeout waiting for the test controller to start.")
68 time.sleep(0.1)
68 time.sleep(0.1)
69 add_engines(1)
69 add_engines(1)
70
70
71 def add_engines(n=1, profile='iptest'):
71 def add_engines(n=1, profile='iptest'):
72 rc = Client(profile=profile)
72 rc = Client(profile=profile)
73 base = len(rc)
73 base = len(rc)
74 eps = []
74 eps = []
75 for i in range(n):
75 for i in range(n):
76 ep = TestProcessLauncher()
76 ep = TestProcessLauncher()
77 ep.cmd_and_args = ipengine_cmd_argv + ['--profile=%s'%profile, '--log_level=50']
77 ep.cmd_and_args = ipengine_cmd_argv + ['--profile=%s'%profile, '--log-level=50']
78 ep.start()
78 ep.start()
79 launchers.append(ep)
79 launchers.append(ep)
80 eps.append(ep)
80 eps.append(ep)
81 tic = time.time()
81 tic = time.time()
82 while len(rc) < base+n:
82 while len(rc) < base+n:
83 if any([ ep.poll() is not None for ep in eps ]):
83 if any([ ep.poll() is not None for ep in eps ]):
84 raise RuntimeError("A test engine failed to start.")
84 raise RuntimeError("A test engine failed to start.")
85 elif time.time()-tic > 10:
85 elif time.time()-tic > 10:
86 raise RuntimeError("Timeout waiting for engines to connect.")
86 raise RuntimeError("Timeout waiting for engines to connect.")
87 time.sleep(.1)
87 time.sleep(.1)
88 rc.spin()
88 rc.spin()
89 rc.close()
89 rc.close()
90 return eps
90 return eps
91
91
92 def teardown():
92 def teardown():
93 time.sleep(1)
93 time.sleep(1)
94 while launchers:
94 while launchers:
95 p = launchers.pop()
95 p = launchers.pop()
96 if p.poll() is None:
96 if p.poll() is None:
97 try:
97 try:
98 p.stop()
98 p.stop()
99 except Exception, e:
99 except Exception, e:
100 print e
100 print e
101 pass
101 pass
102 if p.poll() is None:
102 if p.poll() is None:
103 time.sleep(.25)
103 time.sleep(.25)
104 if p.poll() is None:
104 if p.poll() is None:
105 try:
105 try:
106 print 'cleaning up test process...'
106 print 'cleaning up test process...'
107 p.signal(SIGKILL)
107 p.signal(SIGKILL)
108 except:
108 except:
109 print "couldn't shutdown process: ", p
109 print "couldn't shutdown process: ", p
110 blackhole.close()
110 blackhole.close()
111
111
@@ -1,309 +1,321 b''
1 """Generic testing tools that do NOT depend on Twisted.
1 """Generic testing tools that do NOT depend on Twisted.
2
2
3 In particular, this module exposes a set of top-level assert* functions that
3 In particular, this module exposes a set of top-level assert* functions that
4 can be used in place of nose.tools.assert* in method generators (the ones in
4 can be used in place of nose.tools.assert* in method generators (the ones in
5 nose can not, at least as of nose 0.10.4).
5 nose can not, at least as of nose 0.10.4).
6
6
7 Note: our testing package contains testing.util, which does depend on Twisted
7 Note: our testing package contains testing.util, which does depend on Twisted
8 and provides utilities for tests that manage Deferreds. All testing support
8 and provides utilities for tests that manage Deferreds. All testing support
9 tools that only depend on nose, IPython or the standard library should go here
9 tools that only depend on nose, IPython or the standard library should go here
10 instead.
10 instead.
11
11
12
12
13 Authors
13 Authors
14 -------
14 -------
15 - Fernando Perez <Fernando.Perez@berkeley.edu>
15 - Fernando Perez <Fernando.Perez@berkeley.edu>
16 """
16 """
17
17
18 from __future__ import absolute_import
18 from __future__ import absolute_import
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Copyright (C) 2009 The IPython Development Team
21 # Copyright (C) 2009 The IPython Development Team
22 #
22 #
23 # Distributed under the terms of the BSD License. The full license is in
23 # Distributed under the terms of the BSD License. The full license is in
24 # the file COPYING, distributed as part of this software.
24 # the file COPYING, distributed as part of this software.
25 #-----------------------------------------------------------------------------
25 #-----------------------------------------------------------------------------
26
26
27 #-----------------------------------------------------------------------------
27 #-----------------------------------------------------------------------------
28 # Imports
28 # Imports
29 #-----------------------------------------------------------------------------
29 #-----------------------------------------------------------------------------
30
30
31 import os
31 import os
32 import re
32 import re
33 import sys
33 import sys
34
34
35 from contextlib import contextmanager
36
35 try:
37 try:
36 # These tools are used by parts of the runtime, so we make the nose
38 # These tools are used by parts of the runtime, so we make the nose
37 # dependency optional at this point. Nose is a hard dependency to run the
39 # dependency optional at this point. Nose is a hard dependency to run the
38 # test suite, but NOT to use ipython itself.
40 # test suite, but NOT to use ipython itself.
39 import nose.tools as nt
41 import nose.tools as nt
40 has_nose = True
42 has_nose = True
41 except ImportError:
43 except ImportError:
42 has_nose = False
44 has_nose = False
43
45
44 from IPython.config.loader import Config
46 from IPython.config.loader import Config
45 from IPython.utils.process import find_cmd, getoutputerror
47 from IPython.utils.process import find_cmd, getoutputerror
46 from IPython.utils.text import list_strings
48 from IPython.utils.text import list_strings
47 from IPython.utils.io import temp_pyfile
49 from IPython.utils.io import temp_pyfile
48
50
49 from . import decorators as dec
51 from . import decorators as dec
50 from . import skipdoctest
52 from . import skipdoctest
51
53
52 #-----------------------------------------------------------------------------
54 #-----------------------------------------------------------------------------
53 # Globals
55 # Globals
54 #-----------------------------------------------------------------------------
56 #-----------------------------------------------------------------------------
55
57
56 # Make a bunch of nose.tools assert wrappers that can be used in test
58 # Make a bunch of nose.tools assert wrappers that can be used in test
57 # generators. This will expose an assert* function for each one in nose.tools.
59 # generators. This will expose an assert* function for each one in nose.tools.
58
60
59 _tpl = """
61 _tpl = """
60 def %(name)s(*a,**kw):
62 def %(name)s(*a,**kw):
61 return nt.%(name)s(*a,**kw)
63 return nt.%(name)s(*a,**kw)
62 """
64 """
63
65
64 if has_nose:
66 if has_nose:
65 for _x in [a for a in dir(nt) if a.startswith('assert')]:
67 for _x in [a for a in dir(nt) if a.startswith('assert')]:
66 exec _tpl % dict(name=_x)
68 exec _tpl % dict(name=_x)
67
69
68 #-----------------------------------------------------------------------------
70 #-----------------------------------------------------------------------------
69 # Functions and classes
71 # Functions and classes
70 #-----------------------------------------------------------------------------
72 #-----------------------------------------------------------------------------
71
73
72 # The docstring for full_path doctests differently on win32 (different path
74 # The docstring for full_path doctests differently on win32 (different path
73 # separator) so just skip the doctest there. The example remains informative.
75 # separator) so just skip the doctest there. The example remains informative.
74 doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
76 doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
75
77
76 @doctest_deco
78 @doctest_deco
77 def full_path(startPath,files):
79 def full_path(startPath,files):
78 """Make full paths for all the listed files, based on startPath.
80 """Make full paths for all the listed files, based on startPath.
79
81
80 Only the base part of startPath is kept, since this routine is typically
82 Only the base part of startPath is kept, since this routine is typically
81 used with a script's __file__ variable as startPath. The base of startPath
83 used with a script's __file__ variable as startPath. The base of startPath
82 is then prepended to all the listed files, forming the output list.
84 is then prepended to all the listed files, forming the output list.
83
85
84 Parameters
86 Parameters
85 ----------
87 ----------
86 startPath : string
88 startPath : string
87 Initial path to use as the base for the results. This path is split
89 Initial path to use as the base for the results. This path is split
88 using os.path.split() and only its first component is kept.
90 using os.path.split() and only its first component is kept.
89
91
90 files : string or list
92 files : string or list
91 One or more files.
93 One or more files.
92
94
93 Examples
95 Examples
94 --------
96 --------
95
97
96 >>> full_path('/foo/bar.py',['a.txt','b.txt'])
98 >>> full_path('/foo/bar.py',['a.txt','b.txt'])
97 ['/foo/a.txt', '/foo/b.txt']
99 ['/foo/a.txt', '/foo/b.txt']
98
100
99 >>> full_path('/foo',['a.txt','b.txt'])
101 >>> full_path('/foo',['a.txt','b.txt'])
100 ['/a.txt', '/b.txt']
102 ['/a.txt', '/b.txt']
101
103
102 If a single file is given, the output is still a list:
104 If a single file is given, the output is still a list:
103 >>> full_path('/foo','a.txt')
105 >>> full_path('/foo','a.txt')
104 ['/a.txt']
106 ['/a.txt']
105 """
107 """
106
108
107 files = list_strings(files)
109 files = list_strings(files)
108 base = os.path.split(startPath)[0]
110 base = os.path.split(startPath)[0]
109 return [ os.path.join(base,f) for f in files ]
111 return [ os.path.join(base,f) for f in files ]
110
112
111
113
112 def parse_test_output(txt):
114 def parse_test_output(txt):
113 """Parse the output of a test run and return errors, failures.
115 """Parse the output of a test run and return errors, failures.
114
116
115 Parameters
117 Parameters
116 ----------
118 ----------
117 txt : str
119 txt : str
118 Text output of a test run, assumed to contain a line of one of the
120 Text output of a test run, assumed to contain a line of one of the
119 following forms::
121 following forms::
120 'FAILED (errors=1)'
122 'FAILED (errors=1)'
121 'FAILED (failures=1)'
123 'FAILED (failures=1)'
122 'FAILED (errors=1, failures=1)'
124 'FAILED (errors=1, failures=1)'
123
125
124 Returns
126 Returns
125 -------
127 -------
126 nerr, nfail: number of errors and failures.
128 nerr, nfail: number of errors and failures.
127 """
129 """
128
130
129 err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
131 err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
130 if err_m:
132 if err_m:
131 nerr = int(err_m.group(1))
133 nerr = int(err_m.group(1))
132 nfail = 0
134 nfail = 0
133 return nerr, nfail
135 return nerr, nfail
134
136
135 fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
137 fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
136 if fail_m:
138 if fail_m:
137 nerr = 0
139 nerr = 0
138 nfail = int(fail_m.group(1))
140 nfail = int(fail_m.group(1))
139 return nerr, nfail
141 return nerr, nfail
140
142
141 both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
143 both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
142 re.MULTILINE)
144 re.MULTILINE)
143 if both_m:
145 if both_m:
144 nerr = int(both_m.group(1))
146 nerr = int(both_m.group(1))
145 nfail = int(both_m.group(2))
147 nfail = int(both_m.group(2))
146 return nerr, nfail
148 return nerr, nfail
147
149
148 # If the input didn't match any of these forms, assume no error/failures
150 # If the input didn't match any of these forms, assume no error/failures
149 return 0, 0
151 return 0, 0
150
152
151
153
152 # So nose doesn't think this is a test
154 # So nose doesn't think this is a test
153 parse_test_output.__test__ = False
155 parse_test_output.__test__ = False
154
156
155
157
156 def default_argv():
158 def default_argv():
157 """Return a valid default argv for creating testing instances of ipython"""
159 """Return a valid default argv for creating testing instances of ipython"""
158
160
159 return ['--quick', # so no config file is loaded
161 return ['--quick', # so no config file is loaded
160 # Other defaults to minimize side effects on stdout
162 # Other defaults to minimize side effects on stdout
161 '--colors=NoColor', '--no-term-title','--no-banner',
163 '--colors=NoColor', '--no-term-title','--no-banner',
162 '--autocall=0']
164 '--autocall=0']
163
165
164
166
165 def default_config():
167 def default_config():
166 """Return a config object with good defaults for testing."""
168 """Return a config object with good defaults for testing."""
167 config = Config()
169 config = Config()
168 config.TerminalInteractiveShell.colors = 'NoColor'
170 config.TerminalInteractiveShell.colors = 'NoColor'
169 config.TerminalTerminalInteractiveShell.term_title = False,
171 config.TerminalTerminalInteractiveShell.term_title = False,
170 config.TerminalInteractiveShell.autocall = 0
172 config.TerminalInteractiveShell.autocall = 0
171 config.HistoryManager.hist_file = u'test_hist.sqlite'
173 config.HistoryManager.hist_file = u'test_hist.sqlite'
172 config.HistoryManager.db_cache_size = 10000
174 config.HistoryManager.db_cache_size = 10000
173 return config
175 return config
174
176
175
177
176 def ipexec(fname, options=None):
178 def ipexec(fname, options=None):
177 """Utility to call 'ipython filename'.
179 """Utility to call 'ipython filename'.
178
180
179 Starts IPython witha minimal and safe configuration to make startup as fast
181 Starts IPython witha minimal and safe configuration to make startup as fast
180 as possible.
182 as possible.
181
183
182 Note that this starts IPython in a subprocess!
184 Note that this starts IPython in a subprocess!
183
185
184 Parameters
186 Parameters
185 ----------
187 ----------
186 fname : str
188 fname : str
187 Name of file to be executed (should have .py or .ipy extension).
189 Name of file to be executed (should have .py or .ipy extension).
188
190
189 options : optional, list
191 options : optional, list
190 Extra command-line flags to be passed to IPython.
192 Extra command-line flags to be passed to IPython.
191
193
192 Returns
194 Returns
193 -------
195 -------
194 (stdout, stderr) of ipython subprocess.
196 (stdout, stderr) of ipython subprocess.
195 """
197 """
196 if options is None: options = []
198 if options is None: options = []
197
199
198 # For these subprocess calls, eliminate all prompt printing so we only see
200 # For these subprocess calls, eliminate all prompt printing so we only see
199 # output from script execution
201 # output from script execution
200 prompt_opts = [ '--InteractiveShell.prompt_in1=""',
202 prompt_opts = [ '--InteractiveShell.prompt_in1=""',
201 '--InteractiveShell.prompt_in2=""',
203 '--InteractiveShell.prompt_in2=""',
202 '--InteractiveShell.prompt_out=""'
204 '--InteractiveShell.prompt_out=""'
203 ]
205 ]
204 cmdargs = ' '.join(default_argv() + prompt_opts + options)
206 cmdargs = ' '.join(default_argv() + prompt_opts + options)
205
207
206 _ip = get_ipython()
208 _ip = get_ipython()
207 test_dir = os.path.dirname(__file__)
209 test_dir = os.path.dirname(__file__)
208
210
209 ipython_cmd = find_cmd('ipython')
211 ipython_cmd = find_cmd('ipython')
210 # Absolute path for filename
212 # Absolute path for filename
211 full_fname = os.path.join(test_dir, fname)
213 full_fname = os.path.join(test_dir, fname)
212 full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname)
214 full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname)
213 #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg
215 #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg
214 return getoutputerror(full_cmd)
216 return getoutputerror(full_cmd)
215
217
216
218
217 def ipexec_validate(fname, expected_out, expected_err='',
219 def ipexec_validate(fname, expected_out, expected_err='',
218 options=None):
220 options=None):
219 """Utility to call 'ipython filename' and validate output/error.
221 """Utility to call 'ipython filename' and validate output/error.
220
222
221 This function raises an AssertionError if the validation fails.
223 This function raises an AssertionError if the validation fails.
222
224
223 Note that this starts IPython in a subprocess!
225 Note that this starts IPython in a subprocess!
224
226
225 Parameters
227 Parameters
226 ----------
228 ----------
227 fname : str
229 fname : str
228 Name of the file to be executed (should have .py or .ipy extension).
230 Name of the file to be executed (should have .py or .ipy extension).
229
231
230 expected_out : str
232 expected_out : str
231 Expected stdout of the process.
233 Expected stdout of the process.
232
234
233 expected_err : optional, str
235 expected_err : optional, str
234 Expected stderr of the process.
236 Expected stderr of the process.
235
237
236 options : optional, list
238 options : optional, list
237 Extra command-line flags to be passed to IPython.
239 Extra command-line flags to be passed to IPython.
238
240
239 Returns
241 Returns
240 -------
242 -------
241 None
243 None
242 """
244 """
243
245
244 import nose.tools as nt
246 import nose.tools as nt
245
247
246 out, err = ipexec(fname)
248 out, err = ipexec(fname)
247 #print 'OUT', out # dbg
249 #print 'OUT', out # dbg
248 #print 'ERR', err # dbg
250 #print 'ERR', err # dbg
249 # If there are any errors, we must check those befor stdout, as they may be
251 # If there are any errors, we must check those befor stdout, as they may be
250 # more informative than simply having an empty stdout.
252 # more informative than simply having an empty stdout.
251 if err:
253 if err:
252 if expected_err:
254 if expected_err:
253 nt.assert_equals(err.strip(), expected_err.strip())
255 nt.assert_equals(err.strip(), expected_err.strip())
254 else:
256 else:
255 raise ValueError('Running file %r produced error: %r' %
257 raise ValueError('Running file %r produced error: %r' %
256 (fname, err))
258 (fname, err))
257 # If no errors or output on stderr was expected, match stdout
259 # If no errors or output on stderr was expected, match stdout
258 nt.assert_equals(out.strip(), expected_out.strip())
260 nt.assert_equals(out.strip(), expected_out.strip())
259
261
260
262
261 class TempFileMixin(object):
263 class TempFileMixin(object):
262 """Utility class to create temporary Python/IPython files.
264 """Utility class to create temporary Python/IPython files.
263
265
264 Meant as a mixin class for test cases."""
266 Meant as a mixin class for test cases."""
265
267
266 def mktmp(self, src, ext='.py'):
268 def mktmp(self, src, ext='.py'):
267 """Make a valid python temp file."""
269 """Make a valid python temp file."""
268 fname, f = temp_pyfile(src, ext)
270 fname, f = temp_pyfile(src, ext)
269 self.tmpfile = f
271 self.tmpfile = f
270 self.fname = fname
272 self.fname = fname
271
273
272 def tearDown(self):
274 def tearDown(self):
273 if hasattr(self, 'tmpfile'):
275 if hasattr(self, 'tmpfile'):
274 # If the tmpfile wasn't made because of skipped tests, like in
276 # If the tmpfile wasn't made because of skipped tests, like in
275 # win32, there's nothing to cleanup.
277 # win32, there's nothing to cleanup.
276 self.tmpfile.close()
278 self.tmpfile.close()
277 try:
279 try:
278 os.unlink(self.fname)
280 os.unlink(self.fname)
279 except:
281 except:
280 # On Windows, even though we close the file, we still can't
282 # On Windows, even though we close the file, we still can't
281 # delete it. I have no clue why
283 # delete it. I have no clue why
282 pass
284 pass
283
285
284 pair_fail_msg = ("Testing function {0}\n\n"
286 pair_fail_msg = ("Testing function {0}\n\n"
285 "In:\n"
287 "In:\n"
286 " {1!r}\n"
288 " {1!r}\n"
287 "Expected:\n"
289 "Expected:\n"
288 " {2!r}\n"
290 " {2!r}\n"
289 "Got:\n"
291 "Got:\n"
290 " {3!r}\n")
292 " {3!r}\n")
291 def check_pairs(func, pairs):
293 def check_pairs(func, pairs):
292 """Utility function for the common case of checking a function with a
294 """Utility function for the common case of checking a function with a
293 sequence of input/output pairs.
295 sequence of input/output pairs.
294
296
295 Parameters
297 Parameters
296 ----------
298 ----------
297 func : callable
299 func : callable
298 The function to be tested. Should accept a single argument.
300 The function to be tested. Should accept a single argument.
299 pairs : iterable
301 pairs : iterable
300 A list of (input, expected_output) tuples.
302 A list of (input, expected_output) tuples.
301
303
302 Returns
304 Returns
303 -------
305 -------
304 None. Raises an AssertionError if any output does not match the expected
306 None. Raises an AssertionError if any output does not match the expected
305 value.
307 value.
306 """
308 """
307 for inp, expected in pairs:
309 for inp, expected in pairs:
308 out = func(inp)
310 out = func(inp)
309 assert out == expected, pair_fail_msg.format(func.func_name, inp, expected, out)
311 assert out == expected, pair_fail_msg.format(func.func_name, inp, expected, out)
312
313 @contextmanager
314 def mute_warn():
315 from IPython.utils import warn
316 save_warn = warn.warn
317 warn.warn = lambda *a, **kw: None
318 try:
319 yield
320 finally:
321 warn.warn = save_warn No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now