##// END OF EJS Templates
move default log setup to _log_default from init_logging...
MinRK -
Show More
@@ -1,508 +1,508 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 A base class for a configurable application.
3 A base class for a configurable application.
4
4
5 Authors:
5 Authors:
6
6
7 * Brian Granger
7 * Brian Granger
8 * Min RK
8 * Min RK
9 """
9 """
10
10
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12 # Copyright (C) 2008-2011 The IPython Development Team
12 # Copyright (C) 2008-2011 The IPython Development Team
13 #
13 #
14 # Distributed under the terms of the BSD License. The full license is in
14 # Distributed under the terms of the BSD License. The full license is in
15 # the file COPYING, distributed as part of this software.
15 # the file COPYING, distributed as part of this software.
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17
17
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19 # Imports
19 # Imports
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21
21
22 import logging
22 import logging
23 import os
23 import os
24 import re
24 import re
25 import sys
25 import sys
26 from copy import deepcopy
26 from copy import deepcopy
27 from collections import defaultdict
27 from collections import defaultdict
28
28
29 from IPython.external.decorator import decorator
29 from IPython.external.decorator import decorator
30
30
31 from IPython.config.configurable import SingletonConfigurable
31 from IPython.config.configurable import SingletonConfigurable
32 from IPython.config.loader import (
32 from IPython.config.loader import (
33 KVArgParseConfigLoader, PyFileConfigLoader, Config, ArgumentError, ConfigFileNotFound,
33 KVArgParseConfigLoader, PyFileConfigLoader, Config, ArgumentError, ConfigFileNotFound,
34 )
34 )
35
35
36 from IPython.utils.traitlets import (
36 from IPython.utils.traitlets import (
37 Unicode, List, Enum, Dict, Instance, TraitError
37 Unicode, List, Enum, Dict, Instance, TraitError
38 )
38 )
39 from IPython.utils.importstring import import_item
39 from IPython.utils.importstring import import_item
40 from IPython.utils.text import indent, wrap_paragraphs, dedent
40 from IPython.utils.text import indent, wrap_paragraphs, dedent
41
41
42 #-----------------------------------------------------------------------------
42 #-----------------------------------------------------------------------------
43 # function for re-wrapping a helpstring
43 # function for re-wrapping a helpstring
44 #-----------------------------------------------------------------------------
44 #-----------------------------------------------------------------------------
45
45
46 #-----------------------------------------------------------------------------
46 #-----------------------------------------------------------------------------
47 # Descriptions for the various sections
47 # Descriptions for the various sections
48 #-----------------------------------------------------------------------------
48 #-----------------------------------------------------------------------------
49
49
50 # merge flags&aliases into options
50 # merge flags&aliases into options
51 option_description = """
51 option_description = """
52 Arguments that take values are actually convenience aliases to full
52 Arguments that take values are actually convenience aliases to full
53 Configurables, whose aliases are listed on the help line. For more information
53 Configurables, whose aliases are listed on the help line. For more information
54 on full configurables, see '--help-all'.
54 on full configurables, see '--help-all'.
55 """.strip() # trim newlines of front and back
55 """.strip() # trim newlines of front and back
56
56
57 keyvalue_description = """
57 keyvalue_description = """
58 Parameters are set from command-line arguments of the form:
58 Parameters are set from command-line arguments of the form:
59 `--Class.trait=value`.
59 `--Class.trait=value`.
60 This line is evaluated in Python, so simple expressions are allowed, e.g.::
60 This line is evaluated in Python, so simple expressions are allowed, e.g.::
61 `--C.a='range(3)'` For setting C.a=[0,1,2].
61 `--C.a='range(3)'` For setting C.a=[0,1,2].
62 """.strip() # trim newlines of front and back
62 """.strip() # trim newlines of front and back
63
63
64 subcommand_description = """
64 subcommand_description = """
65 Subcommands are launched as `{app} cmd [args]`. For information on using
65 Subcommands are launched as `{app} cmd [args]`. For information on using
66 subcommand 'cmd', do: `{app} cmd -h`.
66 subcommand 'cmd', do: `{app} cmd -h`.
67 """.strip().format(app=os.path.basename(sys.argv[0]))
67 """.strip().format(app=os.path.basename(sys.argv[0]))
68 # get running program name
68 # get running program name
69
69
70 #-----------------------------------------------------------------------------
70 #-----------------------------------------------------------------------------
71 # Application class
71 # Application class
72 #-----------------------------------------------------------------------------
72 #-----------------------------------------------------------------------------
73
73
74 @decorator
74 @decorator
75 def catch_config_error(method, app, *args, **kwargs):
75 def catch_config_error(method, app, *args, **kwargs):
76 """Method decorator for catching invalid config (Trait/ArgumentErrors) during init.
76 """Method decorator for catching invalid config (Trait/ArgumentErrors) during init.
77
77
78 On a TraitError (generally caused by bad config), this will print the trait's
78 On a TraitError (generally caused by bad config), this will print the trait's
79 message, and exit the app.
79 message, and exit the app.
80
80
81 For use on init methods, to prevent invoking excepthook on invalid input.
81 For use on init methods, to prevent invoking excepthook on invalid input.
82 """
82 """
83 try:
83 try:
84 return method(app, *args, **kwargs)
84 return method(app, *args, **kwargs)
85 except (TraitError, ArgumentError) as e:
85 except (TraitError, ArgumentError) as e:
86 app.print_description()
86 app.print_description()
87 app.print_help()
87 app.print_help()
88 app.print_examples()
88 app.print_examples()
89 app.log.fatal("Bad config encountered during initialization:")
89 app.log.fatal("Bad config encountered during initialization:")
90 app.log.fatal(str(e))
90 app.log.fatal(str(e))
91 app.log.debug("Config at the time: %s", app.config)
91 app.log.debug("Config at the time: %s", app.config)
92 app.exit(1)
92 app.exit(1)
93
93
94
94
95 class ApplicationError(Exception):
95 class ApplicationError(Exception):
96 pass
96 pass
97
97
98
98
99 class Application(SingletonConfigurable):
99 class Application(SingletonConfigurable):
100 """A singleton application with full configuration support."""
100 """A singleton application with full configuration support."""
101
101
102 # The name of the application, will usually match the name of the command
102 # The name of the application, will usually match the name of the command
103 # line application
103 # line application
104 name = Unicode(u'application')
104 name = Unicode(u'application')
105
105
106 # The description of the application that is printed at the beginning
106 # The description of the application that is printed at the beginning
107 # of the help.
107 # of the help.
108 description = Unicode(u'This is an application.')
108 description = Unicode(u'This is an application.')
109 # default section descriptions
109 # default section descriptions
110 option_description = Unicode(option_description)
110 option_description = Unicode(option_description)
111 keyvalue_description = Unicode(keyvalue_description)
111 keyvalue_description = Unicode(keyvalue_description)
112 subcommand_description = Unicode(subcommand_description)
112 subcommand_description = Unicode(subcommand_description)
113
113
114 # The usage and example string that goes at the end of the help string.
114 # The usage and example string that goes at the end of the help string.
115 examples = Unicode()
115 examples = Unicode()
116
116
117 # A sequence of Configurable subclasses whose config=True attributes will
117 # A sequence of Configurable subclasses whose config=True attributes will
118 # be exposed at the command line.
118 # be exposed at the command line.
119 classes = List([])
119 classes = List([])
120
120
121 # The version string of this application.
121 # The version string of this application.
122 version = Unicode(u'0.0')
122 version = Unicode(u'0.0')
123
123
124 # The log level for the application
124 # The log level for the application
125 log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
125 log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
126 default_value=logging.WARN,
126 default_value=logging.WARN,
127 config=True,
127 config=True,
128 help="Set the log level by value or name.")
128 help="Set the log level by value or name.")
129 def _log_level_changed(self, name, old, new):
129 def _log_level_changed(self, name, old, new):
130 """Adjust the log level when log_level is set."""
130 """Adjust the log level when log_level is set."""
131 if isinstance(new, basestring):
131 if isinstance(new, basestring):
132 new = getattr(logging, new)
132 new = getattr(logging, new)
133 self.log_level = new
133 self.log_level = new
134 self.log.setLevel(new)
134 self.log.setLevel(new)
135
136 log = Instance(logging.Logger)
137 def _log_default(self):
138 """Start logging for this application.
139
140 The default is to log to stdout using a StreaHandler. The log level
141 starts at loggin.WARN, but this can be adjusted by setting the
142 ``log_level`` attribute.
143 """
144 log = logging.getLogger(self.__class__.__name__)
145 log.setLevel(self.log_level)
146 if sys.executable.endswith('pythonw.exe'):
147 # this should really go to a file, but file-logging is only
148 # hooked up in parallel applications
149 _log_handler = logging.StreamHandler(open(os.devnull, 'w'))
150 else:
151 _log_handler = logging.StreamHandler()
152 _log_formatter = logging.Formatter("[%(name)s] %(message)s")
153 _log_handler.setFormatter(_log_formatter)
154 log.addHandler(_log_handler)
155 return log
135
156
136 # the alias map for configurables
157 # the alias map for configurables
137 aliases = Dict({'log-level' : 'Application.log_level'})
158 aliases = Dict({'log-level' : 'Application.log_level'})
138
159
139 # flags for loading Configurables or store_const style flags
160 # flags for loading Configurables or store_const style flags
140 # flags are loaded from this dict by '--key' flags
161 # flags are loaded from this dict by '--key' flags
141 # this must be a dict of two-tuples, the first element being the Config/dict
162 # this must be a dict of two-tuples, the first element being the Config/dict
142 # and the second being the help string for the flag
163 # and the second being the help string for the flag
143 flags = Dict()
164 flags = Dict()
144 def _flags_changed(self, name, old, new):
165 def _flags_changed(self, name, old, new):
145 """ensure flags dict is valid"""
166 """ensure flags dict is valid"""
146 for key,value in new.iteritems():
167 for key,value in new.iteritems():
147 assert len(value) == 2, "Bad flag: %r:%s"%(key,value)
168 assert len(value) == 2, "Bad flag: %r:%s"%(key,value)
148 assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value)
169 assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value)
149 assert isinstance(value[1], basestring), "Bad flag: %r:%s"%(key,value)
170 assert isinstance(value[1], basestring), "Bad flag: %r:%s"%(key,value)
150
171
151
172
152 # subcommands for launching other applications
173 # subcommands for launching other applications
153 # if this is not empty, this will be a parent Application
174 # if this is not empty, this will be a parent Application
154 # this must be a dict of two-tuples,
175 # this must be a dict of two-tuples,
155 # the first element being the application class/import string
176 # the first element being the application class/import string
156 # and the second being the help string for the subcommand
177 # and the second being the help string for the subcommand
157 subcommands = Dict()
178 subcommands = Dict()
158 # parse_command_line will initialize a subapp, if requested
179 # parse_command_line will initialize a subapp, if requested
159 subapp = Instance('IPython.config.application.Application', allow_none=True)
180 subapp = Instance('IPython.config.application.Application', allow_none=True)
160
181
161 # extra command-line arguments that don't set config values
182 # extra command-line arguments that don't set config values
162 extra_args = List(Unicode)
183 extra_args = List(Unicode)
163
184
164
185
165 def __init__(self, **kwargs):
186 def __init__(self, **kwargs):
166 SingletonConfigurable.__init__(self, **kwargs)
187 SingletonConfigurable.__init__(self, **kwargs)
167 # Ensure my class is in self.classes, so my attributes appear in command line
188 # Ensure my class is in self.classes, so my attributes appear in command line
168 # options and config files.
189 # options and config files.
169 if self.__class__ not in self.classes:
190 if self.__class__ not in self.classes:
170 self.classes.insert(0, self.__class__)
191 self.classes.insert(0, self.__class__)
171
192
172 self.init_logging()
173
174 def _config_changed(self, name, old, new):
193 def _config_changed(self, name, old, new):
175 SingletonConfigurable._config_changed(self, name, old, new)
194 SingletonConfigurable._config_changed(self, name, old, new)
176 self.log.debug('Config changed:')
195 self.log.debug('Config changed:')
177 self.log.debug(repr(new))
196 self.log.debug(repr(new))
178
197
179 def init_logging(self):
180 """Start logging for this application.
181
182 The default is to log to stdout using a StreaHandler. The log level
183 starts at loggin.WARN, but this can be adjusted by setting the
184 ``log_level`` attribute.
185 """
186 self.log = logging.getLogger(self.__class__.__name__)
187 self.log.setLevel(self.log_level)
188 if sys.executable.endswith('pythonw.exe'):
189 # this should really go to a file, but file-logging is only
190 # hooked up in parallel applications
191 self._log_handler = logging.StreamHandler(open(os.devnull, 'w'))
192 else:
193 self._log_handler = logging.StreamHandler()
194 self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
195 self._log_handler.setFormatter(self._log_formatter)
196 self.log.addHandler(self._log_handler)
197
198 @catch_config_error
198 @catch_config_error
199 def initialize(self, argv=None):
199 def initialize(self, argv=None):
200 """Do the basic steps to configure me.
200 """Do the basic steps to configure me.
201
201
202 Override in subclasses.
202 Override in subclasses.
203 """
203 """
204 self.parse_command_line(argv)
204 self.parse_command_line(argv)
205
205
206
206
207 def start(self):
207 def start(self):
208 """Start the app mainloop.
208 """Start the app mainloop.
209
209
210 Override in subclasses.
210 Override in subclasses.
211 """
211 """
212 if self.subapp is not None:
212 if self.subapp is not None:
213 return self.subapp.start()
213 return self.subapp.start()
214
214
215 def print_alias_help(self):
215 def print_alias_help(self):
216 """Print the alias part of the help."""
216 """Print the alias part of the help."""
217 if not self.aliases:
217 if not self.aliases:
218 return
218 return
219
219
220 lines = []
220 lines = []
221 classdict = {}
221 classdict = {}
222 for cls in self.classes:
222 for cls in self.classes:
223 # include all parents (up to, but excluding Configurable) in available names
223 # include all parents (up to, but excluding Configurable) in available names
224 for c in cls.mro()[:-3]:
224 for c in cls.mro()[:-3]:
225 classdict[c.__name__] = c
225 classdict[c.__name__] = c
226
226
227 for alias, longname in self.aliases.iteritems():
227 for alias, longname in self.aliases.iteritems():
228 classname, traitname = longname.split('.',1)
228 classname, traitname = longname.split('.',1)
229 cls = classdict[classname]
229 cls = classdict[classname]
230
230
231 trait = cls.class_traits(config=True)[traitname]
231 trait = cls.class_traits(config=True)[traitname]
232 help = cls.class_get_trait_help(trait).splitlines()
232 help = cls.class_get_trait_help(trait).splitlines()
233 # reformat first line
233 # reformat first line
234 help[0] = help[0].replace(longname, alias) + ' (%s)'%longname
234 help[0] = help[0].replace(longname, alias) + ' (%s)'%longname
235 if len(alias) == 1:
235 if len(alias) == 1:
236 help[0] = help[0].replace('--%s='%alias, '-%s '%alias)
236 help[0] = help[0].replace('--%s='%alias, '-%s '%alias)
237 lines.extend(help)
237 lines.extend(help)
238 # lines.append('')
238 # lines.append('')
239 print os.linesep.join(lines)
239 print os.linesep.join(lines)
240
240
241 def print_flag_help(self):
241 def print_flag_help(self):
242 """Print the flag part of the help."""
242 """Print the flag part of the help."""
243 if not self.flags:
243 if not self.flags:
244 return
244 return
245
245
246 lines = []
246 lines = []
247 for m, (cfg,help) in self.flags.iteritems():
247 for m, (cfg,help) in self.flags.iteritems():
248 prefix = '--' if len(m) > 1 else '-'
248 prefix = '--' if len(m) > 1 else '-'
249 lines.append(prefix+m)
249 lines.append(prefix+m)
250 lines.append(indent(dedent(help.strip())))
250 lines.append(indent(dedent(help.strip())))
251 # lines.append('')
251 # lines.append('')
252 print os.linesep.join(lines)
252 print os.linesep.join(lines)
253
253
254 def print_options(self):
254 def print_options(self):
255 if not self.flags and not self.aliases:
255 if not self.flags and not self.aliases:
256 return
256 return
257 lines = ['Options']
257 lines = ['Options']
258 lines.append('-'*len(lines[0]))
258 lines.append('-'*len(lines[0]))
259 lines.append('')
259 lines.append('')
260 for p in wrap_paragraphs(self.option_description):
260 for p in wrap_paragraphs(self.option_description):
261 lines.append(p)
261 lines.append(p)
262 lines.append('')
262 lines.append('')
263 print os.linesep.join(lines)
263 print os.linesep.join(lines)
264 self.print_flag_help()
264 self.print_flag_help()
265 self.print_alias_help()
265 self.print_alias_help()
266 print
266 print
267
267
268 def print_subcommands(self):
268 def print_subcommands(self):
269 """Print the subcommand part of the help."""
269 """Print the subcommand part of the help."""
270 if not self.subcommands:
270 if not self.subcommands:
271 return
271 return
272
272
273 lines = ["Subcommands"]
273 lines = ["Subcommands"]
274 lines.append('-'*len(lines[0]))
274 lines.append('-'*len(lines[0]))
275 lines.append('')
275 lines.append('')
276 for p in wrap_paragraphs(self.subcommand_description):
276 for p in wrap_paragraphs(self.subcommand_description):
277 lines.append(p)
277 lines.append(p)
278 lines.append('')
278 lines.append('')
279 for subc, (cls, help) in self.subcommands.iteritems():
279 for subc, (cls, help) in self.subcommands.iteritems():
280 lines.append(subc)
280 lines.append(subc)
281 if help:
281 if help:
282 lines.append(indent(dedent(help.strip())))
282 lines.append(indent(dedent(help.strip())))
283 lines.append('')
283 lines.append('')
284 print os.linesep.join(lines)
284 print os.linesep.join(lines)
285
285
286 def print_help(self, classes=False):
286 def print_help(self, classes=False):
287 """Print the help for each Configurable class in self.classes.
287 """Print the help for each Configurable class in self.classes.
288
288
289 If classes=False (the default), only flags and aliases are printed.
289 If classes=False (the default), only flags and aliases are printed.
290 """
290 """
291 self.print_subcommands()
291 self.print_subcommands()
292 self.print_options()
292 self.print_options()
293
293
294 if classes:
294 if classes:
295 if self.classes:
295 if self.classes:
296 print "Class parameters"
296 print "Class parameters"
297 print "----------------"
297 print "----------------"
298 print
298 print
299 for p in wrap_paragraphs(self.keyvalue_description):
299 for p in wrap_paragraphs(self.keyvalue_description):
300 print p
300 print p
301 print
301 print
302
302
303 for cls in self.classes:
303 for cls in self.classes:
304 cls.class_print_help()
304 cls.class_print_help()
305 print
305 print
306 else:
306 else:
307 print "To see all available configurables, use `--help-all`"
307 print "To see all available configurables, use `--help-all`"
308 print
308 print
309
309
310 def print_description(self):
310 def print_description(self):
311 """Print the application description."""
311 """Print the application description."""
312 for p in wrap_paragraphs(self.description):
312 for p in wrap_paragraphs(self.description):
313 print p
313 print p
314 print
314 print
315
315
316 def print_examples(self):
316 def print_examples(self):
317 """Print usage and examples.
317 """Print usage and examples.
318
318
319 This usage string goes at the end of the command line help string
319 This usage string goes at the end of the command line help string
320 and should contain examples of the application's usage.
320 and should contain examples of the application's usage.
321 """
321 """
322 if self.examples:
322 if self.examples:
323 print "Examples"
323 print "Examples"
324 print "--------"
324 print "--------"
325 print
325 print
326 print indent(dedent(self.examples.strip()))
326 print indent(dedent(self.examples.strip()))
327 print
327 print
328
328
329 def print_version(self):
329 def print_version(self):
330 """Print the version string."""
330 """Print the version string."""
331 print self.version
331 print self.version
332
332
333 def update_config(self, config):
333 def update_config(self, config):
334 """Fire the traits events when the config is updated."""
334 """Fire the traits events when the config is updated."""
335 # Save a copy of the current config.
335 # Save a copy of the current config.
336 newconfig = deepcopy(self.config)
336 newconfig = deepcopy(self.config)
337 # Merge the new config into the current one.
337 # Merge the new config into the current one.
338 newconfig._merge(config)
338 newconfig._merge(config)
339 # Save the combined config as self.config, which triggers the traits
339 # Save the combined config as self.config, which triggers the traits
340 # events.
340 # events.
341 self.config = newconfig
341 self.config = newconfig
342
342
343 @catch_config_error
343 @catch_config_error
344 def initialize_subcommand(self, subc, argv=None):
344 def initialize_subcommand(self, subc, argv=None):
345 """Initialize a subcommand with argv."""
345 """Initialize a subcommand with argv."""
346 subapp,help = self.subcommands.get(subc)
346 subapp,help = self.subcommands.get(subc)
347
347
348 if isinstance(subapp, basestring):
348 if isinstance(subapp, basestring):
349 subapp = import_item(subapp)
349 subapp = import_item(subapp)
350
350
351 # clear existing instances
351 # clear existing instances
352 self.__class__.clear_instance()
352 self.__class__.clear_instance()
353 # instantiate
353 # instantiate
354 self.subapp = subapp.instance()
354 self.subapp = subapp.instance()
355 # and initialize subapp
355 # and initialize subapp
356 self.subapp.initialize(argv)
356 self.subapp.initialize(argv)
357
357
358 def flatten_flags(self):
358 def flatten_flags(self):
359 """flatten flags and aliases, so cl-args override as expected.
359 """flatten flags and aliases, so cl-args override as expected.
360
360
361 This prevents issues such as an alias pointing to InteractiveShell,
361 This prevents issues such as an alias pointing to InteractiveShell,
362 but a config file setting the same trait in TerminalInteraciveShell
362 but a config file setting the same trait in TerminalInteraciveShell
363 getting inappropriate priority over the command-line arg.
363 getting inappropriate priority over the command-line arg.
364
364
365 Only aliases with exactly one descendent in the class list
365 Only aliases with exactly one descendent in the class list
366 will be promoted.
366 will be promoted.
367
367
368 """
368 """
369 # build a tree of classes in our list that inherit from a particular
369 # build a tree of classes in our list that inherit from a particular
370 # it will be a dict by parent classname of classes in our list
370 # it will be a dict by parent classname of classes in our list
371 # that are descendents
371 # that are descendents
372 mro_tree = defaultdict(list)
372 mro_tree = defaultdict(list)
373 for cls in self.classes:
373 for cls in self.classes:
374 clsname = cls.__name__
374 clsname = cls.__name__
375 for parent in cls.mro()[1:-3]:
375 for parent in cls.mro()[1:-3]:
376 # exclude cls itself and Configurable,HasTraits,object
376 # exclude cls itself and Configurable,HasTraits,object
377 mro_tree[parent.__name__].append(clsname)
377 mro_tree[parent.__name__].append(clsname)
378 # flatten aliases, which have the form:
378 # flatten aliases, which have the form:
379 # { 'alias' : 'Class.trait' }
379 # { 'alias' : 'Class.trait' }
380 aliases = {}
380 aliases = {}
381 for alias, cls_trait in self.aliases.iteritems():
381 for alias, cls_trait in self.aliases.iteritems():
382 cls,trait = cls_trait.split('.',1)
382 cls,trait = cls_trait.split('.',1)
383 children = mro_tree[cls]
383 children = mro_tree[cls]
384 if len(children) == 1:
384 if len(children) == 1:
385 # exactly one descendent, promote alias
385 # exactly one descendent, promote alias
386 cls = children[0]
386 cls = children[0]
387 aliases[alias] = '.'.join([cls,trait])
387 aliases[alias] = '.'.join([cls,trait])
388
388
389 # flatten flags, which are of the form:
389 # flatten flags, which are of the form:
390 # { 'key' : ({'Cls' : {'trait' : value}}, 'help')}
390 # { 'key' : ({'Cls' : {'trait' : value}}, 'help')}
391 flags = {}
391 flags = {}
392 for key, (flagdict, help) in self.flags.iteritems():
392 for key, (flagdict, help) in self.flags.iteritems():
393 newflag = {}
393 newflag = {}
394 for cls, subdict in flagdict.iteritems():
394 for cls, subdict in flagdict.iteritems():
395 children = mro_tree[cls]
395 children = mro_tree[cls]
396 # exactly one descendent, promote flag section
396 # exactly one descendent, promote flag section
397 if len(children) == 1:
397 if len(children) == 1:
398 cls = children[0]
398 cls = children[0]
399 newflag[cls] = subdict
399 newflag[cls] = subdict
400 flags[key] = (newflag, help)
400 flags[key] = (newflag, help)
401 return flags, aliases
401 return flags, aliases
402
402
403 @catch_config_error
403 @catch_config_error
404 def parse_command_line(self, argv=None):
404 def parse_command_line(self, argv=None):
405 """Parse the command line arguments."""
405 """Parse the command line arguments."""
406 argv = sys.argv[1:] if argv is None else argv
406 argv = sys.argv[1:] if argv is None else argv
407
407
408 if argv and argv[0] == 'help':
408 if argv and argv[0] == 'help':
409 # turn `ipython help notebook` into `ipython notebook -h`
409 # turn `ipython help notebook` into `ipython notebook -h`
410 argv = argv[1:] + ['-h']
410 argv = argv[1:] + ['-h']
411
411
412 if self.subcommands and len(argv) > 0:
412 if self.subcommands and len(argv) > 0:
413 # we have subcommands, and one may have been specified
413 # we have subcommands, and one may have been specified
414 subc, subargv = argv[0], argv[1:]
414 subc, subargv = argv[0], argv[1:]
415 if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands:
415 if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands:
416 # it's a subcommand, and *not* a flag or class parameter
416 # it's a subcommand, and *not* a flag or class parameter
417 return self.initialize_subcommand(subc, subargv)
417 return self.initialize_subcommand(subc, subargv)
418
418
419 if '-h' in argv or '--help' in argv or '--help-all' in argv:
419 if '-h' in argv or '--help' in argv or '--help-all' in argv:
420 self.print_description()
420 self.print_description()
421 self.print_help('--help-all' in argv)
421 self.print_help('--help-all' in argv)
422 self.print_examples()
422 self.print_examples()
423 self.exit(0)
423 self.exit(0)
424
424
425 if '--version' in argv or '-V' in argv:
425 if '--version' in argv or '-V' in argv:
426 self.print_version()
426 self.print_version()
427 self.exit(0)
427 self.exit(0)
428
428
429 # flatten flags&aliases, so cl-args get appropriate priority:
429 # flatten flags&aliases, so cl-args get appropriate priority:
430 flags,aliases = self.flatten_flags()
430 flags,aliases = self.flatten_flags()
431
431
432 loader = KVArgParseConfigLoader(argv=argv, aliases=aliases,
432 loader = KVArgParseConfigLoader(argv=argv, aliases=aliases,
433 flags=flags)
433 flags=flags)
434 config = loader.load_config()
434 config = loader.load_config()
435 self.update_config(config)
435 self.update_config(config)
436 # store unparsed args in extra_args
436 # store unparsed args in extra_args
437 self.extra_args = loader.extra_args
437 self.extra_args = loader.extra_args
438
438
439 @catch_config_error
439 @catch_config_error
440 def load_config_file(self, filename, path=None):
440 def load_config_file(self, filename, path=None):
441 """Load a .py based config file by filename and path."""
441 """Load a .py based config file by filename and path."""
442 loader = PyFileConfigLoader(filename, path=path)
442 loader = PyFileConfigLoader(filename, path=path)
443 try:
443 try:
444 config = loader.load_config()
444 config = loader.load_config()
445 except ConfigFileNotFound:
445 except ConfigFileNotFound:
446 # problem finding the file, raise
446 # problem finding the file, raise
447 raise
447 raise
448 except Exception:
448 except Exception:
449 # try to get the full filename, but it will be empty in the
449 # try to get the full filename, but it will be empty in the
450 # unlikely event that the error raised before filefind finished
450 # unlikely event that the error raised before filefind finished
451 filename = loader.full_filename or filename
451 filename = loader.full_filename or filename
452 # problem while running the file
452 # problem while running the file
453 self.log.error("Exception while loading config file %s",
453 self.log.error("Exception while loading config file %s",
454 filename, exc_info=True)
454 filename, exc_info=True)
455 else:
455 else:
456 self.log.debug("Loaded config file: %s", loader.full_filename)
456 self.log.debug("Loaded config file: %s", loader.full_filename)
457 self.update_config(config)
457 self.update_config(config)
458
458
459 def generate_config_file(self):
459 def generate_config_file(self):
460 """generate default config file from Configurables"""
460 """generate default config file from Configurables"""
461 lines = ["# Configuration file for %s."%self.name]
461 lines = ["# Configuration file for %s."%self.name]
462 lines.append('')
462 lines.append('')
463 lines.append('c = get_config()')
463 lines.append('c = get_config()')
464 lines.append('')
464 lines.append('')
465 for cls in self.classes:
465 for cls in self.classes:
466 lines.append(cls.class_config_section())
466 lines.append(cls.class_config_section())
467 return '\n'.join(lines)
467 return '\n'.join(lines)
468
468
469 def exit(self, exit_status=0):
469 def exit(self, exit_status=0):
470 self.log.debug("Exiting application: %s" % self.name)
470 self.log.debug("Exiting application: %s" % self.name)
471 sys.exit(exit_status)
471 sys.exit(exit_status)
472
472
473 #-----------------------------------------------------------------------------
473 #-----------------------------------------------------------------------------
474 # utility functions, for convenience
474 # utility functions, for convenience
475 #-----------------------------------------------------------------------------
475 #-----------------------------------------------------------------------------
476
476
477 def boolean_flag(name, configurable, set_help='', unset_help=''):
477 def boolean_flag(name, configurable, set_help='', unset_help=''):
478 """Helper for building basic --trait, --no-trait flags.
478 """Helper for building basic --trait, --no-trait flags.
479
479
480 Parameters
480 Parameters
481 ----------
481 ----------
482
482
483 name : str
483 name : str
484 The name of the flag.
484 The name of the flag.
485 configurable : str
485 configurable : str
486 The 'Class.trait' string of the trait to be set/unset with the flag
486 The 'Class.trait' string of the trait to be set/unset with the flag
487 set_help : unicode
487 set_help : unicode
488 help string for --name flag
488 help string for --name flag
489 unset_help : unicode
489 unset_help : unicode
490 help string for --no-name flag
490 help string for --no-name flag
491
491
492 Returns
492 Returns
493 -------
493 -------
494
494
495 cfg : dict
495 cfg : dict
496 A dict with two keys: 'name', and 'no-name', for setting and unsetting
496 A dict with two keys: 'name', and 'no-name', for setting and unsetting
497 the trait, respectively.
497 the trait, respectively.
498 """
498 """
499 # default helpstrings
499 # default helpstrings
500 set_help = set_help or "set %s=True"%configurable
500 set_help = set_help or "set %s=True"%configurable
501 unset_help = unset_help or "set %s=False"%configurable
501 unset_help = unset_help or "set %s=False"%configurable
502
502
503 cls,trait = configurable.split('.')
503 cls,trait = configurable.split('.')
504
504
505 setter = {cls : {trait : True}}
505 setter = {cls : {trait : True}}
506 unsetter = {cls : {trait : False}}
506 unsetter = {cls : {trait : False}}
507 return {name : (setter, set_help), 'no-'+name : (unsetter, unset_help)}
507 return {name : (setter, set_help), 'no-'+name : (unsetter, unset_help)}
508
508
@@ -1,174 +1,172 b''
1 """Manage IPython.parallel clusters in the notebook.
1 """Manage IPython.parallel clusters in the notebook.
2
2
3 Authors:
3 Authors:
4
4
5 * Brian Granger
5 * Brian Granger
6 """
6 """
7
7
8 #-----------------------------------------------------------------------------
8 #-----------------------------------------------------------------------------
9 # Copyright (C) 2008-2011 The IPython Development Team
9 # Copyright (C) 2008-2011 The IPython Development Team
10 #
10 #
11 # Distributed under the terms of the BSD License. The full license is in
11 # Distributed under the terms of the BSD License. The full license is in
12 # the file COPYING, distributed as part of this software.
12 # the file COPYING, distributed as part of this software.
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14
14
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16 # Imports
16 # Imports
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 import os
19 import os
20
20
21 from tornado import web
21 from tornado import web
22 from zmq.eventloop import ioloop
22 from zmq.eventloop import ioloop
23
23
24 from IPython.config.configurable import LoggingConfigurable
24 from IPython.config.configurable import LoggingConfigurable
25 from IPython.config.loader import load_pyconfig_files
25 from IPython.config.loader import load_pyconfig_files
26 from IPython.utils.traitlets import Dict, Instance, CFloat
26 from IPython.utils.traitlets import Dict, Instance, CFloat
27 from IPython.parallel.apps.ipclusterapp import IPClusterStart
27 from IPython.parallel.apps.ipclusterapp import IPClusterStart
28 from IPython.core.profileapp import list_profiles_in
28 from IPython.core.profileapp import list_profiles_in
29 from IPython.core.profiledir import ProfileDir
29 from IPython.core.profiledir import ProfileDir
30 from IPython.utils.path import get_ipython_dir
30 from IPython.utils.path import get_ipython_dir
31 from IPython.utils.sysinfo import num_cpus
31 from IPython.utils.sysinfo import num_cpus
32
32
33
33
34 #-----------------------------------------------------------------------------
34 #-----------------------------------------------------------------------------
35 # Classes
35 # Classes
36 #-----------------------------------------------------------------------------
36 #-----------------------------------------------------------------------------
37
37
38
38
39 class DummyIPClusterStart(IPClusterStart):
39 class DummyIPClusterStart(IPClusterStart):
40 """Dummy subclass to skip init steps that conflict with global app.
40 """Dummy subclass to skip init steps that conflict with global app.
41
41
42 Instantiating and initializing this class should result in fully configured
42 Instantiating and initializing this class should result in fully configured
43 launchers, but no other side effects or state.
43 launchers, but no other side effects or state.
44 """
44 """
45
45
46 def init_signal(self):
46 def init_signal(self):
47 pass
47 pass
48 def init_logging(self):
49 pass
50 def reinit_logging(self):
48 def reinit_logging(self):
51 pass
49 pass
52
50
53
51
54 class ClusterManager(LoggingConfigurable):
52 class ClusterManager(LoggingConfigurable):
55
53
56 profiles = Dict()
54 profiles = Dict()
57
55
58 delay = CFloat(1., config=True,
56 delay = CFloat(1., config=True,
59 help="delay (in s) between starting the controller and the engines")
57 help="delay (in s) between starting the controller and the engines")
60
58
61 loop = Instance('zmq.eventloop.ioloop.IOLoop')
59 loop = Instance('zmq.eventloop.ioloop.IOLoop')
62 def _loop_default(self):
60 def _loop_default(self):
63 from zmq.eventloop.ioloop import IOLoop
61 from zmq.eventloop.ioloop import IOLoop
64 return IOLoop.instance()
62 return IOLoop.instance()
65
63
66 def build_launchers(self, profile_dir):
64 def build_launchers(self, profile_dir):
67 starter = DummyIPClusterStart(log=self.log)
65 starter = DummyIPClusterStart(log=self.log)
68 starter.initialize(['--profile-dir', profile_dir])
66 starter.initialize(['--profile-dir', profile_dir])
69 cl = starter.controller_launcher
67 cl = starter.controller_launcher
70 esl = starter.engine_launcher
68 esl = starter.engine_launcher
71 n = starter.n
69 n = starter.n
72 return cl, esl, n
70 return cl, esl, n
73
71
74 def get_profile_dir(self, name, path):
72 def get_profile_dir(self, name, path):
75 p = ProfileDir.find_profile_dir_by_name(path,name=name)
73 p = ProfileDir.find_profile_dir_by_name(path,name=name)
76 return p.location
74 return p.location
77
75
78 def update_profiles(self):
76 def update_profiles(self):
79 """List all profiles in the ipython_dir and cwd.
77 """List all profiles in the ipython_dir and cwd.
80 """
78 """
81 for path in [get_ipython_dir(), os.getcwdu()]:
79 for path in [get_ipython_dir(), os.getcwdu()]:
82 for profile in list_profiles_in(path):
80 for profile in list_profiles_in(path):
83 pd = self.get_profile_dir(profile, path)
81 pd = self.get_profile_dir(profile, path)
84 if profile not in self.profiles:
82 if profile not in self.profiles:
85 self.log.debug("Overwriting profile %s" % profile)
83 self.log.debug("Overwriting profile %s" % profile)
86 self.profiles[profile] = {
84 self.profiles[profile] = {
87 'profile': profile,
85 'profile': profile,
88 'profile_dir': pd,
86 'profile_dir': pd,
89 'status': 'stopped'
87 'status': 'stopped'
90 }
88 }
91
89
92 def list_profiles(self):
90 def list_profiles(self):
93 self.update_profiles()
91 self.update_profiles()
94 result = [self.profile_info(p) for p in sorted(self.profiles.keys())]
92 result = [self.profile_info(p) for p in sorted(self.profiles.keys())]
95 return result
93 return result
96
94
97 def check_profile(self, profile):
95 def check_profile(self, profile):
98 if profile not in self.profiles:
96 if profile not in self.profiles:
99 raise web.HTTPError(404, u'profile not found')
97 raise web.HTTPError(404, u'profile not found')
100
98
101 def profile_info(self, profile):
99 def profile_info(self, profile):
102 self.check_profile(profile)
100 self.check_profile(profile)
103 result = {}
101 result = {}
104 data = self.profiles.get(profile)
102 data = self.profiles.get(profile)
105 result['profile'] = profile
103 result['profile'] = profile
106 result['profile_dir'] = data['profile_dir']
104 result['profile_dir'] = data['profile_dir']
107 result['status'] = data['status']
105 result['status'] = data['status']
108 if 'n' in data:
106 if 'n' in data:
109 result['n'] = data['n']
107 result['n'] = data['n']
110 return result
108 return result
111
109
112 def start_cluster(self, profile, n=None):
110 def start_cluster(self, profile, n=None):
113 """Start a cluster for a given profile."""
111 """Start a cluster for a given profile."""
114 self.check_profile(profile)
112 self.check_profile(profile)
115 data = self.profiles[profile]
113 data = self.profiles[profile]
116 if data['status'] == 'running':
114 if data['status'] == 'running':
117 raise web.HTTPError(409, u'cluster already running')
115 raise web.HTTPError(409, u'cluster already running')
118 cl, esl, default_n = self.build_launchers(data['profile_dir'])
116 cl, esl, default_n = self.build_launchers(data['profile_dir'])
119 n = n if n is not None else default_n
117 n = n if n is not None else default_n
120 def clean_data():
118 def clean_data():
121 data.pop('controller_launcher',None)
119 data.pop('controller_launcher',None)
122 data.pop('engine_set_launcher',None)
120 data.pop('engine_set_launcher',None)
123 data.pop('n',None)
121 data.pop('n',None)
124 data['status'] = 'stopped'
122 data['status'] = 'stopped'
125 def engines_stopped(r):
123 def engines_stopped(r):
126 self.log.debug('Engines stopped')
124 self.log.debug('Engines stopped')
127 if cl.running:
125 if cl.running:
128 cl.stop()
126 cl.stop()
129 clean_data()
127 clean_data()
130 esl.on_stop(engines_stopped)
128 esl.on_stop(engines_stopped)
131 def controller_stopped(r):
129 def controller_stopped(r):
132 self.log.debug('Controller stopped')
130 self.log.debug('Controller stopped')
133 if esl.running:
131 if esl.running:
134 esl.stop()
132 esl.stop()
135 clean_data()
133 clean_data()
136 cl.on_stop(controller_stopped)
134 cl.on_stop(controller_stopped)
137
135
138 dc = ioloop.DelayedCallback(lambda: cl.start(), 0, self.loop)
136 dc = ioloop.DelayedCallback(lambda: cl.start(), 0, self.loop)
139 dc.start()
137 dc.start()
140 dc = ioloop.DelayedCallback(lambda: esl.start(n), 1000*self.delay, self.loop)
138 dc = ioloop.DelayedCallback(lambda: esl.start(n), 1000*self.delay, self.loop)
141 dc.start()
139 dc.start()
142
140
143 self.log.debug('Cluster started')
141 self.log.debug('Cluster started')
144 data['controller_launcher'] = cl
142 data['controller_launcher'] = cl
145 data['engine_set_launcher'] = esl
143 data['engine_set_launcher'] = esl
146 data['n'] = n
144 data['n'] = n
147 data['status'] = 'running'
145 data['status'] = 'running'
148 return self.profile_info(profile)
146 return self.profile_info(profile)
149
147
150 def stop_cluster(self, profile):
148 def stop_cluster(self, profile):
151 """Stop a cluster for a given profile."""
149 """Stop a cluster for a given profile."""
152 self.check_profile(profile)
150 self.check_profile(profile)
153 data = self.profiles[profile]
151 data = self.profiles[profile]
154 if data['status'] == 'stopped':
152 if data['status'] == 'stopped':
155 raise web.HTTPError(409, u'cluster not running')
153 raise web.HTTPError(409, u'cluster not running')
156 data = self.profiles[profile]
154 data = self.profiles[profile]
157 cl = data['controller_launcher']
155 cl = data['controller_launcher']
158 esl = data['engine_set_launcher']
156 esl = data['engine_set_launcher']
159 if cl.running:
157 if cl.running:
160 cl.stop()
158 cl.stop()
161 if esl.running:
159 if esl.running:
162 esl.stop()
160 esl.stop()
163 # Return a temp info dict, the real one is updated in the on_stop
161 # Return a temp info dict, the real one is updated in the on_stop
164 # logic above.
162 # logic above.
165 result = {
163 result = {
166 'profile': data['profile'],
164 'profile': data['profile'],
167 'profile_dir': data['profile_dir'],
165 'profile_dir': data['profile_dir'],
168 'status': 'stopped'
166 'status': 'stopped'
169 }
167 }
170 return result
168 return result
171
169
172 def stop_all_clusters(self):
170 def stop_all_clusters(self):
173 for p in self.profiles.keys():
171 for p in self.profiles.keys():
174 self.stop_cluster(p)
172 self.stop_cluster(p)
@@ -1,547 +1,547 b''
1 # coding: utf-8
1 # coding: utf-8
2 """A tornado based IPython notebook server.
2 """A tornado based IPython notebook server.
3
3
4 Authors:
4 Authors:
5
5
6 * Brian Granger
6 * Brian Granger
7 """
7 """
8 #-----------------------------------------------------------------------------
8 #-----------------------------------------------------------------------------
9 # Copyright (C) 2008-2011 The IPython Development Team
9 # Copyright (C) 2008-2011 The IPython Development Team
10 #
10 #
11 # Distributed under the terms of the BSD License. The full license is in
11 # Distributed under the terms of the BSD License. The full license is in
12 # the file COPYING, distributed as part of this software.
12 # the file COPYING, distributed as part of this software.
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14
14
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16 # Imports
16 # Imports
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 # stdlib
19 # stdlib
20 import errno
20 import errno
21 import logging
21 import logging
22 import os
22 import os
23 import re
23 import re
24 import select
24 import select
25 import signal
25 import signal
26 import socket
26 import socket
27 import sys
27 import sys
28 import threading
28 import threading
29 import time
29 import time
30 import webbrowser
30 import webbrowser
31
31
32 # Third party
32 # Third party
33 import zmq
33 import zmq
34
34
35 # Install the pyzmq ioloop. This has to be done before anything else from
35 # Install the pyzmq ioloop. This has to be done before anything else from
36 # tornado is imported.
36 # tornado is imported.
37 from zmq.eventloop import ioloop
37 from zmq.eventloop import ioloop
38 ioloop.install()
38 ioloop.install()
39
39
40 from tornado import httpserver
40 from tornado import httpserver
41 from tornado import web
41 from tornado import web
42
42
43 # Our own libraries
43 # Our own libraries
44 from .kernelmanager import MappingKernelManager
44 from .kernelmanager import MappingKernelManager
45 from .handlers import (LoginHandler, LogoutHandler,
45 from .handlers import (LoginHandler, LogoutHandler,
46 ProjectDashboardHandler, NewHandler, NamedNotebookHandler,
46 ProjectDashboardHandler, NewHandler, NamedNotebookHandler,
47 MainKernelHandler, KernelHandler, KernelActionHandler, IOPubHandler,
47 MainKernelHandler, KernelHandler, KernelActionHandler, IOPubHandler,
48 ShellHandler, NotebookRootHandler, NotebookHandler, NotebookCopyHandler,
48 ShellHandler, NotebookRootHandler, NotebookHandler, NotebookCopyHandler,
49 RSTHandler, AuthenticatedFileHandler, PrintNotebookHandler,
49 RSTHandler, AuthenticatedFileHandler, PrintNotebookHandler,
50 MainClusterHandler, ClusterProfileHandler, ClusterActionHandler
50 MainClusterHandler, ClusterProfileHandler, ClusterActionHandler
51 )
51 )
52 from .notebookmanager import NotebookManager
52 from .notebookmanager import NotebookManager
53 from .clustermanager import ClusterManager
53 from .clustermanager import ClusterManager
54
54
55 from IPython.config.application import catch_config_error, boolean_flag
55 from IPython.config.application import catch_config_error, boolean_flag
56 from IPython.core.application import BaseIPythonApplication
56 from IPython.core.application import BaseIPythonApplication
57 from IPython.core.profiledir import ProfileDir
57 from IPython.core.profiledir import ProfileDir
58 from IPython.lib.kernel import swallow_argv
58 from IPython.lib.kernel import swallow_argv
59 from IPython.zmq.session import Session, default_secure
59 from IPython.zmq.session import Session, default_secure
60 from IPython.zmq.zmqshell import ZMQInteractiveShell
60 from IPython.zmq.zmqshell import ZMQInteractiveShell
61 from IPython.zmq.ipkernel import (
61 from IPython.zmq.ipkernel import (
62 flags as ipkernel_flags,
62 flags as ipkernel_flags,
63 aliases as ipkernel_aliases,
63 aliases as ipkernel_aliases,
64 IPKernelApp
64 IPKernelApp
65 )
65 )
66 from IPython.utils.traitlets import Dict, Unicode, Integer, List, Enum, Bool
66 from IPython.utils.traitlets import Dict, Unicode, Integer, List, Enum, Bool
67 from IPython.utils import py3compat
67 from IPython.utils import py3compat
68
68
69 #-----------------------------------------------------------------------------
69 #-----------------------------------------------------------------------------
70 # Module globals
70 # Module globals
71 #-----------------------------------------------------------------------------
71 #-----------------------------------------------------------------------------
72
72
73 _kernel_id_regex = r"(?P<kernel_id>\w+-\w+-\w+-\w+-\w+)"
73 _kernel_id_regex = r"(?P<kernel_id>\w+-\w+-\w+-\w+-\w+)"
74 _kernel_action_regex = r"(?P<action>restart|interrupt)"
74 _kernel_action_regex = r"(?P<action>restart|interrupt)"
75 _notebook_id_regex = r"(?P<notebook_id>\w+-\w+-\w+-\w+-\w+)"
75 _notebook_id_regex = r"(?P<notebook_id>\w+-\w+-\w+-\w+-\w+)"
76 _profile_regex = r"(?P<profile>[a-zA-Z0-9]+)"
76 _profile_regex = r"(?P<profile>[a-zA-Z0-9]+)"
77 _cluster_action_regex = r"(?P<action>start|stop)"
77 _cluster_action_regex = r"(?P<action>start|stop)"
78
78
79
79
80 LOCALHOST = '127.0.0.1'
80 LOCALHOST = '127.0.0.1'
81
81
82 _examples = """
82 _examples = """
83 ipython notebook # start the notebook
83 ipython notebook # start the notebook
84 ipython notebook --profile=sympy # use the sympy profile
84 ipython notebook --profile=sympy # use the sympy profile
85 ipython notebook --pylab=inline # pylab in inline plotting mode
85 ipython notebook --pylab=inline # pylab in inline plotting mode
86 ipython notebook --certfile=mycert.pem # use SSL/TLS certificate
86 ipython notebook --certfile=mycert.pem # use SSL/TLS certificate
87 ipython notebook --port=5555 --ip=* # Listen on port 5555, all interfaces
87 ipython notebook --port=5555 --ip=* # Listen on port 5555, all interfaces
88 """
88 """
89
89
90 #-----------------------------------------------------------------------------
90 #-----------------------------------------------------------------------------
91 # Helper functions
91 # Helper functions
92 #-----------------------------------------------------------------------------
92 #-----------------------------------------------------------------------------
93
93
94 def url_path_join(a,b):
94 def url_path_join(a,b):
95 if a.endswith('/') and b.startswith('/'):
95 if a.endswith('/') and b.startswith('/'):
96 return a[:-1]+b
96 return a[:-1]+b
97 else:
97 else:
98 return a+b
98 return a+b
99
99
100 #-----------------------------------------------------------------------------
100 #-----------------------------------------------------------------------------
101 # The Tornado web application
101 # The Tornado web application
102 #-----------------------------------------------------------------------------
102 #-----------------------------------------------------------------------------
103
103
104 class NotebookWebApplication(web.Application):
104 class NotebookWebApplication(web.Application):
105
105
106 def __init__(self, ipython_app, kernel_manager, notebook_manager,
106 def __init__(self, ipython_app, kernel_manager, notebook_manager,
107 cluster_manager, log,
107 cluster_manager, log,
108 base_project_url, settings_overrides):
108 base_project_url, settings_overrides):
109 handlers = [
109 handlers = [
110 (r"/", ProjectDashboardHandler),
110 (r"/", ProjectDashboardHandler),
111 (r"/login", LoginHandler),
111 (r"/login", LoginHandler),
112 (r"/logout", LogoutHandler),
112 (r"/logout", LogoutHandler),
113 (r"/new", NewHandler),
113 (r"/new", NewHandler),
114 (r"/%s" % _notebook_id_regex, NamedNotebookHandler),
114 (r"/%s" % _notebook_id_regex, NamedNotebookHandler),
115 (r"/%s/copy" % _notebook_id_regex, NotebookCopyHandler),
115 (r"/%s/copy" % _notebook_id_regex, NotebookCopyHandler),
116 (r"/%s/print" % _notebook_id_regex, PrintNotebookHandler),
116 (r"/%s/print" % _notebook_id_regex, PrintNotebookHandler),
117 (r"/kernels", MainKernelHandler),
117 (r"/kernels", MainKernelHandler),
118 (r"/kernels/%s" % _kernel_id_regex, KernelHandler),
118 (r"/kernels/%s" % _kernel_id_regex, KernelHandler),
119 (r"/kernels/%s/%s" % (_kernel_id_regex, _kernel_action_regex), KernelActionHandler),
119 (r"/kernels/%s/%s" % (_kernel_id_regex, _kernel_action_regex), KernelActionHandler),
120 (r"/kernels/%s/iopub" % _kernel_id_regex, IOPubHandler),
120 (r"/kernels/%s/iopub" % _kernel_id_regex, IOPubHandler),
121 (r"/kernels/%s/shell" % _kernel_id_regex, ShellHandler),
121 (r"/kernels/%s/shell" % _kernel_id_regex, ShellHandler),
122 (r"/notebooks", NotebookRootHandler),
122 (r"/notebooks", NotebookRootHandler),
123 (r"/notebooks/%s" % _notebook_id_regex, NotebookHandler),
123 (r"/notebooks/%s" % _notebook_id_regex, NotebookHandler),
124 (r"/rstservice/render", RSTHandler),
124 (r"/rstservice/render", RSTHandler),
125 (r"/files/(.*)", AuthenticatedFileHandler, {'path' : notebook_manager.notebook_dir}),
125 (r"/files/(.*)", AuthenticatedFileHandler, {'path' : notebook_manager.notebook_dir}),
126 (r"/clusters", MainClusterHandler),
126 (r"/clusters", MainClusterHandler),
127 (r"/clusters/%s/%s" % (_profile_regex, _cluster_action_regex), ClusterActionHandler),
127 (r"/clusters/%s/%s" % (_profile_regex, _cluster_action_regex), ClusterActionHandler),
128 (r"/clusters/%s" % _profile_regex, ClusterProfileHandler),
128 (r"/clusters/%s" % _profile_regex, ClusterProfileHandler),
129 ]
129 ]
130 settings = dict(
130 settings = dict(
131 template_path=os.path.join(os.path.dirname(__file__), "templates"),
131 template_path=os.path.join(os.path.dirname(__file__), "templates"),
132 static_path=os.path.join(os.path.dirname(__file__), "static"),
132 static_path=os.path.join(os.path.dirname(__file__), "static"),
133 cookie_secret=os.urandom(1024),
133 cookie_secret=os.urandom(1024),
134 login_url="/login",
134 login_url="/login",
135 )
135 )
136
136
137 # allow custom overrides for the tornado web app.
137 # allow custom overrides for the tornado web app.
138 settings.update(settings_overrides)
138 settings.update(settings_overrides)
139
139
140 # Python < 2.6.5 doesn't accept unicode keys in f(**kwargs), and
140 # Python < 2.6.5 doesn't accept unicode keys in f(**kwargs), and
141 # base_project_url will always be unicode, which will in turn
141 # base_project_url will always be unicode, which will in turn
142 # make the patterns unicode, and ultimately result in unicode
142 # make the patterns unicode, and ultimately result in unicode
143 # keys in kwargs to handler._execute(**kwargs) in tornado.
143 # keys in kwargs to handler._execute(**kwargs) in tornado.
144 # This enforces that base_project_url be ascii in that situation.
144 # This enforces that base_project_url be ascii in that situation.
145 #
145 #
146 # Note that the URLs these patterns check against are escaped,
146 # Note that the URLs these patterns check against are escaped,
147 # and thus guaranteed to be ASCII: 'hΓ©llo' is really 'h%C3%A9llo'.
147 # and thus guaranteed to be ASCII: 'hΓ©llo' is really 'h%C3%A9llo'.
148 base_project_url = py3compat.unicode_to_str(base_project_url, 'ascii')
148 base_project_url = py3compat.unicode_to_str(base_project_url, 'ascii')
149
149
150 # prepend base_project_url onto the patterns that we match
150 # prepend base_project_url onto the patterns that we match
151 new_handlers = []
151 new_handlers = []
152 for handler in handlers:
152 for handler in handlers:
153 pattern = url_path_join(base_project_url, handler[0])
153 pattern = url_path_join(base_project_url, handler[0])
154 new_handler = tuple([pattern]+list(handler[1:]))
154 new_handler = tuple([pattern]+list(handler[1:]))
155 new_handlers.append( new_handler )
155 new_handlers.append( new_handler )
156
156
157 super(NotebookWebApplication, self).__init__(new_handlers, **settings)
157 super(NotebookWebApplication, self).__init__(new_handlers, **settings)
158
158
159 self.kernel_manager = kernel_manager
159 self.kernel_manager = kernel_manager
160 self.notebook_manager = notebook_manager
160 self.notebook_manager = notebook_manager
161 self.cluster_manager = cluster_manager
161 self.cluster_manager = cluster_manager
162 self.ipython_app = ipython_app
162 self.ipython_app = ipython_app
163 self.read_only = self.ipython_app.read_only
163 self.read_only = self.ipython_app.read_only
164 self.log = log
164 self.log = log
165
165
166
166
167 #-----------------------------------------------------------------------------
167 #-----------------------------------------------------------------------------
168 # Aliases and Flags
168 # Aliases and Flags
169 #-----------------------------------------------------------------------------
169 #-----------------------------------------------------------------------------
170
170
171 flags = dict(ipkernel_flags)
171 flags = dict(ipkernel_flags)
172 flags['no-browser']=(
172 flags['no-browser']=(
173 {'NotebookApp' : {'open_browser' : False}},
173 {'NotebookApp' : {'open_browser' : False}},
174 "Don't open the notebook in a browser after startup."
174 "Don't open the notebook in a browser after startup."
175 )
175 )
176 flags['no-mathjax']=(
176 flags['no-mathjax']=(
177 {'NotebookApp' : {'enable_mathjax' : False}},
177 {'NotebookApp' : {'enable_mathjax' : False}},
178 """Disable MathJax
178 """Disable MathJax
179
179
180 MathJax is the javascript library IPython uses to render math/LaTeX. It is
180 MathJax is the javascript library IPython uses to render math/LaTeX. It is
181 very large, so you may want to disable it if you have a slow internet
181 very large, so you may want to disable it if you have a slow internet
182 connection, or for offline use of the notebook.
182 connection, or for offline use of the notebook.
183
183
184 When disabled, equations etc. will appear as their untransformed TeX source.
184 When disabled, equations etc. will appear as their untransformed TeX source.
185 """
185 """
186 )
186 )
187 flags['read-only'] = (
187 flags['read-only'] = (
188 {'NotebookApp' : {'read_only' : True}},
188 {'NotebookApp' : {'read_only' : True}},
189 """Allow read-only access to notebooks.
189 """Allow read-only access to notebooks.
190
190
191 When using a password to protect the notebook server, this flag
191 When using a password to protect the notebook server, this flag
192 allows unauthenticated clients to view the notebook list, and
192 allows unauthenticated clients to view the notebook list, and
193 individual notebooks, but not edit them, start kernels, or run
193 individual notebooks, but not edit them, start kernels, or run
194 code.
194 code.
195
195
196 If no password is set, the server will be entirely read-only.
196 If no password is set, the server will be entirely read-only.
197 """
197 """
198 )
198 )
199
199
200 # Add notebook manager flags
200 # Add notebook manager flags
201 flags.update(boolean_flag('script', 'NotebookManager.save_script',
201 flags.update(boolean_flag('script', 'NotebookManager.save_script',
202 'Auto-save a .py script everytime the .ipynb notebook is saved',
202 'Auto-save a .py script everytime the .ipynb notebook is saved',
203 'Do not auto-save .py scripts for every notebook'))
203 'Do not auto-save .py scripts for every notebook'))
204
204
205 # the flags that are specific to the frontend
205 # the flags that are specific to the frontend
206 # these must be scrubbed before being passed to the kernel,
206 # these must be scrubbed before being passed to the kernel,
207 # or it will raise an error on unrecognized flags
207 # or it will raise an error on unrecognized flags
208 notebook_flags = ['no-browser', 'no-mathjax', 'read-only', 'script', 'no-script']
208 notebook_flags = ['no-browser', 'no-mathjax', 'read-only', 'script', 'no-script']
209
209
210 aliases = dict(ipkernel_aliases)
210 aliases = dict(ipkernel_aliases)
211
211
212 aliases.update({
212 aliases.update({
213 'ip': 'NotebookApp.ip',
213 'ip': 'NotebookApp.ip',
214 'port': 'NotebookApp.port',
214 'port': 'NotebookApp.port',
215 'keyfile': 'NotebookApp.keyfile',
215 'keyfile': 'NotebookApp.keyfile',
216 'certfile': 'NotebookApp.certfile',
216 'certfile': 'NotebookApp.certfile',
217 'notebook-dir': 'NotebookManager.notebook_dir',
217 'notebook-dir': 'NotebookManager.notebook_dir',
218 'browser': 'NotebookApp.browser',
218 'browser': 'NotebookApp.browser',
219 })
219 })
220
220
221 # remove ipkernel flags that are singletons, and don't make sense in
221 # remove ipkernel flags that are singletons, and don't make sense in
222 # multi-kernel evironment:
222 # multi-kernel evironment:
223 aliases.pop('f', None)
223 aliases.pop('f', None)
224
224
225 notebook_aliases = [u'port', u'ip', u'keyfile', u'certfile',
225 notebook_aliases = [u'port', u'ip', u'keyfile', u'certfile',
226 u'notebook-dir']
226 u'notebook-dir']
227
227
228 #-----------------------------------------------------------------------------
228 #-----------------------------------------------------------------------------
229 # NotebookApp
229 # NotebookApp
230 #-----------------------------------------------------------------------------
230 #-----------------------------------------------------------------------------
231
231
232 class NotebookApp(BaseIPythonApplication):
232 class NotebookApp(BaseIPythonApplication):
233
233
234 name = 'ipython-notebook'
234 name = 'ipython-notebook'
235 default_config_file_name='ipython_notebook_config.py'
235 default_config_file_name='ipython_notebook_config.py'
236
236
237 description = """
237 description = """
238 The IPython HTML Notebook.
238 The IPython HTML Notebook.
239
239
240 This launches a Tornado based HTML Notebook Server that serves up an
240 This launches a Tornado based HTML Notebook Server that serves up an
241 HTML5/Javascript Notebook client.
241 HTML5/Javascript Notebook client.
242 """
242 """
243 examples = _examples
243 examples = _examples
244
244
245 classes = [IPKernelApp, ZMQInteractiveShell, ProfileDir, Session,
245 classes = [IPKernelApp, ZMQInteractiveShell, ProfileDir, Session,
246 MappingKernelManager, NotebookManager]
246 MappingKernelManager, NotebookManager]
247 flags = Dict(flags)
247 flags = Dict(flags)
248 aliases = Dict(aliases)
248 aliases = Dict(aliases)
249
249
250 kernel_argv = List(Unicode)
250 kernel_argv = List(Unicode)
251
251
252 log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
252 log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
253 default_value=logging.INFO,
253 default_value=logging.INFO,
254 config=True,
254 config=True,
255 help="Set the log level by value or name.")
255 help="Set the log level by value or name.")
256
256
257 # create requested profiles by default, if they don't exist:
257 # create requested profiles by default, if they don't exist:
258 auto_create = Bool(True)
258 auto_create = Bool(True)
259
259
260 # Network related information.
260 # Network related information.
261
261
262 ip = Unicode(LOCALHOST, config=True,
262 ip = Unicode(LOCALHOST, config=True,
263 help="The IP address the notebook server will listen on."
263 help="The IP address the notebook server will listen on."
264 )
264 )
265
265
266 def _ip_changed(self, name, old, new):
266 def _ip_changed(self, name, old, new):
267 if new == u'*': self.ip = u''
267 if new == u'*': self.ip = u''
268
268
269 port = Integer(8888, config=True,
269 port = Integer(8888, config=True,
270 help="The port the notebook server will listen on."
270 help="The port the notebook server will listen on."
271 )
271 )
272
272
273 certfile = Unicode(u'', config=True,
273 certfile = Unicode(u'', config=True,
274 help="""The full path to an SSL/TLS certificate file."""
274 help="""The full path to an SSL/TLS certificate file."""
275 )
275 )
276
276
277 keyfile = Unicode(u'', config=True,
277 keyfile = Unicode(u'', config=True,
278 help="""The full path to a private key file for usage with SSL/TLS."""
278 help="""The full path to a private key file for usage with SSL/TLS."""
279 )
279 )
280
280
281 password = Unicode(u'', config=True,
281 password = Unicode(u'', config=True,
282 help="""Hashed password to use for web authentication.
282 help="""Hashed password to use for web authentication.
283
283
284 To generate, type in a python/IPython shell:
284 To generate, type in a python/IPython shell:
285
285
286 from IPython.lib import passwd; passwd()
286 from IPython.lib import passwd; passwd()
287
287
288 The string should be of the form type:salt:hashed-password.
288 The string should be of the form type:salt:hashed-password.
289 """
289 """
290 )
290 )
291
291
292 open_browser = Bool(True, config=True,
292 open_browser = Bool(True, config=True,
293 help="""Whether to open in a browser after starting.
293 help="""Whether to open in a browser after starting.
294 The specific browser used is platform dependent and
294 The specific browser used is platform dependent and
295 determined by the python standard library `webbrowser`
295 determined by the python standard library `webbrowser`
296 module, unless it is overridden using the --browser
296 module, unless it is overridden using the --browser
297 (NotebookApp.browser) configuration option.
297 (NotebookApp.browser) configuration option.
298 """)
298 """)
299
299
300 browser = Unicode(u'', config=True,
300 browser = Unicode(u'', config=True,
301 help="""Specify what command to use to invoke a web
301 help="""Specify what command to use to invoke a web
302 browser when opening the notebook. If not specified, the
302 browser when opening the notebook. If not specified, the
303 default browser will be determined by the `webbrowser`
303 default browser will be determined by the `webbrowser`
304 standard library module, which allows setting of the
304 standard library module, which allows setting of the
305 BROWSER environment variable to override it.
305 BROWSER environment variable to override it.
306 """)
306 """)
307
307
308 read_only = Bool(False, config=True,
308 read_only = Bool(False, config=True,
309 help="Whether to prevent editing/execution of notebooks."
309 help="Whether to prevent editing/execution of notebooks."
310 )
310 )
311
311
312 webapp_settings = Dict(config=True,
312 webapp_settings = Dict(config=True,
313 help="Supply overrides for the tornado.web.Application that the "
313 help="Supply overrides for the tornado.web.Application that the "
314 "IPython notebook uses.")
314 "IPython notebook uses.")
315
315
316 enable_mathjax = Bool(True, config=True,
316 enable_mathjax = Bool(True, config=True,
317 help="""Whether to enable MathJax for typesetting math/TeX
317 help="""Whether to enable MathJax for typesetting math/TeX
318
318
319 MathJax is the javascript library IPython uses to render math/LaTeX. It is
319 MathJax is the javascript library IPython uses to render math/LaTeX. It is
320 very large, so you may want to disable it if you have a slow internet
320 very large, so you may want to disable it if you have a slow internet
321 connection, or for offline use of the notebook.
321 connection, or for offline use of the notebook.
322
322
323 When disabled, equations etc. will appear as their untransformed TeX source.
323 When disabled, equations etc. will appear as their untransformed TeX source.
324 """
324 """
325 )
325 )
326 def _enable_mathjax_changed(self, name, old, new):
326 def _enable_mathjax_changed(self, name, old, new):
327 """set mathjax url to empty if mathjax is disabled"""
327 """set mathjax url to empty if mathjax is disabled"""
328 if not new:
328 if not new:
329 self.mathjax_url = u''
329 self.mathjax_url = u''
330
330
331 base_project_url = Unicode('/', config=True,
331 base_project_url = Unicode('/', config=True,
332 help='''The base URL for the notebook server''')
332 help='''The base URL for the notebook server''')
333 base_kernel_url = Unicode('/', config=True,
333 base_kernel_url = Unicode('/', config=True,
334 help='''The base URL for the kernel server''')
334 help='''The base URL for the kernel server''')
335 websocket_host = Unicode("", config=True,
335 websocket_host = Unicode("", config=True,
336 help="""The hostname for the websocket server."""
336 help="""The hostname for the websocket server."""
337 )
337 )
338
338
339 mathjax_url = Unicode("", config=True,
339 mathjax_url = Unicode("", config=True,
340 help="""The url for MathJax.js."""
340 help="""The url for MathJax.js."""
341 )
341 )
342 def _mathjax_url_default(self):
342 def _mathjax_url_default(self):
343 if not self.enable_mathjax:
343 if not self.enable_mathjax:
344 return u''
344 return u''
345 static_path = self.webapp_settings.get("static_path", os.path.join(os.path.dirname(__file__), "static"))
345 static_path = self.webapp_settings.get("static_path", os.path.join(os.path.dirname(__file__), "static"))
346 static_url_prefix = self.webapp_settings.get("static_url_prefix",
346 static_url_prefix = self.webapp_settings.get("static_url_prefix",
347 "/static/")
347 "/static/")
348 if os.path.exists(os.path.join(static_path, 'mathjax', "MathJax.js")):
348 if os.path.exists(os.path.join(static_path, 'mathjax', "MathJax.js")):
349 self.log.info("Using local MathJax")
349 self.log.info("Using local MathJax")
350 return static_url_prefix+u"mathjax/MathJax.js"
350 return static_url_prefix+u"mathjax/MathJax.js"
351 else:
351 else:
352 if self.certfile:
352 if self.certfile:
353 # HTTPS: load from Rackspace CDN, because SSL certificate requires it
353 # HTTPS: load from Rackspace CDN, because SSL certificate requires it
354 base = u"https://c328740.ssl.cf1.rackcdn.com"
354 base = u"https://c328740.ssl.cf1.rackcdn.com"
355 else:
355 else:
356 base = u"http://cdn.mathjax.org"
356 base = u"http://cdn.mathjax.org"
357
357
358 url = base + u"/mathjax/latest/MathJax.js"
358 url = base + u"/mathjax/latest/MathJax.js"
359 self.log.info("Using MathJax from CDN: %s", url)
359 self.log.info("Using MathJax from CDN: %s", url)
360 return url
360 return url
361
361
362 def _mathjax_url_changed(self, name, old, new):
362 def _mathjax_url_changed(self, name, old, new):
363 if new and not self.enable_mathjax:
363 if new and not self.enable_mathjax:
364 # enable_mathjax=False overrides mathjax_url
364 # enable_mathjax=False overrides mathjax_url
365 self.mathjax_url = u''
365 self.mathjax_url = u''
366 else:
366 else:
367 self.log.info("Using MathJax: %s", new)
367 self.log.info("Using MathJax: %s", new)
368
368
369 def parse_command_line(self, argv=None):
369 def parse_command_line(self, argv=None):
370 super(NotebookApp, self).parse_command_line(argv)
370 super(NotebookApp, self).parse_command_line(argv)
371 if argv is None:
371 if argv is None:
372 argv = sys.argv[1:]
372 argv = sys.argv[1:]
373
373
374 # Scrub frontend-specific flags
374 # Scrub frontend-specific flags
375 self.kernel_argv = swallow_argv(argv, notebook_aliases, notebook_flags)
375 self.kernel_argv = swallow_argv(argv, notebook_aliases, notebook_flags)
376 # Kernel should inherit default config file from frontend
376 # Kernel should inherit default config file from frontend
377 self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name)
377 self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name)
378
378
379 def init_configurables(self):
379 def init_configurables(self):
380 # force Session default to be secure
380 # force Session default to be secure
381 default_secure(self.config)
381 default_secure(self.config)
382 # Create a KernelManager and start a kernel.
382 # Create a KernelManager and start a kernel.
383 self.kernel_manager = MappingKernelManager(
383 self.kernel_manager = MappingKernelManager(
384 config=self.config, log=self.log, kernel_argv=self.kernel_argv,
384 config=self.config, log=self.log, kernel_argv=self.kernel_argv,
385 connection_dir = self.profile_dir.security_dir,
385 connection_dir = self.profile_dir.security_dir,
386 )
386 )
387 self.notebook_manager = NotebookManager(config=self.config, log=self.log)
387 self.notebook_manager = NotebookManager(config=self.config, log=self.log)
388 self.notebook_manager.list_notebooks()
388 self.notebook_manager.list_notebooks()
389 self.cluster_manager = ClusterManager(config=self.config, log=self.log)
389 self.cluster_manager = ClusterManager(config=self.config, log=self.log)
390 self.cluster_manager.update_profiles()
390 self.cluster_manager.update_profiles()
391
391
392 def init_logging(self):
392 def init_logging(self):
393 super(NotebookApp, self).init_logging()
394 # This prevents double log messages because tornado use a root logger that
393 # This prevents double log messages because tornado use a root logger that
395 # self.log is a child of. The logging module dipatches log messages to a log
394 # self.log is a child of. The logging module dipatches log messages to a log
396 # and all of its ancenstors until propagate is set to False.
395 # and all of its ancenstors until propagate is set to False.
397 self.log.propagate = False
396 self.log.propagate = False
398
397
399 def init_webapp(self):
398 def init_webapp(self):
400 """initialize tornado webapp and httpserver"""
399 """initialize tornado webapp and httpserver"""
401 self.web_app = NotebookWebApplication(
400 self.web_app = NotebookWebApplication(
402 self, self.kernel_manager, self.notebook_manager,
401 self, self.kernel_manager, self.notebook_manager,
403 self.cluster_manager, self.log,
402 self.cluster_manager, self.log,
404 self.base_project_url, self.webapp_settings
403 self.base_project_url, self.webapp_settings
405 )
404 )
406 if self.certfile:
405 if self.certfile:
407 ssl_options = dict(certfile=self.certfile)
406 ssl_options = dict(certfile=self.certfile)
408 if self.keyfile:
407 if self.keyfile:
409 ssl_options['keyfile'] = self.keyfile
408 ssl_options['keyfile'] = self.keyfile
410 else:
409 else:
411 ssl_options = None
410 ssl_options = None
412 self.web_app.password = self.password
411 self.web_app.password = self.password
413 self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options)
412 self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options)
414 if ssl_options is None and not self.ip and not (self.read_only and not self.password):
413 if ssl_options is None and not self.ip and not (self.read_only and not self.password):
415 self.log.critical('WARNING: the notebook server is listening on all IP addresses '
414 self.log.critical('WARNING: the notebook server is listening on all IP addresses '
416 'but not using any encryption or authentication. This is highly '
415 'but not using any encryption or authentication. This is highly '
417 'insecure and not recommended.')
416 'insecure and not recommended.')
418
417
419 # Try random ports centered around the default.
418 # Try random ports centered around the default.
420 from random import randint
419 from random import randint
421 n = 50 # Max number of attempts, keep reasonably large.
420 n = 50 # Max number of attempts, keep reasonably large.
422 for port in range(self.port, self.port+5) + [self.port + randint(-2*n, 2*n) for i in range(n-5)]:
421 for port in range(self.port, self.port+5) + [self.port + randint(-2*n, 2*n) for i in range(n-5)]:
423 try:
422 try:
424 self.http_server.listen(port, self.ip)
423 self.http_server.listen(port, self.ip)
425 except socket.error, e:
424 except socket.error, e:
426 if e.errno != errno.EADDRINUSE:
425 if e.errno != errno.EADDRINUSE:
427 raise
426 raise
428 self.log.info('The port %i is already in use, trying another random port.' % port)
427 self.log.info('The port %i is already in use, trying another random port.' % port)
429 else:
428 else:
430 self.port = port
429 self.port = port
431 break
430 break
432
431
433 def init_signal(self):
432 def init_signal(self):
434 # FIXME: remove this check when pyzmq dependency is >= 2.1.11
433 # FIXME: remove this check when pyzmq dependency is >= 2.1.11
435 # safely extract zmq version info:
434 # safely extract zmq version info:
436 try:
435 try:
437 zmq_v = zmq.pyzmq_version_info()
436 zmq_v = zmq.pyzmq_version_info()
438 except AttributeError:
437 except AttributeError:
439 zmq_v = [ int(n) for n in re.findall(r'\d+', zmq.__version__) ]
438 zmq_v = [ int(n) for n in re.findall(r'\d+', zmq.__version__) ]
440 if 'dev' in zmq.__version__:
439 if 'dev' in zmq.__version__:
441 zmq_v.append(999)
440 zmq_v.append(999)
442 zmq_v = tuple(zmq_v)
441 zmq_v = tuple(zmq_v)
443 if zmq_v >= (2,1,9):
442 if zmq_v >= (2,1,9):
444 # This won't work with 2.1.7 and
443 # This won't work with 2.1.7 and
445 # 2.1.9-10 will log ugly 'Interrupted system call' messages,
444 # 2.1.9-10 will log ugly 'Interrupted system call' messages,
446 # but it will work
445 # but it will work
447 signal.signal(signal.SIGINT, self._handle_sigint)
446 signal.signal(signal.SIGINT, self._handle_sigint)
448 signal.signal(signal.SIGTERM, self._signal_stop)
447 signal.signal(signal.SIGTERM, self._signal_stop)
449
448
450 def _handle_sigint(self, sig, frame):
449 def _handle_sigint(self, sig, frame):
451 """SIGINT handler spawns confirmation dialog"""
450 """SIGINT handler spawns confirmation dialog"""
452 # register more forceful signal handler for ^C^C case
451 # register more forceful signal handler for ^C^C case
453 signal.signal(signal.SIGINT, self._signal_stop)
452 signal.signal(signal.SIGINT, self._signal_stop)
454 # request confirmation dialog in bg thread, to avoid
453 # request confirmation dialog in bg thread, to avoid
455 # blocking the App
454 # blocking the App
456 thread = threading.Thread(target=self._confirm_exit)
455 thread = threading.Thread(target=self._confirm_exit)
457 thread.daemon = True
456 thread.daemon = True
458 thread.start()
457 thread.start()
459
458
460 def _restore_sigint_handler(self):
459 def _restore_sigint_handler(self):
461 """callback for restoring original SIGINT handler"""
460 """callback for restoring original SIGINT handler"""
462 signal.signal(signal.SIGINT, self._handle_sigint)
461 signal.signal(signal.SIGINT, self._handle_sigint)
463
462
464 def _confirm_exit(self):
463 def _confirm_exit(self):
465 """confirm shutdown on ^C
464 """confirm shutdown on ^C
466
465
467 A second ^C, or answering 'y' within 5s will cause shutdown,
466 A second ^C, or answering 'y' within 5s will cause shutdown,
468 otherwise original SIGINT handler will be restored.
467 otherwise original SIGINT handler will be restored.
469 """
468 """
470 # FIXME: remove this delay when pyzmq dependency is >= 2.1.11
469 # FIXME: remove this delay when pyzmq dependency is >= 2.1.11
471 time.sleep(0.1)
470 time.sleep(0.1)
472 sys.stdout.write("Shutdown Notebook Server (y/[n])? ")
471 sys.stdout.write("Shutdown Notebook Server (y/[n])? ")
473 sys.stdout.flush()
472 sys.stdout.flush()
474 r,w,x = select.select([sys.stdin], [], [], 5)
473 r,w,x = select.select([sys.stdin], [], [], 5)
475 if r:
474 if r:
476 line = sys.stdin.readline()
475 line = sys.stdin.readline()
477 if line.lower().startswith('y'):
476 if line.lower().startswith('y'):
478 self.log.critical("Shutdown confirmed")
477 self.log.critical("Shutdown confirmed")
479 ioloop.IOLoop.instance().stop()
478 ioloop.IOLoop.instance().stop()
480 return
479 return
481 else:
480 else:
482 print "No answer for 5s:",
481 print "No answer for 5s:",
483 print "resuming operation..."
482 print "resuming operation..."
484 # no answer, or answer is no:
483 # no answer, or answer is no:
485 # set it back to original SIGINT handler
484 # set it back to original SIGINT handler
486 # use IOLoop.add_callback because signal.signal must be called
485 # use IOLoop.add_callback because signal.signal must be called
487 # from main thread
486 # from main thread
488 ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler)
487 ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler)
489
488
490 def _signal_stop(self, sig, frame):
489 def _signal_stop(self, sig, frame):
491 self.log.critical("received signal %s, stopping", sig)
490 self.log.critical("received signal %s, stopping", sig)
492 ioloop.IOLoop.instance().stop()
491 ioloop.IOLoop.instance().stop()
493
492
494 @catch_config_error
493 @catch_config_error
495 def initialize(self, argv=None):
494 def initialize(self, argv=None):
495 self.init_logging()
496 super(NotebookApp, self).initialize(argv)
496 super(NotebookApp, self).initialize(argv)
497 self.init_configurables()
497 self.init_configurables()
498 self.init_webapp()
498 self.init_webapp()
499 self.init_signal()
499 self.init_signal()
500
500
501 def cleanup_kernels(self):
501 def cleanup_kernels(self):
502 """shutdown all kernels
502 """shutdown all kernels
503
503
504 The kernels will shutdown themselves when this process no longer exists,
504 The kernels will shutdown themselves when this process no longer exists,
505 but explicit shutdown allows the KernelManagers to cleanup the connection files.
505 but explicit shutdown allows the KernelManagers to cleanup the connection files.
506 """
506 """
507 self.log.info('Shutting down kernels')
507 self.log.info('Shutting down kernels')
508 km = self.kernel_manager
508 km = self.kernel_manager
509 # copy list, since kill_kernel deletes keys
509 # copy list, since kill_kernel deletes keys
510 for kid in list(km.kernel_ids):
510 for kid in list(km.kernel_ids):
511 km.kill_kernel(kid)
511 km.kill_kernel(kid)
512
512
513 def start(self):
513 def start(self):
514 ip = self.ip if self.ip else '[all ip addresses on your system]'
514 ip = self.ip if self.ip else '[all ip addresses on your system]'
515 proto = 'https' if self.certfile else 'http'
515 proto = 'https' if self.certfile else 'http'
516 info = self.log.info
516 info = self.log.info
517 info("The IPython Notebook is running at: %s://%s:%i%s" %
517 info("The IPython Notebook is running at: %s://%s:%i%s" %
518 (proto, ip, self.port,self.base_project_url) )
518 (proto, ip, self.port,self.base_project_url) )
519 info("Use Control-C to stop this server and shut down all kernels.")
519 info("Use Control-C to stop this server and shut down all kernels.")
520
520
521 if self.open_browser:
521 if self.open_browser:
522 ip = self.ip or '127.0.0.1'
522 ip = self.ip or '127.0.0.1'
523 if self.browser:
523 if self.browser:
524 browser = webbrowser.get(self.browser)
524 browser = webbrowser.get(self.browser)
525 else:
525 else:
526 browser = webbrowser.get()
526 browser = webbrowser.get()
527 b = lambda : browser.open("%s://%s:%i%s" % (proto, ip, self.port,
527 b = lambda : browser.open("%s://%s:%i%s" % (proto, ip, self.port,
528 self.base_project_url),
528 self.base_project_url),
529 new=2)
529 new=2)
530 threading.Thread(target=b).start()
530 threading.Thread(target=b).start()
531 try:
531 try:
532 ioloop.IOLoop.instance().start()
532 ioloop.IOLoop.instance().start()
533 except KeyboardInterrupt:
533 except KeyboardInterrupt:
534 info("Interrupted...")
534 info("Interrupted...")
535 finally:
535 finally:
536 self.cleanup_kernels()
536 self.cleanup_kernels()
537
537
538
538
539 #-----------------------------------------------------------------------------
539 #-----------------------------------------------------------------------------
540 # Main entry point
540 # Main entry point
541 #-----------------------------------------------------------------------------
541 #-----------------------------------------------------------------------------
542
542
543 def launch_new_instance():
543 def launch_new_instance():
544 app = NotebookApp.instance()
544 app = NotebookApp.instance()
545 app.initialize()
545 app.initialize()
546 app.start()
546 app.start()
547
547
@@ -1,265 +1,268 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 The Base Application class for IPython.parallel apps
3 The Base Application class for IPython.parallel apps
4
4
5 Authors:
5 Authors:
6
6
7 * Brian Granger
7 * Brian Granger
8 * Min RK
8 * Min RK
9
9
10 """
10 """
11
11
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13 # Copyright (C) 2008-2011 The IPython Development Team
13 # Copyright (C) 2008-2011 The IPython Development Team
14 #
14 #
15 # Distributed under the terms of the BSD License. The full license is in
15 # Distributed under the terms of the BSD License. The full license is in
16 # the file COPYING, distributed as part of this software.
16 # the file COPYING, distributed as part of this software.
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20 # Imports
20 # Imports
21 #-----------------------------------------------------------------------------
21 #-----------------------------------------------------------------------------
22
22
23 from __future__ import with_statement
23 from __future__ import with_statement
24
24
25 import os
25 import os
26 import logging
26 import logging
27 import re
27 import re
28 import sys
28 import sys
29
29
30 from subprocess import Popen, PIPE
30 from subprocess import Popen, PIPE
31
31
32 from IPython.config.application import catch_config_error
32 from IPython.config.application import catch_config_error
33 from IPython.core import release
33 from IPython.core import release
34 from IPython.core.crashhandler import CrashHandler
34 from IPython.core.crashhandler import CrashHandler
35 from IPython.core.application import (
35 from IPython.core.application import (
36 BaseIPythonApplication,
36 BaseIPythonApplication,
37 base_aliases as base_ip_aliases,
37 base_aliases as base_ip_aliases,
38 base_flags as base_ip_flags
38 base_flags as base_ip_flags
39 )
39 )
40 from IPython.utils.path import expand_path
40 from IPython.utils.path import expand_path
41
41
42 from IPython.utils.traitlets import Unicode, Bool, Instance, Dict, List
42 from IPython.utils.traitlets import Unicode, Bool, Instance, Dict, List
43
43
44 #-----------------------------------------------------------------------------
44 #-----------------------------------------------------------------------------
45 # Module errors
45 # Module errors
46 #-----------------------------------------------------------------------------
46 #-----------------------------------------------------------------------------
47
47
48 class PIDFileError(Exception):
48 class PIDFileError(Exception):
49 pass
49 pass
50
50
51
51
52 #-----------------------------------------------------------------------------
52 #-----------------------------------------------------------------------------
53 # Crash handler for this application
53 # Crash handler for this application
54 #-----------------------------------------------------------------------------
54 #-----------------------------------------------------------------------------
55
55
56 class ParallelCrashHandler(CrashHandler):
56 class ParallelCrashHandler(CrashHandler):
57 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
57 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
58
58
59 def __init__(self, app):
59 def __init__(self, app):
60 contact_name = release.authors['Min'][0]
60 contact_name = release.authors['Min'][0]
61 contact_email = release.author_email
61 contact_email = release.author_email
62 bug_tracker = 'https://github.com/ipython/ipython/issues'
62 bug_tracker = 'https://github.com/ipython/ipython/issues'
63 super(ParallelCrashHandler,self).__init__(
63 super(ParallelCrashHandler,self).__init__(
64 app, contact_name, contact_email, bug_tracker
64 app, contact_name, contact_email, bug_tracker
65 )
65 )
66
66
67
67
68 #-----------------------------------------------------------------------------
68 #-----------------------------------------------------------------------------
69 # Main application
69 # Main application
70 #-----------------------------------------------------------------------------
70 #-----------------------------------------------------------------------------
71 base_aliases = {}
71 base_aliases = {}
72 base_aliases.update(base_ip_aliases)
72 base_aliases.update(base_ip_aliases)
73 base_aliases.update({
73 base_aliases.update({
74 'profile-dir' : 'ProfileDir.location',
74 'profile-dir' : 'ProfileDir.location',
75 'work-dir' : 'BaseParallelApplication.work_dir',
75 'work-dir' : 'BaseParallelApplication.work_dir',
76 'log-to-file' : 'BaseParallelApplication.log_to_file',
76 'log-to-file' : 'BaseParallelApplication.log_to_file',
77 'clean-logs' : 'BaseParallelApplication.clean_logs',
77 'clean-logs' : 'BaseParallelApplication.clean_logs',
78 'log-url' : 'BaseParallelApplication.log_url',
78 'log-url' : 'BaseParallelApplication.log_url',
79 'cluster-id' : 'BaseParallelApplication.cluster_id',
79 'cluster-id' : 'BaseParallelApplication.cluster_id',
80 })
80 })
81
81
82 base_flags = {
82 base_flags = {
83 'log-to-file' : (
83 'log-to-file' : (
84 {'BaseParallelApplication' : {'log_to_file' : True}},
84 {'BaseParallelApplication' : {'log_to_file' : True}},
85 "send log output to a file"
85 "send log output to a file"
86 )
86 )
87 }
87 }
88 base_flags.update(base_ip_flags)
88 base_flags.update(base_ip_flags)
89
89
90 class BaseParallelApplication(BaseIPythonApplication):
90 class BaseParallelApplication(BaseIPythonApplication):
91 """The base Application for IPython.parallel apps
91 """The base Application for IPython.parallel apps
92
92
93 Principle extensions to BaseIPyythonApplication:
93 Principle extensions to BaseIPyythonApplication:
94
94
95 * work_dir
95 * work_dir
96 * remote logging via pyzmq
96 * remote logging via pyzmq
97 * IOLoop instance
97 * IOLoop instance
98 """
98 """
99
99
100 crash_handler_class = ParallelCrashHandler
100 crash_handler_class = ParallelCrashHandler
101
101
102 def _log_level_default(self):
102 def _log_level_default(self):
103 # temporarily override default_log_level to INFO
103 # temporarily override default_log_level to INFO
104 return logging.INFO
104 return logging.INFO
105
105
106 work_dir = Unicode(os.getcwdu(), config=True,
106 work_dir = Unicode(os.getcwdu(), config=True,
107 help='Set the working dir for the process.'
107 help='Set the working dir for the process.'
108 )
108 )
109 def _work_dir_changed(self, name, old, new):
109 def _work_dir_changed(self, name, old, new):
110 self.work_dir = unicode(expand_path(new))
110 self.work_dir = unicode(expand_path(new))
111
111
112 log_to_file = Bool(config=True,
112 log_to_file = Bool(config=True,
113 help="whether to log to a file")
113 help="whether to log to a file")
114
114
115 clean_logs = Bool(False, config=True,
115 clean_logs = Bool(False, config=True,
116 help="whether to cleanup old logfiles before starting")
116 help="whether to cleanup old logfiles before starting")
117
117
118 log_url = Unicode('', config=True,
118 log_url = Unicode('', config=True,
119 help="The ZMQ URL of the iplogger to aggregate logging.")
119 help="The ZMQ URL of the iplogger to aggregate logging.")
120
120
121 cluster_id = Unicode('', config=True,
121 cluster_id = Unicode('', config=True,
122 help="""String id to add to runtime files, to prevent name collisions when
122 help="""String id to add to runtime files, to prevent name collisions when
123 using multiple clusters with a single profile simultaneously.
123 using multiple clusters with a single profile simultaneously.
124
124
125 When set, files will be named like: 'ipcontroller-<cluster_id>-engine.json'
125 When set, files will be named like: 'ipcontroller-<cluster_id>-engine.json'
126
126
127 Since this is text inserted into filenames, typical recommendations apply:
127 Since this is text inserted into filenames, typical recommendations apply:
128 Simple character strings are ideal, and spaces are not recommended (but should
128 Simple character strings are ideal, and spaces are not recommended (but should
129 generally work).
129 generally work).
130 """
130 """
131 )
131 )
132 def _cluster_id_changed(self, name, old, new):
132 def _cluster_id_changed(self, name, old, new):
133 self.name = self.__class__.name
133 self.name = self.__class__.name
134 if new:
134 if new:
135 self.name += '-%s'%new
135 self.name += '-%s'%new
136
136
137 def _config_files_default(self):
137 def _config_files_default(self):
138 return ['ipcontroller_config.py', 'ipengine_config.py', 'ipcluster_config.py']
138 return ['ipcontroller_config.py', 'ipengine_config.py', 'ipcluster_config.py']
139
139
140 loop = Instance('zmq.eventloop.ioloop.IOLoop')
140 loop = Instance('zmq.eventloop.ioloop.IOLoop')
141 def _loop_default(self):
141 def _loop_default(self):
142 from zmq.eventloop.ioloop import IOLoop
142 from zmq.eventloop.ioloop import IOLoop
143 return IOLoop.instance()
143 return IOLoop.instance()
144
144
145 aliases = Dict(base_aliases)
145 aliases = Dict(base_aliases)
146 flags = Dict(base_flags)
146 flags = Dict(base_flags)
147
147
148 @catch_config_error
148 @catch_config_error
149 def initialize(self, argv=None):
149 def initialize(self, argv=None):
150 """initialize the app"""
150 """initialize the app"""
151 super(BaseParallelApplication, self).initialize(argv)
151 super(BaseParallelApplication, self).initialize(argv)
152 self.to_work_dir()
152 self.to_work_dir()
153 self.reinit_logging()
153 self.reinit_logging()
154
154
155 def to_work_dir(self):
155 def to_work_dir(self):
156 wd = self.work_dir
156 wd = self.work_dir
157 if unicode(wd) != os.getcwdu():
157 if unicode(wd) != os.getcwdu():
158 os.chdir(wd)
158 os.chdir(wd)
159 self.log.info("Changing to working dir: %s" % wd)
159 self.log.info("Changing to working dir: %s" % wd)
160 # This is the working dir by now.
160 # This is the working dir by now.
161 sys.path.insert(0, '')
161 sys.path.insert(0, '')
162
162
163 def reinit_logging(self):
163 def reinit_logging(self):
164 # Remove old log files
164 # Remove old log files
165 log_dir = self.profile_dir.log_dir
165 log_dir = self.profile_dir.log_dir
166 if self.clean_logs:
166 if self.clean_logs:
167 for f in os.listdir(log_dir):
167 for f in os.listdir(log_dir):
168 if re.match(r'%s-\d+\.(log|err|out)'%self.name,f):
168 if re.match(r'%s-\d+\.(log|err|out)'%self.name,f):
169 os.remove(os.path.join(log_dir, f))
169 os.remove(os.path.join(log_dir, f))
170 if self.log_to_file:
170 if self.log_to_file:
171 # Start logging to the new log file
171 # Start logging to the new log file
172 log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
172 log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
173 logfile = os.path.join(log_dir, log_filename)
173 logfile = os.path.join(log_dir, log_filename)
174 open_log_file = open(logfile, 'w')
174 open_log_file = open(logfile, 'w')
175 else:
175 else:
176 open_log_file = None
176 open_log_file = None
177 if open_log_file is not None:
177 if open_log_file is not None:
178 self.log.removeHandler(self._log_handler)
178 while self.log.handlers:
179 self.log.removeHandler(self.log.handlers[0])
179 self._log_handler = logging.StreamHandler(open_log_file)
180 self._log_handler = logging.StreamHandler(open_log_file)
180 self.log.addHandler(self._log_handler)
181 self.log.addHandler(self._log_handler)
182 else:
183 self._log_handler = self.log.handlers[0]
181 # Add timestamps to log format:
184 # Add timestamps to log format:
182 self._log_formatter = logging.Formatter("%(asctime)s.%(msecs).03d [%(name)s] %(message)s",
185 self._log_formatter = logging.Formatter("%(asctime)s.%(msecs).03d [%(name)s] %(message)s",
183 datefmt="%Y-%m-%d %H:%M:%S")
186 datefmt="%Y-%m-%d %H:%M:%S")
184 self._log_handler.setFormatter(self._log_formatter)
187 self._log_handler.setFormatter(self._log_formatter)
185 # do not propagate log messages to root logger
188 # do not propagate log messages to root logger
186 # ipcluster app will sometimes print duplicate messages during shutdown
189 # ipcluster app will sometimes print duplicate messages during shutdown
187 # if this is 1 (default):
190 # if this is 1 (default):
188 self.log.propagate = False
191 self.log.propagate = False
189
192
190 def write_pid_file(self, overwrite=False):
193 def write_pid_file(self, overwrite=False):
191 """Create a .pid file in the pid_dir with my pid.
194 """Create a .pid file in the pid_dir with my pid.
192
195
193 This must be called after pre_construct, which sets `self.pid_dir`.
196 This must be called after pre_construct, which sets `self.pid_dir`.
194 This raises :exc:`PIDFileError` if the pid file exists already.
197 This raises :exc:`PIDFileError` if the pid file exists already.
195 """
198 """
196 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
199 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
197 if os.path.isfile(pid_file):
200 if os.path.isfile(pid_file):
198 pid = self.get_pid_from_file()
201 pid = self.get_pid_from_file()
199 if not overwrite:
202 if not overwrite:
200 raise PIDFileError(
203 raise PIDFileError(
201 'The pid file [%s] already exists. \nThis could mean that this '
204 'The pid file [%s] already exists. \nThis could mean that this '
202 'server is already running with [pid=%s].' % (pid_file, pid)
205 'server is already running with [pid=%s].' % (pid_file, pid)
203 )
206 )
204 with open(pid_file, 'w') as f:
207 with open(pid_file, 'w') as f:
205 self.log.info("Creating pid file: %s" % pid_file)
208 self.log.info("Creating pid file: %s" % pid_file)
206 f.write(repr(os.getpid())+'\n')
209 f.write(repr(os.getpid())+'\n')
207
210
208 def remove_pid_file(self):
211 def remove_pid_file(self):
209 """Remove the pid file.
212 """Remove the pid file.
210
213
211 This should be called at shutdown by registering a callback with
214 This should be called at shutdown by registering a callback with
212 :func:`reactor.addSystemEventTrigger`. This needs to return
215 :func:`reactor.addSystemEventTrigger`. This needs to return
213 ``None``.
216 ``None``.
214 """
217 """
215 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
218 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
216 if os.path.isfile(pid_file):
219 if os.path.isfile(pid_file):
217 try:
220 try:
218 self.log.info("Removing pid file: %s" % pid_file)
221 self.log.info("Removing pid file: %s" % pid_file)
219 os.remove(pid_file)
222 os.remove(pid_file)
220 except:
223 except:
221 self.log.warn("Error removing the pid file: %s" % pid_file)
224 self.log.warn("Error removing the pid file: %s" % pid_file)
222
225
223 def get_pid_from_file(self):
226 def get_pid_from_file(self):
224 """Get the pid from the pid file.
227 """Get the pid from the pid file.
225
228
226 If the pid file doesn't exist a :exc:`PIDFileError` is raised.
229 If the pid file doesn't exist a :exc:`PIDFileError` is raised.
227 """
230 """
228 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
231 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
229 if os.path.isfile(pid_file):
232 if os.path.isfile(pid_file):
230 with open(pid_file, 'r') as f:
233 with open(pid_file, 'r') as f:
231 s = f.read().strip()
234 s = f.read().strip()
232 try:
235 try:
233 pid = int(s)
236 pid = int(s)
234 except:
237 except:
235 raise PIDFileError("invalid pid file: %s (contents: %r)"%(pid_file, s))
238 raise PIDFileError("invalid pid file: %s (contents: %r)"%(pid_file, s))
236 return pid
239 return pid
237 else:
240 else:
238 raise PIDFileError('pid file not found: %s' % pid_file)
241 raise PIDFileError('pid file not found: %s' % pid_file)
239
242
240 def check_pid(self, pid):
243 def check_pid(self, pid):
241 if os.name == 'nt':
244 if os.name == 'nt':
242 try:
245 try:
243 import ctypes
246 import ctypes
244 # returns 0 if no such process (of ours) exists
247 # returns 0 if no such process (of ours) exists
245 # positive int otherwise
248 # positive int otherwise
246 p = ctypes.windll.kernel32.OpenProcess(1,0,pid)
249 p = ctypes.windll.kernel32.OpenProcess(1,0,pid)
247 except Exception:
250 except Exception:
248 self.log.warn(
251 self.log.warn(
249 "Could not determine whether pid %i is running via `OpenProcess`. "
252 "Could not determine whether pid %i is running via `OpenProcess`. "
250 " Making the likely assumption that it is."%pid
253 " Making the likely assumption that it is."%pid
251 )
254 )
252 return True
255 return True
253 return bool(p)
256 return bool(p)
254 else:
257 else:
255 try:
258 try:
256 p = Popen(['ps','x'], stdout=PIPE, stderr=PIPE)
259 p = Popen(['ps','x'], stdout=PIPE, stderr=PIPE)
257 output,_ = p.communicate()
260 output,_ = p.communicate()
258 except OSError:
261 except OSError:
259 self.log.warn(
262 self.log.warn(
260 "Could not determine whether pid %i is running via `ps x`. "
263 "Could not determine whether pid %i is running via `ps x`. "
261 " Making the likely assumption that it is."%pid
264 " Making the likely assumption that it is."%pid
262 )
265 )
263 return True
266 return True
264 pids = map(int, re.findall(r'^\W*\d+', output, re.MULTILINE))
267 pids = map(int, re.findall(r'^\W*\d+', output, re.MULTILINE))
265 return pid in pids
268 return pid in pids
@@ -1,493 +1,491 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The IPython controller application.
4 The IPython controller application.
5
5
6 Authors:
6 Authors:
7
7
8 * Brian Granger
8 * Brian Granger
9 * MinRK
9 * MinRK
10
10
11 """
11 """
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Copyright (C) 2008-2011 The IPython Development Team
14 # Copyright (C) 2008-2011 The IPython Development Team
15 #
15 #
16 # Distributed under the terms of the BSD License. The full license is in
16 # Distributed under the terms of the BSD License. The full license is in
17 # the file COPYING, distributed as part of this software.
17 # the file COPYING, distributed as part of this software.
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Imports
21 # Imports
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 from __future__ import with_statement
24 from __future__ import with_statement
25
25
26 import json
26 import json
27 import os
27 import os
28 import socket
28 import socket
29 import stat
29 import stat
30 import sys
30 import sys
31
31
32 from multiprocessing import Process
32 from multiprocessing import Process
33 from signal import signal, SIGINT, SIGABRT, SIGTERM
33 from signal import signal, SIGINT, SIGABRT, SIGTERM
34
34
35 import zmq
35 import zmq
36 from zmq.devices import ProcessMonitoredQueue
36 from zmq.devices import ProcessMonitoredQueue
37 from zmq.log.handlers import PUBHandler
37 from zmq.log.handlers import PUBHandler
38
38
39 from IPython.core.profiledir import ProfileDir
39 from IPython.core.profiledir import ProfileDir
40
40
41 from IPython.parallel.apps.baseapp import (
41 from IPython.parallel.apps.baseapp import (
42 BaseParallelApplication,
42 BaseParallelApplication,
43 base_aliases,
43 base_aliases,
44 base_flags,
44 base_flags,
45 catch_config_error,
45 catch_config_error,
46 )
46 )
47 from IPython.utils.importstring import import_item
47 from IPython.utils.importstring import import_item
48 from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict, TraitError
48 from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict, TraitError
49
49
50 from IPython.zmq.session import (
50 from IPython.zmq.session import (
51 Session, session_aliases, session_flags, default_secure
51 Session, session_aliases, session_flags, default_secure
52 )
52 )
53
53
54 from IPython.parallel.controller.heartmonitor import HeartMonitor
54 from IPython.parallel.controller.heartmonitor import HeartMonitor
55 from IPython.parallel.controller.hub import HubFactory
55 from IPython.parallel.controller.hub import HubFactory
56 from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
56 from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
57 from IPython.parallel.controller.sqlitedb import SQLiteDB
57 from IPython.parallel.controller.sqlitedb import SQLiteDB
58
58
59 from IPython.parallel.util import split_url, disambiguate_url
59 from IPython.parallel.util import split_url, disambiguate_url
60
60
61 # conditional import of MongoDB backend class
61 # conditional import of MongoDB backend class
62
62
63 try:
63 try:
64 from IPython.parallel.controller.mongodb import MongoDB
64 from IPython.parallel.controller.mongodb import MongoDB
65 except ImportError:
65 except ImportError:
66 maybe_mongo = []
66 maybe_mongo = []
67 else:
67 else:
68 maybe_mongo = [MongoDB]
68 maybe_mongo = [MongoDB]
69
69
70
70
71 #-----------------------------------------------------------------------------
71 #-----------------------------------------------------------------------------
72 # Module level variables
72 # Module level variables
73 #-----------------------------------------------------------------------------
73 #-----------------------------------------------------------------------------
74
74
75
75
76 #: The default config file name for this application
76 #: The default config file name for this application
77 default_config_file_name = u'ipcontroller_config.py'
77 default_config_file_name = u'ipcontroller_config.py'
78
78
79
79
80 _description = """Start the IPython controller for parallel computing.
80 _description = """Start the IPython controller for parallel computing.
81
81
82 The IPython controller provides a gateway between the IPython engines and
82 The IPython controller provides a gateway between the IPython engines and
83 clients. The controller needs to be started before the engines and can be
83 clients. The controller needs to be started before the engines and can be
84 configured using command line options or using a cluster directory. Cluster
84 configured using command line options or using a cluster directory. Cluster
85 directories contain config, log and security files and are usually located in
85 directories contain config, log and security files and are usually located in
86 your ipython directory and named as "profile_name". See the `profile`
86 your ipython directory and named as "profile_name". See the `profile`
87 and `profile-dir` options for details.
87 and `profile-dir` options for details.
88 """
88 """
89
89
90 _examples = """
90 _examples = """
91 ipcontroller --ip=192.168.0.1 --port=1000 # listen on ip, port for engines
91 ipcontroller --ip=192.168.0.1 --port=1000 # listen on ip, port for engines
92 ipcontroller --scheme=pure # use the pure zeromq scheduler
92 ipcontroller --scheme=pure # use the pure zeromq scheduler
93 """
93 """
94
94
95
95
96 #-----------------------------------------------------------------------------
96 #-----------------------------------------------------------------------------
97 # The main application
97 # The main application
98 #-----------------------------------------------------------------------------
98 #-----------------------------------------------------------------------------
99 flags = {}
99 flags = {}
100 flags.update(base_flags)
100 flags.update(base_flags)
101 flags.update({
101 flags.update({
102 'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
102 'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
103 'Use threads instead of processes for the schedulers'),
103 'Use threads instead of processes for the schedulers'),
104 'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
104 'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
105 'use the SQLiteDB backend'),
105 'use the SQLiteDB backend'),
106 'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
106 'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
107 'use the MongoDB backend'),
107 'use the MongoDB backend'),
108 'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
108 'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
109 'use the in-memory DictDB backend'),
109 'use the in-memory DictDB backend'),
110 'nodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.NoDB'}},
110 'nodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.NoDB'}},
111 """use dummy DB backend, which doesn't store any information.
111 """use dummy DB backend, which doesn't store any information.
112
112
113 This can be used to prevent growth of the memory footprint of the Hub
113 This can be used to prevent growth of the memory footprint of the Hub
114 in cases where its record-keeping is not required. Requesting results
114 in cases where its record-keeping is not required. Requesting results
115 of tasks submitted by other clients, db_queries, and task resubmission
115 of tasks submitted by other clients, db_queries, and task resubmission
116 will not be available."""),
116 will not be available."""),
117 'reuse' : ({'IPControllerApp' : {'reuse_files' : True}},
117 'reuse' : ({'IPControllerApp' : {'reuse_files' : True}},
118 'reuse existing json connection files')
118 'reuse existing json connection files')
119 })
119 })
120
120
121 flags.update(session_flags)
121 flags.update(session_flags)
122
122
123 aliases = dict(
123 aliases = dict(
124 ssh = 'IPControllerApp.ssh_server',
124 ssh = 'IPControllerApp.ssh_server',
125 enginessh = 'IPControllerApp.engine_ssh_server',
125 enginessh = 'IPControllerApp.engine_ssh_server',
126 location = 'IPControllerApp.location',
126 location = 'IPControllerApp.location',
127
127
128 url = 'HubFactory.url',
128 url = 'HubFactory.url',
129 ip = 'HubFactory.ip',
129 ip = 'HubFactory.ip',
130 transport = 'HubFactory.transport',
130 transport = 'HubFactory.transport',
131 port = 'HubFactory.regport',
131 port = 'HubFactory.regport',
132
132
133 ping = 'HeartMonitor.period',
133 ping = 'HeartMonitor.period',
134
134
135 scheme = 'TaskScheduler.scheme_name',
135 scheme = 'TaskScheduler.scheme_name',
136 hwm = 'TaskScheduler.hwm',
136 hwm = 'TaskScheduler.hwm',
137 )
137 )
138 aliases.update(base_aliases)
138 aliases.update(base_aliases)
139 aliases.update(session_aliases)
139 aliases.update(session_aliases)
140
140
141
141
142 class IPControllerApp(BaseParallelApplication):
142 class IPControllerApp(BaseParallelApplication):
143
143
144 name = u'ipcontroller'
144 name = u'ipcontroller'
145 description = _description
145 description = _description
146 examples = _examples
146 examples = _examples
147 config_file_name = Unicode(default_config_file_name)
147 config_file_name = Unicode(default_config_file_name)
148 classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo
148 classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo
149
149
150 # change default to True
150 # change default to True
151 auto_create = Bool(True, config=True,
151 auto_create = Bool(True, config=True,
152 help="""Whether to create profile dir if it doesn't exist.""")
152 help="""Whether to create profile dir if it doesn't exist.""")
153
153
154 reuse_files = Bool(False, config=True,
154 reuse_files = Bool(False, config=True,
155 help="""Whether to reuse existing json connection files.
155 help="""Whether to reuse existing json connection files.
156 If False, connection files will be removed on a clean exit.
156 If False, connection files will be removed on a clean exit.
157 """
157 """
158 )
158 )
159 ssh_server = Unicode(u'', config=True,
159 ssh_server = Unicode(u'', config=True,
160 help="""ssh url for clients to use when connecting to the Controller
160 help="""ssh url for clients to use when connecting to the Controller
161 processes. It should be of the form: [user@]server[:port]. The
161 processes. It should be of the form: [user@]server[:port]. The
162 Controller's listening addresses must be accessible from the ssh server""",
162 Controller's listening addresses must be accessible from the ssh server""",
163 )
163 )
164 engine_ssh_server = Unicode(u'', config=True,
164 engine_ssh_server = Unicode(u'', config=True,
165 help="""ssh url for engines to use when connecting to the Controller
165 help="""ssh url for engines to use when connecting to the Controller
166 processes. It should be of the form: [user@]server[:port]. The
166 processes. It should be of the form: [user@]server[:port]. The
167 Controller's listening addresses must be accessible from the ssh server""",
167 Controller's listening addresses must be accessible from the ssh server""",
168 )
168 )
169 location = Unicode(u'', config=True,
169 location = Unicode(u'', config=True,
170 help="""The external IP or domain name of the Controller, used for disambiguating
170 help="""The external IP or domain name of the Controller, used for disambiguating
171 engine and client connections.""",
171 engine and client connections.""",
172 )
172 )
173 import_statements = List([], config=True,
173 import_statements = List([], config=True,
174 help="import statements to be run at startup. Necessary in some environments"
174 help="import statements to be run at startup. Necessary in some environments"
175 )
175 )
176
176
177 use_threads = Bool(False, config=True,
177 use_threads = Bool(False, config=True,
178 help='Use threads instead of processes for the schedulers',
178 help='Use threads instead of processes for the schedulers',
179 )
179 )
180
180
181 engine_json_file = Unicode('ipcontroller-engine.json', config=True,
181 engine_json_file = Unicode('ipcontroller-engine.json', config=True,
182 help="JSON filename where engine connection info will be stored.")
182 help="JSON filename where engine connection info will be stored.")
183 client_json_file = Unicode('ipcontroller-client.json', config=True,
183 client_json_file = Unicode('ipcontroller-client.json', config=True,
184 help="JSON filename where client connection info will be stored.")
184 help="JSON filename where client connection info will be stored.")
185
185
186 def _cluster_id_changed(self, name, old, new):
186 def _cluster_id_changed(self, name, old, new):
187 super(IPControllerApp, self)._cluster_id_changed(name, old, new)
187 super(IPControllerApp, self)._cluster_id_changed(name, old, new)
188 self.engine_json_file = "%s-engine.json" % self.name
188 self.engine_json_file = "%s-engine.json" % self.name
189 self.client_json_file = "%s-client.json" % self.name
189 self.client_json_file = "%s-client.json" % self.name
190
190
191
191
192 # internal
192 # internal
193 children = List()
193 children = List()
194 mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')
194 mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')
195
195
196 def _use_threads_changed(self, name, old, new):
196 def _use_threads_changed(self, name, old, new):
197 self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
197 self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
198
198
199 write_connection_files = Bool(True,
199 write_connection_files = Bool(True,
200 help="""Whether to write connection files to disk.
200 help="""Whether to write connection files to disk.
201 True in all cases other than runs with `reuse_files=True` *after the first*
201 True in all cases other than runs with `reuse_files=True` *after the first*
202 """
202 """
203 )
203 )
204
204
205 aliases = Dict(aliases)
205 aliases = Dict(aliases)
206 flags = Dict(flags)
206 flags = Dict(flags)
207
207
208
208
209 def save_connection_dict(self, fname, cdict):
209 def save_connection_dict(self, fname, cdict):
210 """save a connection dict to json file."""
210 """save a connection dict to json file."""
211 c = self.config
211 c = self.config
212 url = cdict['url']
212 url = cdict['url']
213 location = cdict['location']
213 location = cdict['location']
214 if not location:
214 if not location:
215 try:
215 try:
216 proto,ip,port = split_url(url)
216 proto,ip,port = split_url(url)
217 except AssertionError:
217 except AssertionError:
218 pass
218 pass
219 else:
219 else:
220 try:
220 try:
221 location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
221 location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
222 except (socket.gaierror, IndexError):
222 except (socket.gaierror, IndexError):
223 self.log.warn("Could not identify this machine's IP, assuming 127.0.0.1."
223 self.log.warn("Could not identify this machine's IP, assuming 127.0.0.1."
224 " You may need to specify '--location=<external_ip_address>' to help"
224 " You may need to specify '--location=<external_ip_address>' to help"
225 " IPython decide when to connect via loopback.")
225 " IPython decide when to connect via loopback.")
226 location = '127.0.0.1'
226 location = '127.0.0.1'
227 cdict['location'] = location
227 cdict['location'] = location
228 fname = os.path.join(self.profile_dir.security_dir, fname)
228 fname = os.path.join(self.profile_dir.security_dir, fname)
229 self.log.info("writing connection info to %s", fname)
229 self.log.info("writing connection info to %s", fname)
230 with open(fname, 'w') as f:
230 with open(fname, 'w') as f:
231 f.write(json.dumps(cdict, indent=2))
231 f.write(json.dumps(cdict, indent=2))
232 os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
232 os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
233
233
234 def load_config_from_json(self):
234 def load_config_from_json(self):
235 """load config from existing json connector files."""
235 """load config from existing json connector files."""
236 c = self.config
236 c = self.config
237 self.log.debug("loading config from JSON")
237 self.log.debug("loading config from JSON")
238 # load from engine config
238 # load from engine config
239 fname = os.path.join(self.profile_dir.security_dir, self.engine_json_file)
239 fname = os.path.join(self.profile_dir.security_dir, self.engine_json_file)
240 self.log.info("loading connection info from %s", fname)
240 self.log.info("loading connection info from %s", fname)
241 with open(fname) as f:
241 with open(fname) as f:
242 cfg = json.loads(f.read())
242 cfg = json.loads(f.read())
243 key = cfg['exec_key']
243 key = cfg['exec_key']
244 # json gives unicode, Session.key wants bytes
244 # json gives unicode, Session.key wants bytes
245 c.Session.key = key.encode('ascii')
245 c.Session.key = key.encode('ascii')
246 xport,addr = cfg['url'].split('://')
246 xport,addr = cfg['url'].split('://')
247 c.HubFactory.engine_transport = xport
247 c.HubFactory.engine_transport = xport
248 ip,ports = addr.split(':')
248 ip,ports = addr.split(':')
249 c.HubFactory.engine_ip = ip
249 c.HubFactory.engine_ip = ip
250 c.HubFactory.regport = int(ports)
250 c.HubFactory.regport = int(ports)
251 self.location = cfg['location']
251 self.location = cfg['location']
252 if not self.engine_ssh_server:
252 if not self.engine_ssh_server:
253 self.engine_ssh_server = cfg['ssh']
253 self.engine_ssh_server = cfg['ssh']
254 # load client config
254 # load client config
255 fname = os.path.join(self.profile_dir.security_dir, self.client_json_file)
255 fname = os.path.join(self.profile_dir.security_dir, self.client_json_file)
256 self.log.info("loading connection info from %s", fname)
256 self.log.info("loading connection info from %s", fname)
257 with open(fname) as f:
257 with open(fname) as f:
258 cfg = json.loads(f.read())
258 cfg = json.loads(f.read())
259 assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
259 assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
260 xport,addr = cfg['url'].split('://')
260 xport,addr = cfg['url'].split('://')
261 c.HubFactory.client_transport = xport
261 c.HubFactory.client_transport = xport
262 ip,ports = addr.split(':')
262 ip,ports = addr.split(':')
263 c.HubFactory.client_ip = ip
263 c.HubFactory.client_ip = ip
264 if not self.ssh_server:
264 if not self.ssh_server:
265 self.ssh_server = cfg['ssh']
265 self.ssh_server = cfg['ssh']
266 assert int(ports) == c.HubFactory.regport, "regport mismatch"
266 assert int(ports) == c.HubFactory.regport, "regport mismatch"
267
267
268 def cleanup_connection_files(self):
268 def cleanup_connection_files(self):
269 if self.reuse_files:
269 if self.reuse_files:
270 self.log.debug("leaving JSON connection files for reuse")
270 self.log.debug("leaving JSON connection files for reuse")
271 return
271 return
272 self.log.debug("cleaning up JSON connection files")
272 self.log.debug("cleaning up JSON connection files")
273 for f in (self.client_json_file, self.engine_json_file):
273 for f in (self.client_json_file, self.engine_json_file):
274 f = os.path.join(self.profile_dir.security_dir, f)
274 f = os.path.join(self.profile_dir.security_dir, f)
275 try:
275 try:
276 os.remove(f)
276 os.remove(f)
277 except Exception as e:
277 except Exception as e:
278 self.log.error("Failed to cleanup connection file: %s", e)
278 self.log.error("Failed to cleanup connection file: %s", e)
279 else:
279 else:
280 self.log.debug(u"removed %s", f)
280 self.log.debug(u"removed %s", f)
281
281
282 def load_secondary_config(self):
282 def load_secondary_config(self):
283 """secondary config, loading from JSON and setting defaults"""
283 """secondary config, loading from JSON and setting defaults"""
284 if self.reuse_files:
284 if self.reuse_files:
285 try:
285 try:
286 self.load_config_from_json()
286 self.load_config_from_json()
287 except (AssertionError,IOError) as e:
287 except (AssertionError,IOError) as e:
288 self.log.error("Could not load config from JSON: %s" % e)
288 self.log.error("Could not load config from JSON: %s" % e)
289 else:
289 else:
290 # successfully loaded config from JSON, and reuse=True
290 # successfully loaded config from JSON, and reuse=True
291 # no need to wite back the same file
291 # no need to wite back the same file
292 self.write_connection_files = False
292 self.write_connection_files = False
293
293
294 # switch Session.key default to secure
294 # switch Session.key default to secure
295 default_secure(self.config)
295 default_secure(self.config)
296 self.log.debug("Config changed")
296 self.log.debug("Config changed")
297 self.log.debug(repr(self.config))
297 self.log.debug(repr(self.config))
298
298
299 def init_hub(self):
299 def init_hub(self):
300 c = self.config
300 c = self.config
301
301
302 self.do_import_statements()
302 self.do_import_statements()
303
303
304 try:
304 try:
305 self.factory = HubFactory(config=c, log=self.log)
305 self.factory = HubFactory(config=c, log=self.log)
306 # self.start_logging()
306 # self.start_logging()
307 self.factory.init_hub()
307 self.factory.init_hub()
308 except TraitError:
308 except TraitError:
309 raise
309 raise
310 except Exception:
310 except Exception:
311 self.log.error("Couldn't construct the Controller", exc_info=True)
311 self.log.error("Couldn't construct the Controller", exc_info=True)
312 self.exit(1)
312 self.exit(1)
313
313
314 if self.write_connection_files:
314 if self.write_connection_files:
315 # save to new json config files
315 # save to new json config files
316 f = self.factory
316 f = self.factory
317 cdict = {'exec_key' : f.session.key.decode('ascii'),
317 cdict = {'exec_key' : f.session.key.decode('ascii'),
318 'ssh' : self.ssh_server,
318 'ssh' : self.ssh_server,
319 'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
319 'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
320 'location' : self.location
320 'location' : self.location
321 }
321 }
322 self.save_connection_dict(self.client_json_file, cdict)
322 self.save_connection_dict(self.client_json_file, cdict)
323 edict = cdict
323 edict = cdict
324 edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
324 edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
325 edict['ssh'] = self.engine_ssh_server
325 edict['ssh'] = self.engine_ssh_server
326 self.save_connection_dict(self.engine_json_file, edict)
326 self.save_connection_dict(self.engine_json_file, edict)
327
327
328 def init_schedulers(self):
328 def init_schedulers(self):
329 children = self.children
329 children = self.children
330 mq = import_item(str(self.mq_class))
330 mq = import_item(str(self.mq_class))
331
331
332 hub = self.factory
332 hub = self.factory
333 # disambiguate url, in case of *
333 # disambiguate url, in case of *
334 monitor_url = disambiguate_url(hub.monitor_url)
334 monitor_url = disambiguate_url(hub.monitor_url)
335 # maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url
335 # maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url
336 # IOPub relay (in a Process)
336 # IOPub relay (in a Process)
337 q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub')
337 q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub')
338 q.bind_in(hub.client_info['iopub'])
338 q.bind_in(hub.client_info['iopub'])
339 q.bind_out(hub.engine_info['iopub'])
339 q.bind_out(hub.engine_info['iopub'])
340 q.setsockopt_out(zmq.SUBSCRIBE, b'')
340 q.setsockopt_out(zmq.SUBSCRIBE, b'')
341 q.connect_mon(monitor_url)
341 q.connect_mon(monitor_url)
342 q.daemon=True
342 q.daemon=True
343 children.append(q)
343 children.append(q)
344
344
345 # Multiplexer Queue (in a Process)
345 # Multiplexer Queue (in a Process)
346 q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out')
346 q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out')
347 q.bind_in(hub.client_info['mux'])
347 q.bind_in(hub.client_info['mux'])
348 q.setsockopt_in(zmq.IDENTITY, b'mux')
348 q.setsockopt_in(zmq.IDENTITY, b'mux')
349 q.bind_out(hub.engine_info['mux'])
349 q.bind_out(hub.engine_info['mux'])
350 q.connect_mon(monitor_url)
350 q.connect_mon(monitor_url)
351 q.daemon=True
351 q.daemon=True
352 children.append(q)
352 children.append(q)
353
353
354 # Control Queue (in a Process)
354 # Control Queue (in a Process)
355 q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol')
355 q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol')
356 q.bind_in(hub.client_info['control'])
356 q.bind_in(hub.client_info['control'])
357 q.setsockopt_in(zmq.IDENTITY, b'control')
357 q.setsockopt_in(zmq.IDENTITY, b'control')
358 q.bind_out(hub.engine_info['control'])
358 q.bind_out(hub.engine_info['control'])
359 q.connect_mon(monitor_url)
359 q.connect_mon(monitor_url)
360 q.daemon=True
360 q.daemon=True
361 children.append(q)
361 children.append(q)
362 try:
362 try:
363 scheme = self.config.TaskScheduler.scheme_name
363 scheme = self.config.TaskScheduler.scheme_name
364 except AttributeError:
364 except AttributeError:
365 scheme = TaskScheduler.scheme_name.get_default_value()
365 scheme = TaskScheduler.scheme_name.get_default_value()
366 # Task Queue (in a Process)
366 # Task Queue (in a Process)
367 if scheme == 'pure':
367 if scheme == 'pure':
368 self.log.warn("task::using pure XREQ Task scheduler")
368 self.log.warn("task::using pure XREQ Task scheduler")
369 q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask')
369 q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask')
370 # q.setsockopt_out(zmq.HWM, hub.hwm)
370 # q.setsockopt_out(zmq.HWM, hub.hwm)
371 q.bind_in(hub.client_info['task'][1])
371 q.bind_in(hub.client_info['task'][1])
372 q.setsockopt_in(zmq.IDENTITY, b'task')
372 q.setsockopt_in(zmq.IDENTITY, b'task')
373 q.bind_out(hub.engine_info['task'])
373 q.bind_out(hub.engine_info['task'])
374 q.connect_mon(monitor_url)
374 q.connect_mon(monitor_url)
375 q.daemon=True
375 q.daemon=True
376 children.append(q)
376 children.append(q)
377 elif scheme == 'none':
377 elif scheme == 'none':
378 self.log.warn("task::using no Task scheduler")
378 self.log.warn("task::using no Task scheduler")
379
379
380 else:
380 else:
381 self.log.info("task::using Python %s Task scheduler"%scheme)
381 self.log.info("task::using Python %s Task scheduler"%scheme)
382 sargs = (hub.client_info['task'][1], hub.engine_info['task'],
382 sargs = (hub.client_info['task'][1], hub.engine_info['task'],
383 monitor_url, disambiguate_url(hub.client_info['notification']))
383 monitor_url, disambiguate_url(hub.client_info['notification']))
384 kwargs = dict(logname='scheduler', loglevel=self.log_level,
384 kwargs = dict(logname='scheduler', loglevel=self.log_level,
385 log_url = self.log_url, config=dict(self.config))
385 log_url = self.log_url, config=dict(self.config))
386 if 'Process' in self.mq_class:
386 if 'Process' in self.mq_class:
387 # run the Python scheduler in a Process
387 # run the Python scheduler in a Process
388 q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
388 q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
389 q.daemon=True
389 q.daemon=True
390 children.append(q)
390 children.append(q)
391 else:
391 else:
392 # single-threaded Controller
392 # single-threaded Controller
393 kwargs['in_thread'] = True
393 kwargs['in_thread'] = True
394 launch_scheduler(*sargs, **kwargs)
394 launch_scheduler(*sargs, **kwargs)
395
395
396 def terminate_children(self):
396 def terminate_children(self):
397 child_procs = []
397 child_procs = []
398 for child in self.children:
398 for child in self.children:
399 if isinstance(child, ProcessMonitoredQueue):
399 if isinstance(child, ProcessMonitoredQueue):
400 child_procs.append(child.launcher)
400 child_procs.append(child.launcher)
401 elif isinstance(child, Process):
401 elif isinstance(child, Process):
402 child_procs.append(child)
402 child_procs.append(child)
403 if child_procs:
403 if child_procs:
404 self.log.critical("terminating children...")
404 self.log.critical("terminating children...")
405 for child in child_procs:
405 for child in child_procs:
406 try:
406 try:
407 child.terminate()
407 child.terminate()
408 except OSError:
408 except OSError:
409 # already dead
409 # already dead
410 pass
410 pass
411
411
412 def handle_signal(self, sig, frame):
412 def handle_signal(self, sig, frame):
413 self.log.critical("Received signal %i, shutting down", sig)
413 self.log.critical("Received signal %i, shutting down", sig)
414 self.terminate_children()
414 self.terminate_children()
415 self.loop.stop()
415 self.loop.stop()
416
416
417 def init_signal(self):
417 def init_signal(self):
418 for sig in (SIGINT, SIGABRT, SIGTERM):
418 for sig in (SIGINT, SIGABRT, SIGTERM):
419 signal(sig, self.handle_signal)
419 signal(sig, self.handle_signal)
420
420
421 def do_import_statements(self):
421 def do_import_statements(self):
422 statements = self.import_statements
422 statements = self.import_statements
423 for s in statements:
423 for s in statements:
424 try:
424 try:
425 self.log.msg("Executing statement: '%s'" % s)
425 self.log.msg("Executing statement: '%s'" % s)
426 exec s in globals(), locals()
426 exec s in globals(), locals()
427 except:
427 except:
428 self.log.msg("Error running statement: %s" % s)
428 self.log.msg("Error running statement: %s" % s)
429
429
430 def forward_logging(self):
430 def forward_logging(self):
431 if self.log_url:
431 if self.log_url:
432 self.log.info("Forwarding logging to %s"%self.log_url)
432 self.log.info("Forwarding logging to %s"%self.log_url)
433 context = zmq.Context.instance()
433 context = zmq.Context.instance()
434 lsock = context.socket(zmq.PUB)
434 lsock = context.socket(zmq.PUB)
435 lsock.connect(self.log_url)
435 lsock.connect(self.log_url)
436 handler = PUBHandler(lsock)
436 handler = PUBHandler(lsock)
437 self.log.removeHandler(self._log_handler)
438 handler.root_topic = 'controller'
437 handler.root_topic = 'controller'
439 handler.setLevel(self.log_level)
438 handler.setLevel(self.log_level)
440 self.log.addHandler(handler)
439 self.log.addHandler(handler)
441 self._log_handler = handler
442
440
443 @catch_config_error
441 @catch_config_error
444 def initialize(self, argv=None):
442 def initialize(self, argv=None):
445 super(IPControllerApp, self).initialize(argv)
443 super(IPControllerApp, self).initialize(argv)
446 self.forward_logging()
444 self.forward_logging()
447 self.load_secondary_config()
445 self.load_secondary_config()
448 self.init_hub()
446 self.init_hub()
449 self.init_schedulers()
447 self.init_schedulers()
450
448
451 def start(self):
449 def start(self):
452 # Start the subprocesses:
450 # Start the subprocesses:
453 self.factory.start()
451 self.factory.start()
454 # children must be started before signals are setup,
452 # children must be started before signals are setup,
455 # otherwise signal-handling will fire multiple times
453 # otherwise signal-handling will fire multiple times
456 for child in self.children:
454 for child in self.children:
457 child.start()
455 child.start()
458 self.init_signal()
456 self.init_signal()
459
457
460 self.write_pid_file(overwrite=True)
458 self.write_pid_file(overwrite=True)
461
459
462 try:
460 try:
463 self.factory.loop.start()
461 self.factory.loop.start()
464 except KeyboardInterrupt:
462 except KeyboardInterrupt:
465 self.log.critical("Interrupted, Exiting...\n")
463 self.log.critical("Interrupted, Exiting...\n")
466 finally:
464 finally:
467 self.cleanup_connection_files()
465 self.cleanup_connection_files()
468
466
469
467
470
468
471 def launch_new_instance():
469 def launch_new_instance():
472 """Create and run the IPython controller"""
470 """Create and run the IPython controller"""
473 if sys.platform == 'win32':
471 if sys.platform == 'win32':
474 # make sure we don't get called from a multiprocessing subprocess
472 # make sure we don't get called from a multiprocessing subprocess
475 # this can result in infinite Controllers being started on Windows
473 # this can result in infinite Controllers being started on Windows
476 # which doesn't have a proper fork, so multiprocessing is wonky
474 # which doesn't have a proper fork, so multiprocessing is wonky
477
475
478 # this only comes up when IPython has been installed using vanilla
476 # this only comes up when IPython has been installed using vanilla
479 # setuptools, and *not* distribute.
477 # setuptools, and *not* distribute.
480 import multiprocessing
478 import multiprocessing
481 p = multiprocessing.current_process()
479 p = multiprocessing.current_process()
482 # the main process has name 'MainProcess'
480 # the main process has name 'MainProcess'
483 # subprocesses will have names like 'Process-1'
481 # subprocesses will have names like 'Process-1'
484 if p.name != 'MainProcess':
482 if p.name != 'MainProcess':
485 # we are a subprocess, don't start another Controller!
483 # we are a subprocess, don't start another Controller!
486 return
484 return
487 app = IPControllerApp.instance()
485 app = IPControllerApp.instance()
488 app.initialize()
486 app.initialize()
489 app.start()
487 app.start()
490
488
491
489
492 if __name__ == '__main__':
490 if __name__ == '__main__':
493 launch_new_instance()
491 launch_new_instance()
@@ -1,336 +1,334 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The IPython engine application
4 The IPython engine application
5
5
6 Authors:
6 Authors:
7
7
8 * Brian Granger
8 * Brian Granger
9 * MinRK
9 * MinRK
10
10
11 """
11 """
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Copyright (C) 2008-2011 The IPython Development Team
14 # Copyright (C) 2008-2011 The IPython Development Team
15 #
15 #
16 # Distributed under the terms of the BSD License. The full license is in
16 # Distributed under the terms of the BSD License. The full license is in
17 # the file COPYING, distributed as part of this software.
17 # the file COPYING, distributed as part of this software.
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Imports
21 # Imports
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 import json
24 import json
25 import os
25 import os
26 import sys
26 import sys
27 import time
27 import time
28
28
29 import zmq
29 import zmq
30 from zmq.eventloop import ioloop
30 from zmq.eventloop import ioloop
31
31
32 from IPython.core.profiledir import ProfileDir
32 from IPython.core.profiledir import ProfileDir
33 from IPython.parallel.apps.baseapp import (
33 from IPython.parallel.apps.baseapp import (
34 BaseParallelApplication,
34 BaseParallelApplication,
35 base_aliases,
35 base_aliases,
36 base_flags,
36 base_flags,
37 catch_config_error,
37 catch_config_error,
38 )
38 )
39 from IPython.zmq.log import EnginePUBHandler
39 from IPython.zmq.log import EnginePUBHandler
40 from IPython.zmq.ipkernel import Kernel
40 from IPython.zmq.ipkernel import Kernel
41 from IPython.zmq.session import (
41 from IPython.zmq.session import (
42 Session, session_aliases, session_flags
42 Session, session_aliases, session_flags
43 )
43 )
44
44
45 from IPython.config.configurable import Configurable
45 from IPython.config.configurable import Configurable
46
46
47 from IPython.parallel.engine.engine import EngineFactory
47 from IPython.parallel.engine.engine import EngineFactory
48 from IPython.parallel.util import disambiguate_url
48 from IPython.parallel.util import disambiguate_url
49
49
50 from IPython.utils.importstring import import_item
50 from IPython.utils.importstring import import_item
51 from IPython.utils.py3compat import cast_bytes
51 from IPython.utils.py3compat import cast_bytes
52 from IPython.utils.traitlets import Bool, Unicode, Dict, List, Float
52 from IPython.utils.traitlets import Bool, Unicode, Dict, List, Float
53
53
54
54
55 #-----------------------------------------------------------------------------
55 #-----------------------------------------------------------------------------
56 # Module level variables
56 # Module level variables
57 #-----------------------------------------------------------------------------
57 #-----------------------------------------------------------------------------
58
58
59 #: The default config file name for this application
59 #: The default config file name for this application
60 default_config_file_name = u'ipengine_config.py'
60 default_config_file_name = u'ipengine_config.py'
61
61
62 _description = """Start an IPython engine for parallel computing.
62 _description = """Start an IPython engine for parallel computing.
63
63
64 IPython engines run in parallel and perform computations on behalf of a client
64 IPython engines run in parallel and perform computations on behalf of a client
65 and controller. A controller needs to be started before the engines. The
65 and controller. A controller needs to be started before the engines. The
66 engine can be configured using command line options or using a cluster
66 engine can be configured using command line options or using a cluster
67 directory. Cluster directories contain config, log and security files and are
67 directory. Cluster directories contain config, log and security files and are
68 usually located in your ipython directory and named as "profile_name".
68 usually located in your ipython directory and named as "profile_name".
69 See the `profile` and `profile-dir` options for details.
69 See the `profile` and `profile-dir` options for details.
70 """
70 """
71
71
72 _examples = """
72 _examples = """
73 ipengine --ip=192.168.0.1 --port=1000 # connect to hub at ip and port
73 ipengine --ip=192.168.0.1 --port=1000 # connect to hub at ip and port
74 ipengine --log-to-file --log-level=DEBUG # log to a file with DEBUG verbosity
74 ipengine --log-to-file --log-level=DEBUG # log to a file with DEBUG verbosity
75 """
75 """
76
76
77 #-----------------------------------------------------------------------------
77 #-----------------------------------------------------------------------------
78 # MPI configuration
78 # MPI configuration
79 #-----------------------------------------------------------------------------
79 #-----------------------------------------------------------------------------
80
80
81 mpi4py_init = """from mpi4py import MPI as mpi
81 mpi4py_init = """from mpi4py import MPI as mpi
82 mpi.size = mpi.COMM_WORLD.Get_size()
82 mpi.size = mpi.COMM_WORLD.Get_size()
83 mpi.rank = mpi.COMM_WORLD.Get_rank()
83 mpi.rank = mpi.COMM_WORLD.Get_rank()
84 """
84 """
85
85
86
86
87 pytrilinos_init = """from PyTrilinos import Epetra
87 pytrilinos_init = """from PyTrilinos import Epetra
88 class SimpleStruct:
88 class SimpleStruct:
89 pass
89 pass
90 mpi = SimpleStruct()
90 mpi = SimpleStruct()
91 mpi.rank = 0
91 mpi.rank = 0
92 mpi.size = 0
92 mpi.size = 0
93 """
93 """
94
94
95 class MPI(Configurable):
95 class MPI(Configurable):
96 """Configurable for MPI initialization"""
96 """Configurable for MPI initialization"""
97 use = Unicode('', config=True,
97 use = Unicode('', config=True,
98 help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).'
98 help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).'
99 )
99 )
100
100
101 def _use_changed(self, name, old, new):
101 def _use_changed(self, name, old, new):
102 # load default init script if it's not set
102 # load default init script if it's not set
103 if not self.init_script:
103 if not self.init_script:
104 self.init_script = self.default_inits.get(new, '')
104 self.init_script = self.default_inits.get(new, '')
105
105
106 init_script = Unicode('', config=True,
106 init_script = Unicode('', config=True,
107 help="Initialization code for MPI")
107 help="Initialization code for MPI")
108
108
109 default_inits = Dict({'mpi4py' : mpi4py_init, 'pytrilinos':pytrilinos_init},
109 default_inits = Dict({'mpi4py' : mpi4py_init, 'pytrilinos':pytrilinos_init},
110 config=True)
110 config=True)
111
111
112
112
113 #-----------------------------------------------------------------------------
113 #-----------------------------------------------------------------------------
114 # Main application
114 # Main application
115 #-----------------------------------------------------------------------------
115 #-----------------------------------------------------------------------------
116 aliases = dict(
116 aliases = dict(
117 file = 'IPEngineApp.url_file',
117 file = 'IPEngineApp.url_file',
118 c = 'IPEngineApp.startup_command',
118 c = 'IPEngineApp.startup_command',
119 s = 'IPEngineApp.startup_script',
119 s = 'IPEngineApp.startup_script',
120
120
121 url = 'EngineFactory.url',
121 url = 'EngineFactory.url',
122 ssh = 'EngineFactory.sshserver',
122 ssh = 'EngineFactory.sshserver',
123 sshkey = 'EngineFactory.sshkey',
123 sshkey = 'EngineFactory.sshkey',
124 ip = 'EngineFactory.ip',
124 ip = 'EngineFactory.ip',
125 transport = 'EngineFactory.transport',
125 transport = 'EngineFactory.transport',
126 port = 'EngineFactory.regport',
126 port = 'EngineFactory.regport',
127 location = 'EngineFactory.location',
127 location = 'EngineFactory.location',
128
128
129 timeout = 'EngineFactory.timeout',
129 timeout = 'EngineFactory.timeout',
130
130
131 mpi = 'MPI.use',
131 mpi = 'MPI.use',
132
132
133 )
133 )
134 aliases.update(base_aliases)
134 aliases.update(base_aliases)
135 aliases.update(session_aliases)
135 aliases.update(session_aliases)
136 flags = {}
136 flags = {}
137 flags.update(base_flags)
137 flags.update(base_flags)
138 flags.update(session_flags)
138 flags.update(session_flags)
139
139
140 class IPEngineApp(BaseParallelApplication):
140 class IPEngineApp(BaseParallelApplication):
141
141
142 name = 'ipengine'
142 name = 'ipengine'
143 description = _description
143 description = _description
144 examples = _examples
144 examples = _examples
145 config_file_name = Unicode(default_config_file_name)
145 config_file_name = Unicode(default_config_file_name)
146 classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI])
146 classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI])
147
147
148 startup_script = Unicode(u'', config=True,
148 startup_script = Unicode(u'', config=True,
149 help='specify a script to be run at startup')
149 help='specify a script to be run at startup')
150 startup_command = Unicode('', config=True,
150 startup_command = Unicode('', config=True,
151 help='specify a command to be run at startup')
151 help='specify a command to be run at startup')
152
152
153 url_file = Unicode(u'', config=True,
153 url_file = Unicode(u'', config=True,
154 help="""The full location of the file containing the connection information for
154 help="""The full location of the file containing the connection information for
155 the controller. If this is not given, the file must be in the
155 the controller. If this is not given, the file must be in the
156 security directory of the cluster directory. This location is
156 security directory of the cluster directory. This location is
157 resolved using the `profile` or `profile_dir` options.""",
157 resolved using the `profile` or `profile_dir` options.""",
158 )
158 )
159 wait_for_url_file = Float(5, config=True,
159 wait_for_url_file = Float(5, config=True,
160 help="""The maximum number of seconds to wait for url_file to exist.
160 help="""The maximum number of seconds to wait for url_file to exist.
161 This is useful for batch-systems and shared-filesystems where the
161 This is useful for batch-systems and shared-filesystems where the
162 controller and engine are started at the same time and it
162 controller and engine are started at the same time and it
163 may take a moment for the controller to write the connector files.""")
163 may take a moment for the controller to write the connector files.""")
164
164
165 url_file_name = Unicode(u'ipcontroller-engine.json', config=True)
165 url_file_name = Unicode(u'ipcontroller-engine.json', config=True)
166
166
167 def _cluster_id_changed(self, name, old, new):
167 def _cluster_id_changed(self, name, old, new):
168 if new:
168 if new:
169 base = 'ipcontroller-%s' % new
169 base = 'ipcontroller-%s' % new
170 else:
170 else:
171 base = 'ipcontroller'
171 base = 'ipcontroller'
172 self.url_file_name = "%s-engine.json" % base
172 self.url_file_name = "%s-engine.json" % base
173
173
174 log_url = Unicode('', config=True,
174 log_url = Unicode('', config=True,
175 help="""The URL for the iploggerapp instance, for forwarding
175 help="""The URL for the iploggerapp instance, for forwarding
176 logging to a central location.""")
176 logging to a central location.""")
177
177
178 aliases = Dict(aliases)
178 aliases = Dict(aliases)
179 flags = Dict(flags)
179 flags = Dict(flags)
180
180
181 @property
181 @property
182 def kernel(self):
182 def kernel(self):
183 """allow access to the Kernel object, so I look like IPKernelApp"""
183 """allow access to the Kernel object, so I look like IPKernelApp"""
184 return self.engine.kernel
184 return self.engine.kernel
185
185
186 def find_url_file(self):
186 def find_url_file(self):
187 """Set the url file.
187 """Set the url file.
188
188
189 Here we don't try to actually see if it exists for is valid as that
189 Here we don't try to actually see if it exists for is valid as that
190 is hadled by the connection logic.
190 is hadled by the connection logic.
191 """
191 """
192 config = self.config
192 config = self.config
193 # Find the actual controller key file
193 # Find the actual controller key file
194 if not self.url_file:
194 if not self.url_file:
195 self.url_file = os.path.join(
195 self.url_file = os.path.join(
196 self.profile_dir.security_dir,
196 self.profile_dir.security_dir,
197 self.url_file_name
197 self.url_file_name
198 )
198 )
199
199
200 def load_connector_file(self):
200 def load_connector_file(self):
201 """load config from a JSON connector file,
201 """load config from a JSON connector file,
202 at a *lower* priority than command-line/config files.
202 at a *lower* priority than command-line/config files.
203 """
203 """
204
204
205 self.log.info("Loading url_file %r", self.url_file)
205 self.log.info("Loading url_file %r", self.url_file)
206 config = self.config
206 config = self.config
207
207
208 with open(self.url_file) as f:
208 with open(self.url_file) as f:
209 d = json.loads(f.read())
209 d = json.loads(f.read())
210
210
211 if 'exec_key' in d:
211 if 'exec_key' in d:
212 config.Session.key = cast_bytes(d['exec_key'])
212 config.Session.key = cast_bytes(d['exec_key'])
213
213
214 try:
214 try:
215 config.EngineFactory.location
215 config.EngineFactory.location
216 except AttributeError:
216 except AttributeError:
217 config.EngineFactory.location = d['location']
217 config.EngineFactory.location = d['location']
218
218
219 d['url'] = disambiguate_url(d['url'], config.EngineFactory.location)
219 d['url'] = disambiguate_url(d['url'], config.EngineFactory.location)
220 try:
220 try:
221 config.EngineFactory.url
221 config.EngineFactory.url
222 except AttributeError:
222 except AttributeError:
223 config.EngineFactory.url = d['url']
223 config.EngineFactory.url = d['url']
224
224
225 try:
225 try:
226 config.EngineFactory.sshserver
226 config.EngineFactory.sshserver
227 except AttributeError:
227 except AttributeError:
228 config.EngineFactory.sshserver = d['ssh']
228 config.EngineFactory.sshserver = d['ssh']
229
229
230 def init_engine(self):
230 def init_engine(self):
231 # This is the working dir by now.
231 # This is the working dir by now.
232 sys.path.insert(0, '')
232 sys.path.insert(0, '')
233 config = self.config
233 config = self.config
234 # print config
234 # print config
235 self.find_url_file()
235 self.find_url_file()
236
236
237 # was the url manually specified?
237 # was the url manually specified?
238 keys = set(self.config.EngineFactory.keys())
238 keys = set(self.config.EngineFactory.keys())
239 keys = keys.union(set(self.config.RegistrationFactory.keys()))
239 keys = keys.union(set(self.config.RegistrationFactory.keys()))
240
240
241 if keys.intersection(set(['ip', 'url', 'port'])):
241 if keys.intersection(set(['ip', 'url', 'port'])):
242 # Connection info was specified, don't wait for the file
242 # Connection info was specified, don't wait for the file
243 url_specified = True
243 url_specified = True
244 self.wait_for_url_file = 0
244 self.wait_for_url_file = 0
245 else:
245 else:
246 url_specified = False
246 url_specified = False
247
247
248 if self.wait_for_url_file and not os.path.exists(self.url_file):
248 if self.wait_for_url_file and not os.path.exists(self.url_file):
249 self.log.warn("url_file %r not found", self.url_file)
249 self.log.warn("url_file %r not found", self.url_file)
250 self.log.warn("Waiting up to %.1f seconds for it to arrive.", self.wait_for_url_file)
250 self.log.warn("Waiting up to %.1f seconds for it to arrive.", self.wait_for_url_file)
251 tic = time.time()
251 tic = time.time()
252 while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file):
252 while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file):
253 # wait for url_file to exist, or until time limit
253 # wait for url_file to exist, or until time limit
254 time.sleep(0.1)
254 time.sleep(0.1)
255
255
256 if os.path.exists(self.url_file):
256 if os.path.exists(self.url_file):
257 self.load_connector_file()
257 self.load_connector_file()
258 elif not url_specified:
258 elif not url_specified:
259 self.log.fatal("Fatal: url file never arrived: %s", self.url_file)
259 self.log.fatal("Fatal: url file never arrived: %s", self.url_file)
260 self.exit(1)
260 self.exit(1)
261
261
262
262
263 try:
263 try:
264 exec_lines = config.Kernel.exec_lines
264 exec_lines = config.Kernel.exec_lines
265 except AttributeError:
265 except AttributeError:
266 config.Kernel.exec_lines = []
266 config.Kernel.exec_lines = []
267 exec_lines = config.Kernel.exec_lines
267 exec_lines = config.Kernel.exec_lines
268
268
269 if self.startup_script:
269 if self.startup_script:
270 enc = sys.getfilesystemencoding() or 'utf8'
270 enc = sys.getfilesystemencoding() or 'utf8'
271 cmd="execfile(%r)" % self.startup_script.encode(enc)
271 cmd="execfile(%r)" % self.startup_script.encode(enc)
272 exec_lines.append(cmd)
272 exec_lines.append(cmd)
273 if self.startup_command:
273 if self.startup_command:
274 exec_lines.append(self.startup_command)
274 exec_lines.append(self.startup_command)
275
275
276 # Create the underlying shell class and Engine
276 # Create the underlying shell class and Engine
277 # shell_class = import_item(self.master_config.Global.shell_class)
277 # shell_class = import_item(self.master_config.Global.shell_class)
278 # print self.config
278 # print self.config
279 try:
279 try:
280 self.engine = EngineFactory(config=config, log=self.log)
280 self.engine = EngineFactory(config=config, log=self.log)
281 except:
281 except:
282 self.log.error("Couldn't start the Engine", exc_info=True)
282 self.log.error("Couldn't start the Engine", exc_info=True)
283 self.exit(1)
283 self.exit(1)
284
284
285 def forward_logging(self):
285 def forward_logging(self):
286 if self.log_url:
286 if self.log_url:
287 self.log.info("Forwarding logging to %s", self.log_url)
287 self.log.info("Forwarding logging to %s", self.log_url)
288 context = self.engine.context
288 context = self.engine.context
289 lsock = context.socket(zmq.PUB)
289 lsock = context.socket(zmq.PUB)
290 lsock.connect(self.log_url)
290 lsock.connect(self.log_url)
291 self.log.removeHandler(self._log_handler)
292 handler = EnginePUBHandler(self.engine, lsock)
291 handler = EnginePUBHandler(self.engine, lsock)
293 handler.setLevel(self.log_level)
292 handler.setLevel(self.log_level)
294 self.log.addHandler(handler)
293 self.log.addHandler(handler)
295 self._log_handler = handler
296
294
297 def init_mpi(self):
295 def init_mpi(self):
298 global mpi
296 global mpi
299 self.mpi = MPI(config=self.config)
297 self.mpi = MPI(config=self.config)
300
298
301 mpi_import_statement = self.mpi.init_script
299 mpi_import_statement = self.mpi.init_script
302 if mpi_import_statement:
300 if mpi_import_statement:
303 try:
301 try:
304 self.log.info("Initializing MPI:")
302 self.log.info("Initializing MPI:")
305 self.log.info(mpi_import_statement)
303 self.log.info(mpi_import_statement)
306 exec mpi_import_statement in globals()
304 exec mpi_import_statement in globals()
307 except:
305 except:
308 mpi = None
306 mpi = None
309 else:
307 else:
310 mpi = None
308 mpi = None
311
309
312 @catch_config_error
310 @catch_config_error
313 def initialize(self, argv=None):
311 def initialize(self, argv=None):
314 super(IPEngineApp, self).initialize(argv)
312 super(IPEngineApp, self).initialize(argv)
315 self.init_mpi()
313 self.init_mpi()
316 self.init_engine()
314 self.init_engine()
317 self.forward_logging()
315 self.forward_logging()
318
316
319 def start(self):
317 def start(self):
320 self.engine.start()
318 self.engine.start()
321 try:
319 try:
322 self.engine.loop.start()
320 self.engine.loop.start()
323 except KeyboardInterrupt:
321 except KeyboardInterrupt:
324 self.log.critical("Engine Interrupted, shutting down...\n")
322 self.log.critical("Engine Interrupted, shutting down...\n")
325
323
326
324
327 def launch_new_instance():
325 def launch_new_instance():
328 """Create and run the IPython engine"""
326 """Create and run the IPython engine"""
329 app = IPEngineApp.instance()
327 app = IPEngineApp.instance()
330 app.initialize()
328 app.initialize()
331 app.start()
329 app.start()
332
330
333
331
334 if __name__ == '__main__':
332 if __name__ == '__main__':
335 launch_new_instance()
333 launch_new_instance()
336
334
General Comments 0
You need to be logged in to leave comments. Login now