##// END OF EJS Templates
Show invalid config message on TraitErrors during initialization...
MinRK -
Show More
@@ -1,485 +1,504 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 A base class for a configurable application.
3 A base class for a configurable application.
4
4
5 Authors:
5 Authors:
6
6
7 * Brian Granger
7 * Brian Granger
8 * Min RK
8 * Min RK
9 """
9 """
10
10
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12 # Copyright (C) 2008-2011 The IPython Development Team
12 # Copyright (C) 2008-2011 The IPython Development Team
13 #
13 #
14 # Distributed under the terms of the BSD License. The full license is in
14 # Distributed under the terms of the BSD License. The full license is in
15 # the file COPYING, distributed as part of this software.
15 # the file COPYING, distributed as part of this software.
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17
17
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19 # Imports
19 # Imports
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21
21
22 import logging
22 import logging
23 import os
23 import os
24 import re
24 import re
25 import sys
25 import sys
26 from copy import deepcopy
26 from copy import deepcopy
27 from collections import defaultdict
27 from collections import defaultdict
28
28
29 from IPython.external.decorator import decorator
30
29 from IPython.config.configurable import SingletonConfigurable
31 from IPython.config.configurable import SingletonConfigurable
30 from IPython.config.loader import (
32 from IPython.config.loader import (
31 KVArgParseConfigLoader, PyFileConfigLoader, Config, ArgumentError, ConfigFileNotFound,
33 KVArgParseConfigLoader, PyFileConfigLoader, Config, ArgumentError, ConfigFileNotFound,
32 )
34 )
33
35
34 from IPython.utils.traitlets import (
36 from IPython.utils.traitlets import (
35 Unicode, List, Int, Enum, Dict, Instance, TraitError
37 Unicode, List, Int, Enum, Dict, Instance, TraitError
36 )
38 )
37 from IPython.utils.importstring import import_item
39 from IPython.utils.importstring import import_item
38 from IPython.utils.text import indent, wrap_paragraphs, dedent
40 from IPython.utils.text import indent, wrap_paragraphs, dedent
39
41
40 #-----------------------------------------------------------------------------
42 #-----------------------------------------------------------------------------
41 # function for re-wrapping a helpstring
43 # function for re-wrapping a helpstring
42 #-----------------------------------------------------------------------------
44 #-----------------------------------------------------------------------------
43
45
44 #-----------------------------------------------------------------------------
46 #-----------------------------------------------------------------------------
45 # Descriptions for the various sections
47 # Descriptions for the various sections
46 #-----------------------------------------------------------------------------
48 #-----------------------------------------------------------------------------
47
49
48 # merge flags&aliases into options
50 # merge flags&aliases into options
49 option_description = """
51 option_description = """
50 Arguments that take values are actually convenience aliases to full
52 Arguments that take values are actually convenience aliases to full
51 Configurables, whose aliases are listed on the help line. For more information
53 Configurables, whose aliases are listed on the help line. For more information
52 on full configurables, see '--help-all'.
54 on full configurables, see '--help-all'.
53 """.strip() # trim newlines of front and back
55 """.strip() # trim newlines of front and back
54
56
55 keyvalue_description = """
57 keyvalue_description = """
56 Parameters are set from command-line arguments of the form:
58 Parameters are set from command-line arguments of the form:
57 `--Class.trait=value`.
59 `--Class.trait=value`.
58 This line is evaluated in Python, so simple expressions are allowed, e.g.::
60 This line is evaluated in Python, so simple expressions are allowed, e.g.::
59 `--C.a='range(3)'` For setting C.a=[0,1,2].
61 `--C.a='range(3)'` For setting C.a=[0,1,2].
60 """.strip() # trim newlines of front and back
62 """.strip() # trim newlines of front and back
61
63
62 subcommand_description = """
64 subcommand_description = """
63 Subcommands are launched as `{app} cmd [args]`. For information on using
65 Subcommands are launched as `{app} cmd [args]`. For information on using
64 subcommand 'cmd', do: `{app} cmd -h`.
66 subcommand 'cmd', do: `{app} cmd -h`.
65 """.strip().format(app=os.path.basename(sys.argv[0]))
67 """.strip().format(app=os.path.basename(sys.argv[0]))
66 # get running program name
68 # get running program name
67
69
68 #-----------------------------------------------------------------------------
70 #-----------------------------------------------------------------------------
69 # Application class
71 # Application class
70 #-----------------------------------------------------------------------------
72 #-----------------------------------------------------------------------------
71
73
74 @decorator
75 def catch_config(method, app, *args, **kwargs):
76 """Method decorator for catching invalid config (Trait/ArgumentErrors) during init.
77
78 On a TraitError (generally caused by bad config), this will print the trait's
79 message, and exit the app.
80
81 For use on init methods, to prevent invoking excepthook on invalid input.
82 """
83 try:
84 return method(app, *args, **kwargs)
85 except (TraitError, ArgumentError) as e:
86 app.print_description()
87 app.print_help()
88 app.print_examples()
89 app.log.fatal("Bad config encountered during initialization:")
90 app.log.fatal(str(e))
91 app.log.debug("Config at the time: %s", app.config)
92 app.exit(1)
93
72
94
73 class ApplicationError(Exception):
95 class ApplicationError(Exception):
74 pass
96 pass
75
97
76
98
77 class Application(SingletonConfigurable):
99 class Application(SingletonConfigurable):
78 """A singleton application with full configuration support."""
100 """A singleton application with full configuration support."""
79
101
80 # The name of the application, will usually match the name of the command
102 # The name of the application, will usually match the name of the command
81 # line application
103 # line application
82 name = Unicode(u'application')
104 name = Unicode(u'application')
83
105
84 # The description of the application that is printed at the beginning
106 # The description of the application that is printed at the beginning
85 # of the help.
107 # of the help.
86 description = Unicode(u'This is an application.')
108 description = Unicode(u'This is an application.')
87 # default section descriptions
109 # default section descriptions
88 option_description = Unicode(option_description)
110 option_description = Unicode(option_description)
89 keyvalue_description = Unicode(keyvalue_description)
111 keyvalue_description = Unicode(keyvalue_description)
90 subcommand_description = Unicode(subcommand_description)
112 subcommand_description = Unicode(subcommand_description)
91
113
92 # The usage and example string that goes at the end of the help string.
114 # The usage and example string that goes at the end of the help string.
93 examples = Unicode()
115 examples = Unicode()
94
116
95 # A sequence of Configurable subclasses whose config=True attributes will
117 # A sequence of Configurable subclasses whose config=True attributes will
96 # be exposed at the command line.
118 # be exposed at the command line.
97 classes = List([])
119 classes = List([])
98
120
99 # The version string of this application.
121 # The version string of this application.
100 version = Unicode(u'0.0')
122 version = Unicode(u'0.0')
101
123
102 # The log level for the application
124 # The log level for the application
103 log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
125 log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
104 default_value=logging.WARN,
126 default_value=logging.WARN,
105 config=True,
127 config=True,
106 help="Set the log level by value or name.")
128 help="Set the log level by value or name.")
107 def _log_level_changed(self, name, old, new):
129 def _log_level_changed(self, name, old, new):
108 """Adjust the log level when log_level is set."""
130 """Adjust the log level when log_level is set."""
109 if isinstance(new, basestring):
131 if isinstance(new, basestring):
110 new = getattr(logging, new)
132 new = getattr(logging, new)
111 self.log_level = new
133 self.log_level = new
112 self.log.setLevel(new)
134 self.log.setLevel(new)
113
135
114 # the alias map for configurables
136 # the alias map for configurables
115 aliases = Dict({'log-level' : 'Application.log_level'})
137 aliases = Dict({'log-level' : 'Application.log_level'})
116
138
117 # flags for loading Configurables or store_const style flags
139 # flags for loading Configurables or store_const style flags
118 # flags are loaded from this dict by '--key' flags
140 # flags are loaded from this dict by '--key' flags
119 # this must be a dict of two-tuples, the first element being the Config/dict
141 # this must be a dict of two-tuples, the first element being the Config/dict
120 # and the second being the help string for the flag
142 # and the second being the help string for the flag
121 flags = Dict()
143 flags = Dict()
122 def _flags_changed(self, name, old, new):
144 def _flags_changed(self, name, old, new):
123 """ensure flags dict is valid"""
145 """ensure flags dict is valid"""
124 for key,value in new.iteritems():
146 for key,value in new.iteritems():
125 assert len(value) == 2, "Bad flag: %r:%s"%(key,value)
147 assert len(value) == 2, "Bad flag: %r:%s"%(key,value)
126 assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value)
148 assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value)
127 assert isinstance(value[1], basestring), "Bad flag: %r:%s"%(key,value)
149 assert isinstance(value[1], basestring), "Bad flag: %r:%s"%(key,value)
128
150
129
151
130 # subcommands for launching other applications
152 # subcommands for launching other applications
131 # if this is not empty, this will be a parent Application
153 # if this is not empty, this will be a parent Application
132 # this must be a dict of two-tuples,
154 # this must be a dict of two-tuples,
133 # the first element being the application class/import string
155 # the first element being the application class/import string
134 # and the second being the help string for the subcommand
156 # and the second being the help string for the subcommand
135 subcommands = Dict()
157 subcommands = Dict()
136 # parse_command_line will initialize a subapp, if requested
158 # parse_command_line will initialize a subapp, if requested
137 subapp = Instance('IPython.config.application.Application', allow_none=True)
159 subapp = Instance('IPython.config.application.Application', allow_none=True)
138
160
139 # extra command-line arguments that don't set config values
161 # extra command-line arguments that don't set config values
140 extra_args = List(Unicode)
162 extra_args = List(Unicode)
141
163
142
164
143 def __init__(self, **kwargs):
165 def __init__(self, **kwargs):
144 SingletonConfigurable.__init__(self, **kwargs)
166 SingletonConfigurable.__init__(self, **kwargs)
145 # Ensure my class is in self.classes, so my attributes appear in command line
167 # Ensure my class is in self.classes, so my attributes appear in command line
146 # options and config files.
168 # options and config files.
147 if self.__class__ not in self.classes:
169 if self.__class__ not in self.classes:
148 self.classes.insert(0, self.__class__)
170 self.classes.insert(0, self.__class__)
149
171
150 self.init_logging()
172 self.init_logging()
151
173
152 def _config_changed(self, name, old, new):
174 def _config_changed(self, name, old, new):
153 SingletonConfigurable._config_changed(self, name, old, new)
175 SingletonConfigurable._config_changed(self, name, old, new)
154 self.log.debug('Config changed:')
176 self.log.debug('Config changed:')
155 self.log.debug(repr(new))
177 self.log.debug(repr(new))
156
178
157 def init_logging(self):
179 def init_logging(self):
158 """Start logging for this application.
180 """Start logging for this application.
159
181
160 The default is to log to stdout using a StreaHandler. The log level
182 The default is to log to stdout using a StreaHandler. The log level
161 starts at loggin.WARN, but this can be adjusted by setting the
183 starts at loggin.WARN, but this can be adjusted by setting the
162 ``log_level`` attribute.
184 ``log_level`` attribute.
163 """
185 """
164 self.log = logging.getLogger(self.__class__.__name__)
186 self.log = logging.getLogger(self.__class__.__name__)
165 self.log.setLevel(self.log_level)
187 self.log.setLevel(self.log_level)
166 if sys.executable.endswith('pythonw.exe'):
188 if sys.executable.endswith('pythonw.exe'):
167 # this should really go to a file, but file-logging is only
189 # this should really go to a file, but file-logging is only
168 # hooked up in parallel applications
190 # hooked up in parallel applications
169 self._log_handler = logging.StreamHandler(open(os.devnull, 'w'))
191 self._log_handler = logging.StreamHandler(open(os.devnull, 'w'))
170 else:
192 else:
171 self._log_handler = logging.StreamHandler()
193 self._log_handler = logging.StreamHandler()
172 self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
194 self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
173 self._log_handler.setFormatter(self._log_formatter)
195 self._log_handler.setFormatter(self._log_formatter)
174 self.log.addHandler(self._log_handler)
196 self.log.addHandler(self._log_handler)
175
197
198 @catch_config
176 def initialize(self, argv=None):
199 def initialize(self, argv=None):
177 """Do the basic steps to configure me.
200 """Do the basic steps to configure me.
178
201
179 Override in subclasses.
202 Override in subclasses.
180 """
203 """
181 self.parse_command_line(argv)
204 self.parse_command_line(argv)
182
205
183
206
184 def start(self):
207 def start(self):
185 """Start the app mainloop.
208 """Start the app mainloop.
186
209
187 Override in subclasses.
210 Override in subclasses.
188 """
211 """
189 if self.subapp is not None:
212 if self.subapp is not None:
190 return self.subapp.start()
213 return self.subapp.start()
191
214
192 def print_alias_help(self):
215 def print_alias_help(self):
193 """Print the alias part of the help."""
216 """Print the alias part of the help."""
194 if not self.aliases:
217 if not self.aliases:
195 return
218 return
196
219
197 lines = []
220 lines = []
198 classdict = {}
221 classdict = {}
199 for cls in self.classes:
222 for cls in self.classes:
200 # include all parents (up to, but excluding Configurable) in available names
223 # include all parents (up to, but excluding Configurable) in available names
201 for c in cls.mro()[:-3]:
224 for c in cls.mro()[:-3]:
202 classdict[c.__name__] = c
225 classdict[c.__name__] = c
203
226
204 for alias, longname in self.aliases.iteritems():
227 for alias, longname in self.aliases.iteritems():
205 classname, traitname = longname.split('.',1)
228 classname, traitname = longname.split('.',1)
206 cls = classdict[classname]
229 cls = classdict[classname]
207
230
208 trait = cls.class_traits(config=True)[traitname]
231 trait = cls.class_traits(config=True)[traitname]
209 help = cls.class_get_trait_help(trait).splitlines()
232 help = cls.class_get_trait_help(trait).splitlines()
210 # reformat first line
233 # reformat first line
211 help[0] = help[0].replace(longname, alias) + ' (%s)'%longname
234 help[0] = help[0].replace(longname, alias) + ' (%s)'%longname
212 if len(alias) == 1:
235 if len(alias) == 1:
213 help[0] = help[0].replace('--%s='%alias, '-%s '%alias)
236 help[0] = help[0].replace('--%s='%alias, '-%s '%alias)
214 lines.extend(help)
237 lines.extend(help)
215 # lines.append('')
238 # lines.append('')
216 print os.linesep.join(lines)
239 print os.linesep.join(lines)
217
240
218 def print_flag_help(self):
241 def print_flag_help(self):
219 """Print the flag part of the help."""
242 """Print the flag part of the help."""
220 if not self.flags:
243 if not self.flags:
221 return
244 return
222
245
223 lines = []
246 lines = []
224 for m, (cfg,help) in self.flags.iteritems():
247 for m, (cfg,help) in self.flags.iteritems():
225 prefix = '--' if len(m) > 1 else '-'
248 prefix = '--' if len(m) > 1 else '-'
226 lines.append(prefix+m)
249 lines.append(prefix+m)
227 lines.append(indent(dedent(help.strip())))
250 lines.append(indent(dedent(help.strip())))
228 # lines.append('')
251 # lines.append('')
229 print os.linesep.join(lines)
252 print os.linesep.join(lines)
230
253
231 def print_options(self):
254 def print_options(self):
232 if not self.flags and not self.aliases:
255 if not self.flags and not self.aliases:
233 return
256 return
234 lines = ['Options']
257 lines = ['Options']
235 lines.append('-'*len(lines[0]))
258 lines.append('-'*len(lines[0]))
236 lines.append('')
259 lines.append('')
237 for p in wrap_paragraphs(self.option_description):
260 for p in wrap_paragraphs(self.option_description):
238 lines.append(p)
261 lines.append(p)
239 lines.append('')
262 lines.append('')
240 print os.linesep.join(lines)
263 print os.linesep.join(lines)
241 self.print_flag_help()
264 self.print_flag_help()
242 self.print_alias_help()
265 self.print_alias_help()
243 print
266 print
244
267
245 def print_subcommands(self):
268 def print_subcommands(self):
246 """Print the subcommand part of the help."""
269 """Print the subcommand part of the help."""
247 if not self.subcommands:
270 if not self.subcommands:
248 return
271 return
249
272
250 lines = ["Subcommands"]
273 lines = ["Subcommands"]
251 lines.append('-'*len(lines[0]))
274 lines.append('-'*len(lines[0]))
252 lines.append('')
275 lines.append('')
253 for p in wrap_paragraphs(self.subcommand_description):
276 for p in wrap_paragraphs(self.subcommand_description):
254 lines.append(p)
277 lines.append(p)
255 lines.append('')
278 lines.append('')
256 for subc, (cls, help) in self.subcommands.iteritems():
279 for subc, (cls, help) in self.subcommands.iteritems():
257 lines.append(subc)
280 lines.append(subc)
258 if help:
281 if help:
259 lines.append(indent(dedent(help.strip())))
282 lines.append(indent(dedent(help.strip())))
260 lines.append('')
283 lines.append('')
261 print os.linesep.join(lines)
284 print os.linesep.join(lines)
262
285
263 def print_help(self, classes=False):
286 def print_help(self, classes=False):
264 """Print the help for each Configurable class in self.classes.
287 """Print the help for each Configurable class in self.classes.
265
288
266 If classes=False (the default), only flags and aliases are printed.
289 If classes=False (the default), only flags and aliases are printed.
267 """
290 """
268 self.print_subcommands()
291 self.print_subcommands()
269 self.print_options()
292 self.print_options()
270
293
271 if classes:
294 if classes:
272 if self.classes:
295 if self.classes:
273 print "Class parameters"
296 print "Class parameters"
274 print "----------------"
297 print "----------------"
275 print
298 print
276 for p in wrap_paragraphs(self.keyvalue_description):
299 for p in wrap_paragraphs(self.keyvalue_description):
277 print p
300 print p
278 print
301 print
279
302
280 for cls in self.classes:
303 for cls in self.classes:
281 cls.class_print_help()
304 cls.class_print_help()
282 print
305 print
283 else:
306 else:
284 print "To see all available configurables, use `--help-all`"
307 print "To see all available configurables, use `--help-all`"
285 print
308 print
286
309
287 def print_description(self):
310 def print_description(self):
288 """Print the application description."""
311 """Print the application description."""
289 for p in wrap_paragraphs(self.description):
312 for p in wrap_paragraphs(self.description):
290 print p
313 print p
291 print
314 print
292
315
293 def print_examples(self):
316 def print_examples(self):
294 """Print usage and examples.
317 """Print usage and examples.
295
318
296 This usage string goes at the end of the command line help string
319 This usage string goes at the end of the command line help string
297 and should contain examples of the application's usage.
320 and should contain examples of the application's usage.
298 """
321 """
299 if self.examples:
322 if self.examples:
300 print "Examples"
323 print "Examples"
301 print "--------"
324 print "--------"
302 print
325 print
303 print indent(dedent(self.examples.strip()))
326 print indent(dedent(self.examples.strip()))
304 print
327 print
305
328
306 def print_version(self):
329 def print_version(self):
307 """Print the version string."""
330 """Print the version string."""
308 print self.version
331 print self.version
309
332
310 def update_config(self, config):
333 def update_config(self, config):
311 """Fire the traits events when the config is updated."""
334 """Fire the traits events when the config is updated."""
312 # Save a copy of the current config.
335 # Save a copy of the current config.
313 newconfig = deepcopy(self.config)
336 newconfig = deepcopy(self.config)
314 # Merge the new config into the current one.
337 # Merge the new config into the current one.
315 newconfig._merge(config)
338 newconfig._merge(config)
316 # Save the combined config as self.config, which triggers the traits
339 # Save the combined config as self.config, which triggers the traits
317 # events.
340 # events.
318 self.config = newconfig
341 self.config = newconfig
319
342
343 @catch_config
320 def initialize_subcommand(self, subc, argv=None):
344 def initialize_subcommand(self, subc, argv=None):
321 """Initialize a subcommand with argv."""
345 """Initialize a subcommand with argv."""
322 subapp,help = self.subcommands.get(subc)
346 subapp,help = self.subcommands.get(subc)
323
347
324 if isinstance(subapp, basestring):
348 if isinstance(subapp, basestring):
325 subapp = import_item(subapp)
349 subapp = import_item(subapp)
326
350
327 # clear existing instances
351 # clear existing instances
328 self.__class__.clear_instance()
352 self.__class__.clear_instance()
329 # instantiate
353 # instantiate
330 self.subapp = subapp.instance()
354 self.subapp = subapp.instance()
331 # and initialize subapp
355 # and initialize subapp
332 self.subapp.initialize(argv)
356 self.subapp.initialize(argv)
333
357
334 def flatten_flags(self):
358 def flatten_flags(self):
335 """flatten flags and aliases, so cl-args override as expected.
359 """flatten flags and aliases, so cl-args override as expected.
336
360
337 This prevents issues such as an alias pointing to InteractiveShell,
361 This prevents issues such as an alias pointing to InteractiveShell,
338 but a config file setting the same trait in TerminalInteraciveShell
362 but a config file setting the same trait in TerminalInteraciveShell
339 getting inappropriate priority over the command-line arg.
363 getting inappropriate priority over the command-line arg.
340
364
341 Only aliases with exactly one descendent in the class list
365 Only aliases with exactly one descendent in the class list
342 will be promoted.
366 will be promoted.
343
367
344 """
368 """
345 # build a tree of classes in our list that inherit from a particular
369 # build a tree of classes in our list that inherit from a particular
346 # it will be a dict by parent classname of classes in our list
370 # it will be a dict by parent classname of classes in our list
347 # that are descendents
371 # that are descendents
348 mro_tree = defaultdict(list)
372 mro_tree = defaultdict(list)
349 for cls in self.classes:
373 for cls in self.classes:
350 clsname = cls.__name__
374 clsname = cls.__name__
351 for parent in cls.mro()[1:-3]:
375 for parent in cls.mro()[1:-3]:
352 # exclude cls itself and Configurable,HasTraits,object
376 # exclude cls itself and Configurable,HasTraits,object
353 mro_tree[parent.__name__].append(clsname)
377 mro_tree[parent.__name__].append(clsname)
354 # flatten aliases, which have the form:
378 # flatten aliases, which have the form:
355 # { 'alias' : 'Class.trait' }
379 # { 'alias' : 'Class.trait' }
356 aliases = {}
380 aliases = {}
357 for alias, cls_trait in self.aliases.iteritems():
381 for alias, cls_trait in self.aliases.iteritems():
358 cls,trait = cls_trait.split('.',1)
382 cls,trait = cls_trait.split('.',1)
359 children = mro_tree[cls]
383 children = mro_tree[cls]
360 if len(children) == 1:
384 if len(children) == 1:
361 # exactly one descendent, promote alias
385 # exactly one descendent, promote alias
362 cls = children[0]
386 cls = children[0]
363 aliases[alias] = '.'.join([cls,trait])
387 aliases[alias] = '.'.join([cls,trait])
364
388
365 # flatten flags, which are of the form:
389 # flatten flags, which are of the form:
366 # { 'key' : ({'Cls' : {'trait' : value}}, 'help')}
390 # { 'key' : ({'Cls' : {'trait' : value}}, 'help')}
367 flags = {}
391 flags = {}
368 for key, (flagdict, help) in self.flags.iteritems():
392 for key, (flagdict, help) in self.flags.iteritems():
369 newflag = {}
393 newflag = {}
370 for cls, subdict in flagdict.iteritems():
394 for cls, subdict in flagdict.iteritems():
371 children = mro_tree[cls]
395 children = mro_tree[cls]
372 # exactly one descendent, promote flag section
396 # exactly one descendent, promote flag section
373 if len(children) == 1:
397 if len(children) == 1:
374 cls = children[0]
398 cls = children[0]
375 newflag[cls] = subdict
399 newflag[cls] = subdict
376 flags[key] = (newflag, help)
400 flags[key] = (newflag, help)
377 return flags, aliases
401 return flags, aliases
378
402
403 @catch_config
379 def parse_command_line(self, argv=None):
404 def parse_command_line(self, argv=None):
380 """Parse the command line arguments."""
405 """Parse the command line arguments."""
381 argv = sys.argv[1:] if argv is None else argv
406 argv = sys.argv[1:] if argv is None else argv
382
407
383 if self.subcommands and len(argv) > 0:
408 if self.subcommands and len(argv) > 0:
384 # we have subcommands, and one may have been specified
409 # we have subcommands, and one may have been specified
385 subc, subargv = argv[0], argv[1:]
410 subc, subargv = argv[0], argv[1:]
386 if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands:
411 if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands:
387 # it's a subcommand, and *not* a flag or class parameter
412 # it's a subcommand, and *not* a flag or class parameter
388 return self.initialize_subcommand(subc, subargv)
413 return self.initialize_subcommand(subc, subargv)
389
414
390 if '-h' in argv or '--help' in argv or '--help-all' in argv:
415 if '-h' in argv or '--help' in argv or '--help-all' in argv:
391 self.print_description()
416 self.print_description()
392 self.print_help('--help-all' in argv)
417 self.print_help('--help-all' in argv)
393 self.print_examples()
418 self.print_examples()
394 self.exit(0)
419 self.exit(0)
395
420
396 if '--version' in argv:
421 if '--version' in argv:
397 self.print_version()
422 self.print_version()
398 self.exit(0)
423 self.exit(0)
399
424
400 # flatten flags&aliases, so cl-args get appropriate priority:
425 # flatten flags&aliases, so cl-args get appropriate priority:
401 flags,aliases = self.flatten_flags()
426 flags,aliases = self.flatten_flags()
402
427
403 loader = KVArgParseConfigLoader(argv=argv, aliases=aliases,
428 loader = KVArgParseConfigLoader(argv=argv, aliases=aliases,
404 flags=flags)
429 flags=flags)
405 try:
406 config = loader.load_config()
430 config = loader.load_config()
407 self.update_config(config)
431 self.update_config(config)
408 except (TraitError, ArgumentError) as e:
409 self.print_description()
410 self.print_help()
411 self.print_examples()
412 self.log.fatal(str(e))
413 self.exit(1)
414 # store unparsed args in extra_args
432 # store unparsed args in extra_args
415 self.extra_args = loader.extra_args
433 self.extra_args = loader.extra_args
416
434
435 @catch_config
417 def load_config_file(self, filename, path=None):
436 def load_config_file(self, filename, path=None):
418 """Load a .py based config file by filename and path."""
437 """Load a .py based config file by filename and path."""
419 loader = PyFileConfigLoader(filename, path=path)
438 loader = PyFileConfigLoader(filename, path=path)
420 try:
439 try:
421 config = loader.load_config()
440 config = loader.load_config()
422 except ConfigFileNotFound:
441 except ConfigFileNotFound:
423 # problem finding the file, raise
442 # problem finding the file, raise
424 raise
443 raise
425 except Exception:
444 except Exception:
426 # try to get the full filename, but it will be empty in the
445 # try to get the full filename, but it will be empty in the
427 # unlikely event that the error raised before filefind finished
446 # unlikely event that the error raised before filefind finished
428 filename = loader.full_filename or filename
447 filename = loader.full_filename or filename
429 # problem while running the file
448 # problem while running the file
430 self.log.error("Exception while loading config file %s",
449 self.log.error("Exception while loading config file %s",
431 filename, exc_info=True)
450 filename, exc_info=True)
432 else:
451 else:
433 self.log.debug("Loaded config file: %s", loader.full_filename)
452 self.log.debug("Loaded config file: %s", loader.full_filename)
434 self.update_config(config)
453 self.update_config(config)
435
454
436 def generate_config_file(self):
455 def generate_config_file(self):
437 """generate default config file from Configurables"""
456 """generate default config file from Configurables"""
438 lines = ["# Configuration file for %s."%self.name]
457 lines = ["# Configuration file for %s."%self.name]
439 lines.append('')
458 lines.append('')
440 lines.append('c = get_config()')
459 lines.append('c = get_config()')
441 lines.append('')
460 lines.append('')
442 for cls in self.classes:
461 for cls in self.classes:
443 lines.append(cls.class_config_section())
462 lines.append(cls.class_config_section())
444 return '\n'.join(lines)
463 return '\n'.join(lines)
445
464
446 def exit(self, exit_status=0):
465 def exit(self, exit_status=0):
447 self.log.debug("Exiting application: %s" % self.name)
466 self.log.debug("Exiting application: %s" % self.name)
448 sys.exit(exit_status)
467 sys.exit(exit_status)
449
468
450 #-----------------------------------------------------------------------------
469 #-----------------------------------------------------------------------------
451 # utility functions, for convenience
470 # utility functions, for convenience
452 #-----------------------------------------------------------------------------
471 #-----------------------------------------------------------------------------
453
472
454 def boolean_flag(name, configurable, set_help='', unset_help=''):
473 def boolean_flag(name, configurable, set_help='', unset_help=''):
455 """Helper for building basic --trait, --no-trait flags.
474 """Helper for building basic --trait, --no-trait flags.
456
475
457 Parameters
476 Parameters
458 ----------
477 ----------
459
478
460 name : str
479 name : str
461 The name of the flag.
480 The name of the flag.
462 configurable : str
481 configurable : str
463 The 'Class.trait' string of the trait to be set/unset with the flag
482 The 'Class.trait' string of the trait to be set/unset with the flag
464 set_help : unicode
483 set_help : unicode
465 help string for --name flag
484 help string for --name flag
466 unset_help : unicode
485 unset_help : unicode
467 help string for --no-name flag
486 help string for --no-name flag
468
487
469 Returns
488 Returns
470 -------
489 -------
471
490
472 cfg : dict
491 cfg : dict
473 A dict with two keys: 'name', and 'no-name', for setting and unsetting
492 A dict with two keys: 'name', and 'no-name', for setting and unsetting
474 the trait, respectively.
493 the trait, respectively.
475 """
494 """
476 # default helpstrings
495 # default helpstrings
477 set_help = set_help or "set %s=True"%configurable
496 set_help = set_help or "set %s=True"%configurable
478 unset_help = unset_help or "set %s=False"%configurable
497 unset_help = unset_help or "set %s=False"%configurable
479
498
480 cls,trait = configurable.split('.')
499 cls,trait = configurable.split('.')
481
500
482 setter = {cls : {trait : True}}
501 setter = {cls : {trait : True}}
483 unsetter = {cls : {trait : False}}
502 unsetter = {cls : {trait : False}}
484 return {name : (setter, set_help), 'no-'+name : (unsetter, unset_help)}
503 return {name : (setter, set_help), 'no-'+name : (unsetter, unset_help)}
485
504
@@ -1,321 +1,321 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 An application for IPython.
3 An application for IPython.
4
4
5 All top-level applications should use the classes in this module for
5 All top-level applications should use the classes in this module for
6 handling configuration and creating componenets.
6 handling configuration and creating componenets.
7
7
8 The job of an :class:`Application` is to create the master configuration
8 The job of an :class:`Application` is to create the master configuration
9 object and then create the configurable objects, passing the config to them.
9 object and then create the configurable objects, passing the config to them.
10
10
11 Authors:
11 Authors:
12
12
13 * Brian Granger
13 * Brian Granger
14 * Fernando Perez
14 * Fernando Perez
15 * Min RK
15 * Min RK
16
16
17 """
17 """
18
18
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20 # Copyright (C) 2008-2011 The IPython Development Team
20 # Copyright (C) 2008-2011 The IPython Development Team
21 #
21 #
22 # Distributed under the terms of the BSD License. The full license is in
22 # Distributed under the terms of the BSD License. The full license is in
23 # the file COPYING, distributed as part of this software.
23 # the file COPYING, distributed as part of this software.
24 #-----------------------------------------------------------------------------
24 #-----------------------------------------------------------------------------
25
25
26 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
27 # Imports
27 # Imports
28 #-----------------------------------------------------------------------------
28 #-----------------------------------------------------------------------------
29
29
30 import atexit
30 import atexit
31 import glob
31 import glob
32 import logging
32 import logging
33 import os
33 import os
34 import shutil
34 import shutil
35 import sys
35 import sys
36
36
37 from IPython.config.application import Application
37 from IPython.config.application import Application, catch_config
38 from IPython.config.configurable import Configurable
38 from IPython.config.configurable import Configurable
39 from IPython.config.loader import Config, ConfigFileNotFound
39 from IPython.config.loader import Config, ConfigFileNotFound
40 from IPython.core import release, crashhandler
40 from IPython.core import release, crashhandler
41 from IPython.core.profiledir import ProfileDir, ProfileDirError
41 from IPython.core.profiledir import ProfileDir, ProfileDirError
42 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
42 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
43 from IPython.utils.traitlets import List, Unicode, Type, Bool, Dict
43 from IPython.utils.traitlets import List, Unicode, Type, Bool, Dict
44 from IPython.utils import py3compat
44 from IPython.utils import py3compat
45
45
46 #-----------------------------------------------------------------------------
46 #-----------------------------------------------------------------------------
47 # Classes and functions
47 # Classes and functions
48 #-----------------------------------------------------------------------------
48 #-----------------------------------------------------------------------------
49
49
50
50
51 #-----------------------------------------------------------------------------
51 #-----------------------------------------------------------------------------
52 # Base Application Class
52 # Base Application Class
53 #-----------------------------------------------------------------------------
53 #-----------------------------------------------------------------------------
54
54
55 # aliases and flags
55 # aliases and flags
56
56
57 base_aliases = {
57 base_aliases = {
58 'profile' : 'BaseIPythonApplication.profile',
58 'profile' : 'BaseIPythonApplication.profile',
59 'ipython-dir' : 'BaseIPythonApplication.ipython_dir',
59 'ipython-dir' : 'BaseIPythonApplication.ipython_dir',
60 'log-level' : 'Application.log_level',
60 'log-level' : 'Application.log_level',
61 }
61 }
62
62
63 base_flags = dict(
63 base_flags = dict(
64 debug = ({'Application' : {'log_level' : logging.DEBUG}},
64 debug = ({'Application' : {'log_level' : logging.DEBUG}},
65 "set log level to logging.DEBUG (maximize logging output)"),
65 "set log level to logging.DEBUG (maximize logging output)"),
66 quiet = ({'Application' : {'log_level' : logging.CRITICAL}},
66 quiet = ({'Application' : {'log_level' : logging.CRITICAL}},
67 "set log level to logging.CRITICAL (minimize logging output)"),
67 "set log level to logging.CRITICAL (minimize logging output)"),
68 init = ({'BaseIPythonApplication' : {
68 init = ({'BaseIPythonApplication' : {
69 'copy_config_files' : True,
69 'copy_config_files' : True,
70 'auto_create' : True}
70 'auto_create' : True}
71 }, """Initialize profile with default config files. This is equivalent
71 }, """Initialize profile with default config files. This is equivalent
72 to running `ipython profile create <profile>` prior to startup.
72 to running `ipython profile create <profile>` prior to startup.
73 """)
73 """)
74 )
74 )
75
75
76
76
77 class BaseIPythonApplication(Application):
77 class BaseIPythonApplication(Application):
78
78
79 name = Unicode(u'ipython')
79 name = Unicode(u'ipython')
80 description = Unicode(u'IPython: an enhanced interactive Python shell.')
80 description = Unicode(u'IPython: an enhanced interactive Python shell.')
81 version = Unicode(release.version)
81 version = Unicode(release.version)
82
82
83 aliases = Dict(base_aliases)
83 aliases = Dict(base_aliases)
84 flags = Dict(base_flags)
84 flags = Dict(base_flags)
85 classes = List([ProfileDir])
85 classes = List([ProfileDir])
86
86
87 # Track whether the config_file has changed,
87 # Track whether the config_file has changed,
88 # because some logic happens only if we aren't using the default.
88 # because some logic happens only if we aren't using the default.
89 config_file_specified = Bool(False)
89 config_file_specified = Bool(False)
90
90
91 config_file_name = Unicode(u'ipython_config.py')
91 config_file_name = Unicode(u'ipython_config.py')
92 def _config_file_name_default(self):
92 def _config_file_name_default(self):
93 return self.name.replace('-','_') + u'_config.py'
93 return self.name.replace('-','_') + u'_config.py'
94 def _config_file_name_changed(self, name, old, new):
94 def _config_file_name_changed(self, name, old, new):
95 if new != old:
95 if new != old:
96 self.config_file_specified = True
96 self.config_file_specified = True
97
97
98 # The directory that contains IPython's builtin profiles.
98 # The directory that contains IPython's builtin profiles.
99 builtin_profile_dir = Unicode(
99 builtin_profile_dir = Unicode(
100 os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
100 os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
101 )
101 )
102
102
103 config_file_paths = List(Unicode)
103 config_file_paths = List(Unicode)
104 def _config_file_paths_default(self):
104 def _config_file_paths_default(self):
105 return [os.getcwdu()]
105 return [os.getcwdu()]
106
106
107 profile = Unicode(u'', config=True,
107 profile = Unicode(u'', config=True,
108 help="""The IPython profile to use."""
108 help="""The IPython profile to use."""
109 )
109 )
110 def _profile_default(self):
110 def _profile_default(self):
111 return "python3" if py3compat.PY3 else "default"
111 return "python3" if py3compat.PY3 else "default"
112
112
113 def _profile_changed(self, name, old, new):
113 def _profile_changed(self, name, old, new):
114 self.builtin_profile_dir = os.path.join(
114 self.builtin_profile_dir = os.path.join(
115 get_ipython_package_dir(), u'config', u'profile', new
115 get_ipython_package_dir(), u'config', u'profile', new
116 )
116 )
117
117
118 ipython_dir = Unicode(get_ipython_dir(), config=True,
118 ipython_dir = Unicode(get_ipython_dir(), config=True,
119 help="""
119 help="""
120 The name of the IPython directory. This directory is used for logging
120 The name of the IPython directory. This directory is used for logging
121 configuration (through profiles), history storage, etc. The default
121 configuration (through profiles), history storage, etc. The default
122 is usually $HOME/.ipython. This options can also be specified through
122 is usually $HOME/.ipython. This options can also be specified through
123 the environment variable IPYTHON_DIR.
123 the environment variable IPYTHON_DIR.
124 """
124 """
125 )
125 )
126
126
127 overwrite = Bool(False, config=True,
127 overwrite = Bool(False, config=True,
128 help="""Whether to overwrite existing config files when copying""")
128 help="""Whether to overwrite existing config files when copying""")
129 auto_create = Bool(False, config=True,
129 auto_create = Bool(False, config=True,
130 help="""Whether to create profile dir if it doesn't exist""")
130 help="""Whether to create profile dir if it doesn't exist""")
131
131
132 config_files = List(Unicode)
132 config_files = List(Unicode)
133 def _config_files_default(self):
133 def _config_files_default(self):
134 return [u'ipython_config.py']
134 return [u'ipython_config.py']
135
135
136 copy_config_files = Bool(False, config=True,
136 copy_config_files = Bool(False, config=True,
137 help="""Whether to install the default config files into the profile dir.
137 help="""Whether to install the default config files into the profile dir.
138 If a new profile is being created, and IPython contains config files for that
138 If a new profile is being created, and IPython contains config files for that
139 profile, then they will be staged into the new directory. Otherwise,
139 profile, then they will be staged into the new directory. Otherwise,
140 default config files will be automatically generated.
140 default config files will be automatically generated.
141 """)
141 """)
142
142
143 # The class to use as the crash handler.
143 # The class to use as the crash handler.
144 crash_handler_class = Type(crashhandler.CrashHandler)
144 crash_handler_class = Type(crashhandler.CrashHandler)
145
145
146 def __init__(self, **kwargs):
146 def __init__(self, **kwargs):
147 super(BaseIPythonApplication, self).__init__(**kwargs)
147 super(BaseIPythonApplication, self).__init__(**kwargs)
148 # ensure even default IPYTHON_DIR exists
148 # ensure even default IPYTHON_DIR exists
149 if not os.path.exists(self.ipython_dir):
149 if not os.path.exists(self.ipython_dir):
150 self._ipython_dir_changed('ipython_dir', self.ipython_dir, self.ipython_dir)
150 self._ipython_dir_changed('ipython_dir', self.ipython_dir, self.ipython_dir)
151
151
152 #-------------------------------------------------------------------------
152 #-------------------------------------------------------------------------
153 # Various stages of Application creation
153 # Various stages of Application creation
154 #-------------------------------------------------------------------------
154 #-------------------------------------------------------------------------
155
155
156 def init_crash_handler(self):
156 def init_crash_handler(self):
157 """Create a crash handler, typically setting sys.excepthook to it."""
157 """Create a crash handler, typically setting sys.excepthook to it."""
158 self.crash_handler = self.crash_handler_class(self)
158 self.crash_handler = self.crash_handler_class(self)
159 sys.excepthook = self.crash_handler
159 sys.excepthook = self.crash_handler
160 def unset_crashhandler():
160 def unset_crashhandler():
161 sys.excepthook = sys.__excepthook__
161 sys.excepthook = sys.__excepthook__
162 atexit.register(unset_crashhandler)
162 atexit.register(unset_crashhandler)
163
163
164 def _ipython_dir_changed(self, name, old, new):
164 def _ipython_dir_changed(self, name, old, new):
165 if old in sys.path:
165 if old in sys.path:
166 sys.path.remove(old)
166 sys.path.remove(old)
167 sys.path.append(os.path.abspath(new))
167 sys.path.append(os.path.abspath(new))
168 if not os.path.isdir(new):
168 if not os.path.isdir(new):
169 os.makedirs(new, mode=0777)
169 os.makedirs(new, mode=0777)
170 readme = os.path.join(new, 'README')
170 readme = os.path.join(new, 'README')
171 if not os.path.exists(readme):
171 if not os.path.exists(readme):
172 path = os.path.join(get_ipython_package_dir(), u'config', u'profile')
172 path = os.path.join(get_ipython_package_dir(), u'config', u'profile')
173 shutil.copy(os.path.join(path, 'README'), readme)
173 shutil.copy(os.path.join(path, 'README'), readme)
174 self.log.debug("IPYTHON_DIR set to: %s" % new)
174 self.log.debug("IPYTHON_DIR set to: %s" % new)
175
175
176 def load_config_file(self, suppress_errors=True):
176 def load_config_file(self, suppress_errors=True):
177 """Load the config file.
177 """Load the config file.
178
178
179 By default, errors in loading config are handled, and a warning
179 By default, errors in loading config are handled, and a warning
180 printed on screen. For testing, the suppress_errors option is set
180 printed on screen. For testing, the suppress_errors option is set
181 to False, so errors will make tests fail.
181 to False, so errors will make tests fail.
182 """
182 """
183 self.log.debug("Searching path %s for config files", self.config_file_paths)
183 self.log.debug("Searching path %s for config files", self.config_file_paths)
184 base_config = 'ipython_config.py'
184 base_config = 'ipython_config.py'
185 self.log.debug("Attempting to load config file: %s" %
185 self.log.debug("Attempting to load config file: %s" %
186 base_config)
186 base_config)
187 try:
187 try:
188 Application.load_config_file(
188 Application.load_config_file(
189 self,
189 self,
190 base_config,
190 base_config,
191 path=self.config_file_paths
191 path=self.config_file_paths
192 )
192 )
193 except ConfigFileNotFound:
193 except ConfigFileNotFound:
194 # ignore errors loading parent
194 # ignore errors loading parent
195 self.log.debug("Config file %s not found", base_config)
195 self.log.debug("Config file %s not found", base_config)
196 pass
196 pass
197 if self.config_file_name == base_config:
197 if self.config_file_name == base_config:
198 # don't load secondary config
198 # don't load secondary config
199 return
199 return
200 self.log.debug("Attempting to load config file: %s" %
200 self.log.debug("Attempting to load config file: %s" %
201 self.config_file_name)
201 self.config_file_name)
202 try:
202 try:
203 Application.load_config_file(
203 Application.load_config_file(
204 self,
204 self,
205 self.config_file_name,
205 self.config_file_name,
206 path=self.config_file_paths
206 path=self.config_file_paths
207 )
207 )
208 except ConfigFileNotFound:
208 except ConfigFileNotFound:
209 # Only warn if the default config file was NOT being used.
209 # Only warn if the default config file was NOT being used.
210 if self.config_file_specified:
210 if self.config_file_specified:
211 msg = self.log.warn
211 msg = self.log.warn
212 else:
212 else:
213 msg = self.log.debug
213 msg = self.log.debug
214 msg("Config file not found, skipping: %s", self.config_file_name)
214 msg("Config file not found, skipping: %s", self.config_file_name)
215 except:
215 except:
216 # For testing purposes.
216 # For testing purposes.
217 if not suppress_errors:
217 if not suppress_errors:
218 raise
218 raise
219 self.log.warn("Error loading config file: %s" %
219 self.log.warn("Error loading config file: %s" %
220 self.config_file_name, exc_info=True)
220 self.config_file_name, exc_info=True)
221
221
222 def init_profile_dir(self):
222 def init_profile_dir(self):
223 """initialize the profile dir"""
223 """initialize the profile dir"""
224 try:
224 try:
225 # location explicitly specified:
225 # location explicitly specified:
226 location = self.config.ProfileDir.location
226 location = self.config.ProfileDir.location
227 except AttributeError:
227 except AttributeError:
228 # location not specified, find by profile name
228 # location not specified, find by profile name
229 try:
229 try:
230 p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
230 p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
231 except ProfileDirError:
231 except ProfileDirError:
232 # not found, maybe create it (always create default profile)
232 # not found, maybe create it (always create default profile)
233 if self.auto_create or self.profile==self._profile_default():
233 if self.auto_create or self.profile==self._profile_default():
234 try:
234 try:
235 p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
235 p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
236 except ProfileDirError:
236 except ProfileDirError:
237 self.log.fatal("Could not create profile: %r"%self.profile)
237 self.log.fatal("Could not create profile: %r"%self.profile)
238 self.exit(1)
238 self.exit(1)
239 else:
239 else:
240 self.log.info("Created profile dir: %r"%p.location)
240 self.log.info("Created profile dir: %r"%p.location)
241 else:
241 else:
242 self.log.fatal("Profile %r not found."%self.profile)
242 self.log.fatal("Profile %r not found."%self.profile)
243 self.exit(1)
243 self.exit(1)
244 else:
244 else:
245 self.log.info("Using existing profile dir: %r"%p.location)
245 self.log.info("Using existing profile dir: %r"%p.location)
246 else:
246 else:
247 # location is fully specified
247 # location is fully specified
248 try:
248 try:
249 p = ProfileDir.find_profile_dir(location, self.config)
249 p = ProfileDir.find_profile_dir(location, self.config)
250 except ProfileDirError:
250 except ProfileDirError:
251 # not found, maybe create it
251 # not found, maybe create it
252 if self.auto_create:
252 if self.auto_create:
253 try:
253 try:
254 p = ProfileDir.create_profile_dir(location, self.config)
254 p = ProfileDir.create_profile_dir(location, self.config)
255 except ProfileDirError:
255 except ProfileDirError:
256 self.log.fatal("Could not create profile directory: %r"%location)
256 self.log.fatal("Could not create profile directory: %r"%location)
257 self.exit(1)
257 self.exit(1)
258 else:
258 else:
259 self.log.info("Creating new profile dir: %r"%location)
259 self.log.info("Creating new profile dir: %r"%location)
260 else:
260 else:
261 self.log.fatal("Profile directory %r not found."%location)
261 self.log.fatal("Profile directory %r not found."%location)
262 self.exit(1)
262 self.exit(1)
263 else:
263 else:
264 self.log.info("Using existing profile dir: %r"%location)
264 self.log.info("Using existing profile dir: %r"%location)
265
265
266 self.profile_dir = p
266 self.profile_dir = p
267 self.config_file_paths.append(p.location)
267 self.config_file_paths.append(p.location)
268
268
269 def init_config_files(self):
269 def init_config_files(self):
270 """[optionally] copy default config files into profile dir."""
270 """[optionally] copy default config files into profile dir."""
271 # copy config files
271 # copy config files
272 path = self.builtin_profile_dir
272 path = self.builtin_profile_dir
273 if self.copy_config_files:
273 if self.copy_config_files:
274 src = self.profile
274 src = self.profile
275
275
276 cfg = self.config_file_name
276 cfg = self.config_file_name
277 if path and os.path.exists(os.path.join(path, cfg)):
277 if path and os.path.exists(os.path.join(path, cfg)):
278 self.log.warn("Staging %r from %s into %r [overwrite=%s]"%(
278 self.log.warn("Staging %r from %s into %r [overwrite=%s]"%(
279 cfg, src, self.profile_dir.location, self.overwrite)
279 cfg, src, self.profile_dir.location, self.overwrite)
280 )
280 )
281 self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite)
281 self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite)
282 else:
282 else:
283 self.stage_default_config_file()
283 self.stage_default_config_file()
284 else:
284 else:
285 # Still stage *bundled* config files, but not generated ones
285 # Still stage *bundled* config files, but not generated ones
286 # This is necessary for `ipython profile=sympy` to load the profile
286 # This is necessary for `ipython profile=sympy` to load the profile
287 # on the first go
287 # on the first go
288 files = glob.glob(os.path.join(path, '*.py'))
288 files = glob.glob(os.path.join(path, '*.py'))
289 for fullpath in files:
289 for fullpath in files:
290 cfg = os.path.basename(fullpath)
290 cfg = os.path.basename(fullpath)
291 if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False):
291 if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False):
292 # file was copied
292 # file was copied
293 self.log.warn("Staging bundled %s from %s into %r"%(
293 self.log.warn("Staging bundled %s from %s into %r"%(
294 cfg, self.profile, self.profile_dir.location)
294 cfg, self.profile, self.profile_dir.location)
295 )
295 )
296
296
297
297
298 def stage_default_config_file(self):
298 def stage_default_config_file(self):
299 """auto generate default config file, and stage it into the profile."""
299 """auto generate default config file, and stage it into the profile."""
300 s = self.generate_config_file()
300 s = self.generate_config_file()
301 fname = os.path.join(self.profile_dir.location, self.config_file_name)
301 fname = os.path.join(self.profile_dir.location, self.config_file_name)
302 if self.overwrite or not os.path.exists(fname):
302 if self.overwrite or not os.path.exists(fname):
303 self.log.warn("Generating default config file: %r"%(fname))
303 self.log.warn("Generating default config file: %r"%(fname))
304 with open(fname, 'w') as f:
304 with open(fname, 'w') as f:
305 f.write(s)
305 f.write(s)
306
306
307
307 @catch_config
308 def initialize(self, argv=None):
308 def initialize(self, argv=None):
309 # don't hook up crash handler before parsing command-line
309 # don't hook up crash handler before parsing command-line
310 self.parse_command_line(argv)
310 self.parse_command_line(argv)
311 self.init_crash_handler()
311 self.init_crash_handler()
312 if self.subapp is not None:
312 if self.subapp is not None:
313 # stop here if subapp is taking over
313 # stop here if subapp is taking over
314 return
314 return
315 cl_config = self.config
315 cl_config = self.config
316 self.init_profile_dir()
316 self.init_profile_dir()
317 self.init_config_files()
317 self.init_config_files()
318 self.load_config_file()
318 self.load_config_file()
319 # enforce cl-opts override configfile opts:
319 # enforce cl-opts override configfile opts:
320 self.update_config(cl_config)
320 self.update_config(cl_config)
321
321
@@ -1,315 +1,317 b''
1 """A tornado based IPython notebook server.
1 """A tornado based IPython notebook server.
2
2
3 Authors:
3 Authors:
4
4
5 * Brian Granger
5 * Brian Granger
6 """
6 """
7
7
8 #-----------------------------------------------------------------------------
8 #-----------------------------------------------------------------------------
9 # Copyright (C) 2008-2011 The IPython Development Team
9 # Copyright (C) 2008-2011 The IPython Development Team
10 #
10 #
11 # Distributed under the terms of the BSD License. The full license is in
11 # Distributed under the terms of the BSD License. The full license is in
12 # the file COPYING, distributed as part of this software.
12 # the file COPYING, distributed as part of this software.
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14
14
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16 # Imports
16 # Imports
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 import errno
19 import errno
20 import logging
20 import logging
21 import os
21 import os
22 import signal
22 import signal
23 import socket
23 import socket
24 import sys
24 import sys
25 import webbrowser
25 import webbrowser
26
26
27 import zmq
27 import zmq
28
28
29 # Install the pyzmq ioloop. This has to be done before anything else from
29 # Install the pyzmq ioloop. This has to be done before anything else from
30 # tornado is imported.
30 # tornado is imported.
31 from zmq.eventloop import ioloop
31 from zmq.eventloop import ioloop
32 import tornado.ioloop
32 import tornado.ioloop
33 tornado.ioloop.IOLoop = ioloop.IOLoop
33 tornado.ioloop.IOLoop = ioloop.IOLoop
34
34
35 from tornado import httpserver
35 from tornado import httpserver
36 from tornado import web
36 from tornado import web
37
37
38 from .kernelmanager import MappingKernelManager
38 from .kernelmanager import MappingKernelManager
39 from .handlers import (LoginHandler,
39 from .handlers import (LoginHandler,
40 ProjectDashboardHandler, NewHandler, NamedNotebookHandler,
40 ProjectDashboardHandler, NewHandler, NamedNotebookHandler,
41 MainKernelHandler, KernelHandler, KernelActionHandler, IOPubHandler,
41 MainKernelHandler, KernelHandler, KernelActionHandler, IOPubHandler,
42 ShellHandler, NotebookRootHandler, NotebookHandler, RSTHandler
42 ShellHandler, NotebookRootHandler, NotebookHandler, RSTHandler
43 )
43 )
44 from .notebookmanager import NotebookManager
44 from .notebookmanager import NotebookManager
45
45
46 from IPython.config.application import catch_config
46 from IPython.core.application import BaseIPythonApplication
47 from IPython.core.application import BaseIPythonApplication
47 from IPython.core.profiledir import ProfileDir
48 from IPython.core.profiledir import ProfileDir
48 from IPython.zmq.session import Session, default_secure
49 from IPython.zmq.session import Session, default_secure
49 from IPython.zmq.zmqshell import ZMQInteractiveShell
50 from IPython.zmq.zmqshell import ZMQInteractiveShell
50 from IPython.zmq.ipkernel import (
51 from IPython.zmq.ipkernel import (
51 flags as ipkernel_flags,
52 flags as ipkernel_flags,
52 aliases as ipkernel_aliases,
53 aliases as ipkernel_aliases,
53 IPKernelApp
54 IPKernelApp
54 )
55 )
55 from IPython.utils.traitlets import Dict, Unicode, Int, List, Enum, Bool
56 from IPython.utils.traitlets import Dict, Unicode, Int, List, Enum, Bool
56
57
57 #-----------------------------------------------------------------------------
58 #-----------------------------------------------------------------------------
58 # Module globals
59 # Module globals
59 #-----------------------------------------------------------------------------
60 #-----------------------------------------------------------------------------
60
61
61 _kernel_id_regex = r"(?P<kernel_id>\w+-\w+-\w+-\w+-\w+)"
62 _kernel_id_regex = r"(?P<kernel_id>\w+-\w+-\w+-\w+-\w+)"
62 _kernel_action_regex = r"(?P<action>restart|interrupt)"
63 _kernel_action_regex = r"(?P<action>restart|interrupt)"
63 _notebook_id_regex = r"(?P<notebook_id>\w+-\w+-\w+-\w+-\w+)"
64 _notebook_id_regex = r"(?P<notebook_id>\w+-\w+-\w+-\w+-\w+)"
64
65
65 LOCALHOST = '127.0.0.1'
66 LOCALHOST = '127.0.0.1'
66
67
67 _examples = """
68 _examples = """
68 ipython notebook # start the notebook
69 ipython notebook # start the notebook
69 ipython notebook --profile=sympy # use the sympy profile
70 ipython notebook --profile=sympy # use the sympy profile
70 ipython notebook --pylab=inline # pylab in inline plotting mode
71 ipython notebook --pylab=inline # pylab in inline plotting mode
71 ipython notebook --certfile=mycert.pem # use SSL/TLS certificate
72 ipython notebook --certfile=mycert.pem # use SSL/TLS certificate
72 ipython notebook --port=5555 --ip=* # Listen on port 5555, all interfaces
73 ipython notebook --port=5555 --ip=* # Listen on port 5555, all interfaces
73 """
74 """
74
75
75 #-----------------------------------------------------------------------------
76 #-----------------------------------------------------------------------------
76 # The Tornado web application
77 # The Tornado web application
77 #-----------------------------------------------------------------------------
78 #-----------------------------------------------------------------------------
78
79
79 class NotebookWebApplication(web.Application):
80 class NotebookWebApplication(web.Application):
80
81
81 def __init__(self, ipython_app, kernel_manager, notebook_manager, log):
82 def __init__(self, ipython_app, kernel_manager, notebook_manager, log):
82 handlers = [
83 handlers = [
83 (r"/", ProjectDashboardHandler),
84 (r"/", ProjectDashboardHandler),
84 (r"/login", LoginHandler),
85 (r"/login", LoginHandler),
85 (r"/new", NewHandler),
86 (r"/new", NewHandler),
86 (r"/%s" % _notebook_id_regex, NamedNotebookHandler),
87 (r"/%s" % _notebook_id_regex, NamedNotebookHandler),
87 (r"/kernels", MainKernelHandler),
88 (r"/kernels", MainKernelHandler),
88 (r"/kernels/%s" % _kernel_id_regex, KernelHandler),
89 (r"/kernels/%s" % _kernel_id_regex, KernelHandler),
89 (r"/kernels/%s/%s" % (_kernel_id_regex, _kernel_action_regex), KernelActionHandler),
90 (r"/kernels/%s/%s" % (_kernel_id_regex, _kernel_action_regex), KernelActionHandler),
90 (r"/kernels/%s/iopub" % _kernel_id_regex, IOPubHandler),
91 (r"/kernels/%s/iopub" % _kernel_id_regex, IOPubHandler),
91 (r"/kernels/%s/shell" % _kernel_id_regex, ShellHandler),
92 (r"/kernels/%s/shell" % _kernel_id_regex, ShellHandler),
92 (r"/notebooks", NotebookRootHandler),
93 (r"/notebooks", NotebookRootHandler),
93 (r"/notebooks/%s" % _notebook_id_regex, NotebookHandler),
94 (r"/notebooks/%s" % _notebook_id_regex, NotebookHandler),
94 (r"/rstservice/render", RSTHandler)
95 (r"/rstservice/render", RSTHandler)
95 ]
96 ]
96 settings = dict(
97 settings = dict(
97 template_path=os.path.join(os.path.dirname(__file__), "templates"),
98 template_path=os.path.join(os.path.dirname(__file__), "templates"),
98 static_path=os.path.join(os.path.dirname(__file__), "static"),
99 static_path=os.path.join(os.path.dirname(__file__), "static"),
99 cookie_secret=os.urandom(1024),
100 cookie_secret=os.urandom(1024),
100 login_url="/login",
101 login_url="/login",
101 )
102 )
102 web.Application.__init__(self, handlers, **settings)
103 web.Application.__init__(self, handlers, **settings)
103
104
104 self.kernel_manager = kernel_manager
105 self.kernel_manager = kernel_manager
105 self.log = log
106 self.log = log
106 self.notebook_manager = notebook_manager
107 self.notebook_manager = notebook_manager
107 self.ipython_app = ipython_app
108 self.ipython_app = ipython_app
108
109
109
110
110 #-----------------------------------------------------------------------------
111 #-----------------------------------------------------------------------------
111 # Aliases and Flags
112 # Aliases and Flags
112 #-----------------------------------------------------------------------------
113 #-----------------------------------------------------------------------------
113
114
114 flags = dict(ipkernel_flags)
115 flags = dict(ipkernel_flags)
115 flags['no-browser']=(
116 flags['no-browser']=(
116 {'NotebookApp' : {'open_browser' : False}},
117 {'NotebookApp' : {'open_browser' : False}},
117 "Don't open the notebook in a browser after startup."
118 "Don't open the notebook in a browser after startup."
118 )
119 )
119
120
120 # the flags that are specific to the frontend
121 # the flags that are specific to the frontend
121 # these must be scrubbed before being passed to the kernel,
122 # these must be scrubbed before being passed to the kernel,
122 # or it will raise an error on unrecognized flags
123 # or it will raise an error on unrecognized flags
123 notebook_flags = ['no-browser']
124 notebook_flags = ['no-browser']
124
125
125 aliases = dict(ipkernel_aliases)
126 aliases = dict(ipkernel_aliases)
126
127
127 aliases.update({
128 aliases.update({
128 'ip': 'NotebookApp.ip',
129 'ip': 'NotebookApp.ip',
129 'port': 'NotebookApp.port',
130 'port': 'NotebookApp.port',
130 'keyfile': 'NotebookApp.keyfile',
131 'keyfile': 'NotebookApp.keyfile',
131 'certfile': 'NotebookApp.certfile',
132 'certfile': 'NotebookApp.certfile',
132 'ws-hostname': 'NotebookApp.ws_hostname',
133 'ws-hostname': 'NotebookApp.ws_hostname',
133 'notebook-dir': 'NotebookManager.notebook_dir',
134 'notebook-dir': 'NotebookManager.notebook_dir',
134 })
135 })
135
136
136 # remove ipkernel flags that are singletons, and don't make sense in
137 # remove ipkernel flags that are singletons, and don't make sense in
137 # multi-kernel evironment:
138 # multi-kernel evironment:
138 aliases.pop('f', None)
139 aliases.pop('f', None)
139
140
140 notebook_aliases = [u'port', u'ip', u'keyfile', u'certfile', u'ws-hostname',
141 notebook_aliases = [u'port', u'ip', u'keyfile', u'certfile', u'ws-hostname',
141 u'notebook-dir']
142 u'notebook-dir']
142
143
143 #-----------------------------------------------------------------------------
144 #-----------------------------------------------------------------------------
144 # NotebookApp
145 # NotebookApp
145 #-----------------------------------------------------------------------------
146 #-----------------------------------------------------------------------------
146
147
147 class NotebookApp(BaseIPythonApplication):
148 class NotebookApp(BaseIPythonApplication):
148
149
149 name = 'ipython-notebook'
150 name = 'ipython-notebook'
150 default_config_file_name='ipython_notebook_config.py'
151 default_config_file_name='ipython_notebook_config.py'
151
152
152 description = """
153 description = """
153 The IPython HTML Notebook.
154 The IPython HTML Notebook.
154
155
155 This launches a Tornado based HTML Notebook Server that serves up an
156 This launches a Tornado based HTML Notebook Server that serves up an
156 HTML5/Javascript Notebook client.
157 HTML5/Javascript Notebook client.
157 """
158 """
158 examples = _examples
159 examples = _examples
159
160
160 classes = [IPKernelApp, ZMQInteractiveShell, ProfileDir, Session,
161 classes = [IPKernelApp, ZMQInteractiveShell, ProfileDir, Session,
161 MappingKernelManager, NotebookManager]
162 MappingKernelManager, NotebookManager]
162 flags = Dict(flags)
163 flags = Dict(flags)
163 aliases = Dict(aliases)
164 aliases = Dict(aliases)
164
165
165 kernel_argv = List(Unicode)
166 kernel_argv = List(Unicode)
166
167
167 log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
168 log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
168 default_value=logging.INFO,
169 default_value=logging.INFO,
169 config=True,
170 config=True,
170 help="Set the log level by value or name.")
171 help="Set the log level by value or name.")
171
172
172 # Network related information.
173 # Network related information.
173
174
174 ip = Unicode(LOCALHOST, config=True,
175 ip = Unicode(LOCALHOST, config=True,
175 help="The IP address the notebook server will listen on."
176 help="The IP address the notebook server will listen on."
176 )
177 )
177
178
178 def _ip_changed(self, name, old, new):
179 def _ip_changed(self, name, old, new):
179 if new == u'*': self.ip = u''
180 if new == u'*': self.ip = u''
180
181
181 port = Int(8888, config=True,
182 port = Int(8888, config=True,
182 help="The port the notebook server will listen on."
183 help="The port the notebook server will listen on."
183 )
184 )
184
185
185 ws_hostname = Unicode(LOCALHOST, config=True,
186 ws_hostname = Unicode(LOCALHOST, config=True,
186 help="""The FQDN or IP for WebSocket connections. The default will work
187 help="""The FQDN or IP for WebSocket connections. The default will work
187 fine when the server is listening on localhost, but this needs to
188 fine when the server is listening on localhost, but this needs to
188 be set if the ip option is used. It will be used as the hostname part
189 be set if the ip option is used. It will be used as the hostname part
189 of the WebSocket url: ws://hostname/path."""
190 of the WebSocket url: ws://hostname/path."""
190 )
191 )
191
192
192 certfile = Unicode(u'', config=True,
193 certfile = Unicode(u'', config=True,
193 help="""The full path to an SSL/TLS certificate file."""
194 help="""The full path to an SSL/TLS certificate file."""
194 )
195 )
195
196
196 keyfile = Unicode(u'', config=True,
197 keyfile = Unicode(u'', config=True,
197 help="""The full path to a private key file for usage with SSL/TLS."""
198 help="""The full path to a private key file for usage with SSL/TLS."""
198 )
199 )
199
200
200 password = Unicode(u'', config=True,
201 password = Unicode(u'', config=True,
201 help="""Password to use for web authentication"""
202 help="""Password to use for web authentication"""
202 )
203 )
203
204
204 open_browser = Bool(True, config=True,
205 open_browser = Bool(True, config=True,
205 help="Whether to open in a browser after starting.")
206 help="Whether to open in a browser after starting.")
206
207
207 def get_ws_url(self):
208 def get_ws_url(self):
208 """Return the WebSocket URL for this server."""
209 """Return the WebSocket URL for this server."""
209 if self.certfile:
210 if self.certfile:
210 prefix = u'wss://'
211 prefix = u'wss://'
211 else:
212 else:
212 prefix = u'ws://'
213 prefix = u'ws://'
213 return prefix + self.ws_hostname + u':' + unicode(self.port)
214 return prefix + self.ws_hostname + u':' + unicode(self.port)
214
215
215 def parse_command_line(self, argv=None):
216 def parse_command_line(self, argv=None):
216 super(NotebookApp, self).parse_command_line(argv)
217 super(NotebookApp, self).parse_command_line(argv)
217 if argv is None:
218 if argv is None:
218 argv = sys.argv[1:]
219 argv = sys.argv[1:]
219
220
220 self.kernel_argv = list(argv) # copy
221 self.kernel_argv = list(argv) # copy
221 # Kernel should inherit default config file from frontend
222 # Kernel should inherit default config file from frontend
222 self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name)
223 self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name)
223 # Scrub frontend-specific flags
224 # Scrub frontend-specific flags
224 for a in argv:
225 for a in argv:
225 if a.startswith('-') and a.lstrip('-') in notebook_flags:
226 if a.startswith('-') and a.lstrip('-') in notebook_flags:
226 self.kernel_argv.remove(a)
227 self.kernel_argv.remove(a)
227 swallow_next = False
228 swallow_next = False
228 for a in argv:
229 for a in argv:
229 if swallow_next:
230 if swallow_next:
230 self.kernel_argv.remove(a)
231 self.kernel_argv.remove(a)
231 swallow_next = False
232 swallow_next = False
232 continue
233 continue
233 if a.startswith('-'):
234 if a.startswith('-'):
234 split = a.lstrip('-').split('=')
235 split = a.lstrip('-').split('=')
235 alias = split[0]
236 alias = split[0]
236 if alias in notebook_aliases:
237 if alias in notebook_aliases:
237 self.kernel_argv.remove(a)
238 self.kernel_argv.remove(a)
238 if len(split) == 1:
239 if len(split) == 1:
239 # alias passed with arg via space
240 # alias passed with arg via space
240 swallow_next = True
241 swallow_next = True
241
242
242 def init_configurables(self):
243 def init_configurables(self):
243 # Don't let Qt or ZMQ swallow KeyboardInterupts.
244 # Don't let Qt or ZMQ swallow KeyboardInterupts.
244 signal.signal(signal.SIGINT, signal.SIG_DFL)
245 signal.signal(signal.SIGINT, signal.SIG_DFL)
245
246
246 # force Session default to be secure
247 # force Session default to be secure
247 default_secure(self.config)
248 default_secure(self.config)
248 # Create a KernelManager and start a kernel.
249 # Create a KernelManager and start a kernel.
249 self.kernel_manager = MappingKernelManager(
250 self.kernel_manager = MappingKernelManager(
250 config=self.config, log=self.log, kernel_argv=self.kernel_argv,
251 config=self.config, log=self.log, kernel_argv=self.kernel_argv,
251 connection_dir = self.profile_dir.security_dir,
252 connection_dir = self.profile_dir.security_dir,
252 )
253 )
253 self.notebook_manager = NotebookManager(config=self.config, log=self.log)
254 self.notebook_manager = NotebookManager(config=self.config, log=self.log)
254 self.notebook_manager.list_notebooks()
255 self.notebook_manager.list_notebooks()
255
256
256 def init_logging(self):
257 def init_logging(self):
257 super(NotebookApp, self).init_logging()
258 super(NotebookApp, self).init_logging()
258 # This prevents double log messages because tornado use a root logger that
259 # This prevents double log messages because tornado use a root logger that
259 # self.log is a child of. The logging module dipatches log messages to a log
260 # self.log is a child of. The logging module dipatches log messages to a log
260 # and all of its ancenstors until propagate is set to False.
261 # and all of its ancenstors until propagate is set to False.
261 self.log.propagate = False
262 self.log.propagate = False
262
263
264 @catch_config
263 def initialize(self, argv=None):
265 def initialize(self, argv=None):
264 super(NotebookApp, self).initialize(argv)
266 super(NotebookApp, self).initialize(argv)
265 self.init_configurables()
267 self.init_configurables()
266 self.web_app = NotebookWebApplication(
268 self.web_app = NotebookWebApplication(
267 self, self.kernel_manager, self.notebook_manager, self.log
269 self, self.kernel_manager, self.notebook_manager, self.log
268 )
270 )
269 if self.certfile:
271 if self.certfile:
270 ssl_options = dict(certfile=self.certfile)
272 ssl_options = dict(certfile=self.certfile)
271 if self.keyfile:
273 if self.keyfile:
272 ssl_options['keyfile'] = self.keyfile
274 ssl_options['keyfile'] = self.keyfile
273 else:
275 else:
274 ssl_options = None
276 ssl_options = None
275 self.web_app.password = self.password
277 self.web_app.password = self.password
276 self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options)
278 self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options)
277 if ssl_options is None and not self.ip:
279 if ssl_options is None and not self.ip:
278 self.log.critical('WARNING: the notebook server is listening on all IP addresses '
280 self.log.critical('WARNING: the notebook server is listening on all IP addresses '
279 'but not using any encryption or authentication. This is highly '
281 'but not using any encryption or authentication. This is highly '
280 'insecure and not recommended.')
282 'insecure and not recommended.')
281
283
282 # Try random ports centered around the default.
284 # Try random ports centered around the default.
283 from random import randint
285 from random import randint
284 n = 50 # Max number of attempts, keep reasonably large.
286 n = 50 # Max number of attempts, keep reasonably large.
285 for port in [self.port] + [self.port + randint(-2*n, 2*n) for i in range(n)]:
287 for port in [self.port] + [self.port + randint(-2*n, 2*n) for i in range(n)]:
286 try:
288 try:
287 self.http_server.listen(port, self.ip)
289 self.http_server.listen(port, self.ip)
288 except socket.error, e:
290 except socket.error, e:
289 if e.errno != errno.EADDRINUSE:
291 if e.errno != errno.EADDRINUSE:
290 raise
292 raise
291 self.log.info('The port %i is already in use, trying another random port.' % port)
293 self.log.info('The port %i is already in use, trying another random port.' % port)
292 else:
294 else:
293 self.port = port
295 self.port = port
294 break
296 break
295
297
296 def start(self):
298 def start(self):
297 ip = self.ip if self.ip else '[all ip addresses on your system]'
299 ip = self.ip if self.ip else '[all ip addresses on your system]'
298 proto = 'https' if self.certfile else 'http'
300 proto = 'https' if self.certfile else 'http'
299 self.log.info("The IPython Notebook is running at: %s://%s:%i" % (proto,
301 self.log.info("The IPython Notebook is running at: %s://%s:%i" % (proto,
300 ip,
302 ip,
301 self.port))
303 self.port))
302 if self.open_browser:
304 if self.open_browser:
303 ip = self.ip or '127.0.0.1'
305 ip = self.ip or '127.0.0.1'
304 webbrowser.open("%s://%s:%i" % (proto, ip, self.port), new=2)
306 webbrowser.open("%s://%s:%i" % (proto, ip, self.port), new=2)
305 ioloop.IOLoop.instance().start()
307 ioloop.IOLoop.instance().start()
306
308
307 #-----------------------------------------------------------------------------
309 #-----------------------------------------------------------------------------
308 # Main entry point
310 # Main entry point
309 #-----------------------------------------------------------------------------
311 #-----------------------------------------------------------------------------
310
312
311 def launch_new_instance():
313 def launch_new_instance():
312 app = NotebookApp()
314 app = NotebookApp()
313 app.initialize()
315 app.initialize()
314 app.start()
316 app.start()
315
317
@@ -1,547 +1,548 b''
1 """ A minimal application using the Qt console-style IPython frontend.
1 """ A minimal application using the Qt console-style IPython frontend.
2
2
3 This is not a complete console app, as subprocess will not be able to receive
3 This is not a complete console app, as subprocess will not be able to receive
4 input, there is no real readline support, among other limitations.
4 input, there is no real readline support, among other limitations.
5
5
6 Authors:
6 Authors:
7
7
8 * Evan Patterson
8 * Evan Patterson
9 * Min RK
9 * Min RK
10 * Erik Tollerud
10 * Erik Tollerud
11 * Fernando Perez
11 * Fernando Perez
12 * Bussonnier Matthias
12 * Bussonnier Matthias
13 * Thomas Kluyver
13 * Thomas Kluyver
14
14
15 """
15 """
16
16
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18 # Imports
18 # Imports
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20
20
21 # stdlib imports
21 # stdlib imports
22 import json
22 import json
23 import os
23 import os
24 import signal
24 import signal
25 import sys
25 import sys
26 import uuid
26 import uuid
27
27
28 # System library imports
28 # System library imports
29 from IPython.external.qt import QtGui
29 from IPython.external.qt import QtGui
30
30
31 # Local imports
31 # Local imports
32 from IPython.config.application import boolean_flag
32 from IPython.config.application import boolean_flag, catch_config
33 from IPython.core.application import BaseIPythonApplication
33 from IPython.core.application import BaseIPythonApplication
34 from IPython.core.profiledir import ProfileDir
34 from IPython.core.profiledir import ProfileDir
35 from IPython.lib.kernel import tunnel_to_kernel, find_connection_file
35 from IPython.lib.kernel import tunnel_to_kernel, find_connection_file
36 from IPython.frontend.qt.console.frontend_widget import FrontendWidget
36 from IPython.frontend.qt.console.frontend_widget import FrontendWidget
37 from IPython.frontend.qt.console.ipython_widget import IPythonWidget
37 from IPython.frontend.qt.console.ipython_widget import IPythonWidget
38 from IPython.frontend.qt.console.rich_ipython_widget import RichIPythonWidget
38 from IPython.frontend.qt.console.rich_ipython_widget import RichIPythonWidget
39 from IPython.frontend.qt.console import styles
39 from IPython.frontend.qt.console import styles
40 from IPython.frontend.qt.console.mainwindow import MainWindow
40 from IPython.frontend.qt.console.mainwindow import MainWindow
41 from IPython.frontend.qt.kernelmanager import QtKernelManager
41 from IPython.frontend.qt.kernelmanager import QtKernelManager
42 from IPython.utils.path import filefind
42 from IPython.utils.path import filefind
43 from IPython.utils.py3compat import str_to_bytes
43 from IPython.utils.py3compat import str_to_bytes
44 from IPython.utils.traitlets import (
44 from IPython.utils.traitlets import (
45 Dict, List, Unicode, Int, CaselessStrEnum, CBool, Any
45 Dict, List, Unicode, Int, CaselessStrEnum, CBool, Any
46 )
46 )
47 from IPython.zmq.ipkernel import (
47 from IPython.zmq.ipkernel import (
48 flags as ipkernel_flags,
48 flags as ipkernel_flags,
49 aliases as ipkernel_aliases,
49 aliases as ipkernel_aliases,
50 IPKernelApp
50 IPKernelApp
51 )
51 )
52 from IPython.zmq.session import Session, default_secure
52 from IPython.zmq.session import Session, default_secure
53 from IPython.zmq.zmqshell import ZMQInteractiveShell
53 from IPython.zmq.zmqshell import ZMQInteractiveShell
54
54
55 #-----------------------------------------------------------------------------
55 #-----------------------------------------------------------------------------
56 # Network Constants
56 # Network Constants
57 #-----------------------------------------------------------------------------
57 #-----------------------------------------------------------------------------
58
58
59 from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS
59 from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS
60
60
61 #-----------------------------------------------------------------------------
61 #-----------------------------------------------------------------------------
62 # Globals
62 # Globals
63 #-----------------------------------------------------------------------------
63 #-----------------------------------------------------------------------------
64
64
65 _examples = """
65 _examples = """
66 ipython qtconsole # start the qtconsole
66 ipython qtconsole # start the qtconsole
67 ipython qtconsole --pylab=inline # start with pylab in inline plotting mode
67 ipython qtconsole --pylab=inline # start with pylab in inline plotting mode
68 """
68 """
69
69
70 #-----------------------------------------------------------------------------
70 #-----------------------------------------------------------------------------
71 # Aliases and Flags
71 # Aliases and Flags
72 #-----------------------------------------------------------------------------
72 #-----------------------------------------------------------------------------
73
73
74 flags = dict(ipkernel_flags)
74 flags = dict(ipkernel_flags)
75 qt_flags = {
75 qt_flags = {
76 'existing' : ({'IPythonQtConsoleApp' : {'existing' : 'kernel*.json'}},
76 'existing' : ({'IPythonQtConsoleApp' : {'existing' : 'kernel*.json'}},
77 "Connect to an existing kernel. If no argument specified, guess most recent"),
77 "Connect to an existing kernel. If no argument specified, guess most recent"),
78 'pure' : ({'IPythonQtConsoleApp' : {'pure' : True}},
78 'pure' : ({'IPythonQtConsoleApp' : {'pure' : True}},
79 "Use a pure Python kernel instead of an IPython kernel."),
79 "Use a pure Python kernel instead of an IPython kernel."),
80 'plain' : ({'ConsoleWidget' : {'kind' : 'plain'}},
80 'plain' : ({'ConsoleWidget' : {'kind' : 'plain'}},
81 "Disable rich text support."),
81 "Disable rich text support."),
82 }
82 }
83 qt_flags.update(boolean_flag(
83 qt_flags.update(boolean_flag(
84 'gui-completion', 'ConsoleWidget.gui_completion',
84 'gui-completion', 'ConsoleWidget.gui_completion',
85 "use a GUI widget for tab completion",
85 "use a GUI widget for tab completion",
86 "use plaintext output for completion"
86 "use plaintext output for completion"
87 ))
87 ))
88 qt_flags.update(boolean_flag(
88 qt_flags.update(boolean_flag(
89 'confirm-exit', 'IPythonQtConsoleApp.confirm_exit',
89 'confirm-exit', 'IPythonQtConsoleApp.confirm_exit',
90 """Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
90 """Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
91 to force a direct exit without any confirmation.
91 to force a direct exit without any confirmation.
92 """,
92 """,
93 """Don't prompt the user when exiting. This will terminate the kernel
93 """Don't prompt the user when exiting. This will terminate the kernel
94 if it is owned by the frontend, and leave it alive if it is external.
94 if it is owned by the frontend, and leave it alive if it is external.
95 """
95 """
96 ))
96 ))
97 flags.update(qt_flags)
97 flags.update(qt_flags)
98
98
99 aliases = dict(ipkernel_aliases)
99 aliases = dict(ipkernel_aliases)
100
100
101 qt_aliases = dict(
101 qt_aliases = dict(
102 hb = 'IPythonQtConsoleApp.hb_port',
102 hb = 'IPythonQtConsoleApp.hb_port',
103 shell = 'IPythonQtConsoleApp.shell_port',
103 shell = 'IPythonQtConsoleApp.shell_port',
104 iopub = 'IPythonQtConsoleApp.iopub_port',
104 iopub = 'IPythonQtConsoleApp.iopub_port',
105 stdin = 'IPythonQtConsoleApp.stdin_port',
105 stdin = 'IPythonQtConsoleApp.stdin_port',
106 ip = 'IPythonQtConsoleApp.ip',
106 ip = 'IPythonQtConsoleApp.ip',
107 existing = 'IPythonQtConsoleApp.existing',
107 existing = 'IPythonQtConsoleApp.existing',
108 f = 'IPythonQtConsoleApp.connection_file',
108 f = 'IPythonQtConsoleApp.connection_file',
109
109
110 style = 'IPythonWidget.syntax_style',
110 style = 'IPythonWidget.syntax_style',
111 stylesheet = 'IPythonQtConsoleApp.stylesheet',
111 stylesheet = 'IPythonQtConsoleApp.stylesheet',
112 colors = 'ZMQInteractiveShell.colors',
112 colors = 'ZMQInteractiveShell.colors',
113
113
114 editor = 'IPythonWidget.editor',
114 editor = 'IPythonWidget.editor',
115 paging = 'ConsoleWidget.paging',
115 paging = 'ConsoleWidget.paging',
116 ssh = 'IPythonQtConsoleApp.sshserver',
116 ssh = 'IPythonQtConsoleApp.sshserver',
117 )
117 )
118 aliases.update(qt_aliases)
118 aliases.update(qt_aliases)
119
119
120 #-----------------------------------------------------------------------------
120 #-----------------------------------------------------------------------------
121 # Classes
121 # Classes
122 #-----------------------------------------------------------------------------
122 #-----------------------------------------------------------------------------
123
123
124 #-----------------------------------------------------------------------------
124 #-----------------------------------------------------------------------------
125 # IPythonQtConsole
125 # IPythonQtConsole
126 #-----------------------------------------------------------------------------
126 #-----------------------------------------------------------------------------
127
127
128
128
129 class IPythonQtConsoleApp(BaseIPythonApplication):
129 class IPythonQtConsoleApp(BaseIPythonApplication):
130 name = 'ipython-qtconsole'
130 name = 'ipython-qtconsole'
131 default_config_file_name='ipython_config.py'
131 default_config_file_name='ipython_config.py'
132
132
133 description = """
133 description = """
134 The IPython QtConsole.
134 The IPython QtConsole.
135
135
136 This launches a Console-style application using Qt. It is not a full
136 This launches a Console-style application using Qt. It is not a full
137 console, in that launched terminal subprocesses will not be able to accept
137 console, in that launched terminal subprocesses will not be able to accept
138 input.
138 input.
139
139
140 The QtConsole supports various extra features beyond the Terminal IPython
140 The QtConsole supports various extra features beyond the Terminal IPython
141 shell, such as inline plotting with matplotlib, via:
141 shell, such as inline plotting with matplotlib, via:
142
142
143 ipython qtconsole --pylab=inline
143 ipython qtconsole --pylab=inline
144
144
145 as well as saving your session as HTML, and printing the output.
145 as well as saving your session as HTML, and printing the output.
146
146
147 """
147 """
148 examples = _examples
148 examples = _examples
149
149
150 classes = [IPKernelApp, IPythonWidget, ZMQInteractiveShell, ProfileDir, Session]
150 classes = [IPKernelApp, IPythonWidget, ZMQInteractiveShell, ProfileDir, Session]
151 flags = Dict(flags)
151 flags = Dict(flags)
152 aliases = Dict(aliases)
152 aliases = Dict(aliases)
153
153
154 kernel_argv = List(Unicode)
154 kernel_argv = List(Unicode)
155
155
156 # create requested profiles by default, if they don't exist:
156 # create requested profiles by default, if they don't exist:
157 auto_create = CBool(True)
157 auto_create = CBool(True)
158 # connection info:
158 # connection info:
159 ip = Unicode(LOCALHOST, config=True,
159 ip = Unicode(LOCALHOST, config=True,
160 help="""Set the kernel\'s IP address [default localhost].
160 help="""Set the kernel\'s IP address [default localhost].
161 If the IP address is something other than localhost, then
161 If the IP address is something other than localhost, then
162 Consoles on other machines will be able to connect
162 Consoles on other machines will be able to connect
163 to the Kernel, so be careful!"""
163 to the Kernel, so be careful!"""
164 )
164 )
165
165
166 sshserver = Unicode('', config=True,
166 sshserver = Unicode('', config=True,
167 help="""The SSH server to use to connect to the kernel.""")
167 help="""The SSH server to use to connect to the kernel.""")
168 sshkey = Unicode('', config=True,
168 sshkey = Unicode('', config=True,
169 help="""Path to the ssh key to use for logging in to the ssh server.""")
169 help="""Path to the ssh key to use for logging in to the ssh server.""")
170
170
171 hb_port = Int(0, config=True,
171 hb_port = Int(0, config=True,
172 help="set the heartbeat port [default: random]")
172 help="set the heartbeat port [default: random]")
173 shell_port = Int(0, config=True,
173 shell_port = Int(0, config=True,
174 help="set the shell (XREP) port [default: random]")
174 help="set the shell (XREP) port [default: random]")
175 iopub_port = Int(0, config=True,
175 iopub_port = Int(0, config=True,
176 help="set the iopub (PUB) port [default: random]")
176 help="set the iopub (PUB) port [default: random]")
177 stdin_port = Int(0, config=True,
177 stdin_port = Int(0, config=True,
178 help="set the stdin (XREQ) port [default: random]")
178 help="set the stdin (XREQ) port [default: random]")
179 connection_file = Unicode('', config=True,
179 connection_file = Unicode('', config=True,
180 help="""JSON file in which to store connection info [default: kernel-<pid>.json]
180 help="""JSON file in which to store connection info [default: kernel-<pid>.json]
181
181
182 This file will contain the IP, ports, and authentication key needed to connect
182 This file will contain the IP, ports, and authentication key needed to connect
183 clients to this kernel. By default, this file will be created in the security-dir
183 clients to this kernel. By default, this file will be created in the security-dir
184 of the current profile, but can be specified by absolute path.
184 of the current profile, but can be specified by absolute path.
185 """)
185 """)
186 def _connection_file_default(self):
186 def _connection_file_default(self):
187 return 'kernel-%i.json' % os.getpid()
187 return 'kernel-%i.json' % os.getpid()
188
188
189 existing = Unicode('', config=True,
189 existing = Unicode('', config=True,
190 help="""Connect to an already running kernel""")
190 help="""Connect to an already running kernel""")
191
191
192 stylesheet = Unicode('', config=True,
192 stylesheet = Unicode('', config=True,
193 help="path to a custom CSS stylesheet")
193 help="path to a custom CSS stylesheet")
194
194
195 pure = CBool(False, config=True,
195 pure = CBool(False, config=True,
196 help="Use a pure Python kernel instead of an IPython kernel.")
196 help="Use a pure Python kernel instead of an IPython kernel.")
197 plain = CBool(False, config=True,
197 plain = CBool(False, config=True,
198 help="Use a plaintext widget instead of rich text (plain can't print/save).")
198 help="Use a plaintext widget instead of rich text (plain can't print/save).")
199
199
200 def _pure_changed(self, name, old, new):
200 def _pure_changed(self, name, old, new):
201 kind = 'plain' if self.plain else 'rich'
201 kind = 'plain' if self.plain else 'rich'
202 self.config.ConsoleWidget.kind = kind
202 self.config.ConsoleWidget.kind = kind
203 if self.pure:
203 if self.pure:
204 self.widget_factory = FrontendWidget
204 self.widget_factory = FrontendWidget
205 elif self.plain:
205 elif self.plain:
206 self.widget_factory = IPythonWidget
206 self.widget_factory = IPythonWidget
207 else:
207 else:
208 self.widget_factory = RichIPythonWidget
208 self.widget_factory = RichIPythonWidget
209
209
210 _plain_changed = _pure_changed
210 _plain_changed = _pure_changed
211
211
212 confirm_exit = CBool(True, config=True,
212 confirm_exit = CBool(True, config=True,
213 help="""
213 help="""
214 Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
214 Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
215 to force a direct exit without any confirmation.""",
215 to force a direct exit without any confirmation.""",
216 )
216 )
217
217
218 # the factory for creating a widget
218 # the factory for creating a widget
219 widget_factory = Any(RichIPythonWidget)
219 widget_factory = Any(RichIPythonWidget)
220
220
221 def parse_command_line(self, argv=None):
221 def parse_command_line(self, argv=None):
222 super(IPythonQtConsoleApp, self).parse_command_line(argv)
222 super(IPythonQtConsoleApp, self).parse_command_line(argv)
223 if argv is None:
223 if argv is None:
224 argv = sys.argv[1:]
224 argv = sys.argv[1:]
225 self.kernel_argv = list(argv) # copy
225 self.kernel_argv = list(argv) # copy
226 # kernel should inherit default config file from frontend
226 # kernel should inherit default config file from frontend
227 self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name)
227 self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name)
228 # Scrub frontend-specific flags
228 # Scrub frontend-specific flags
229 swallow_next = False
229 swallow_next = False
230 was_flag = False
230 was_flag = False
231 # copy again, in case some aliases have the same name as a flag
231 # copy again, in case some aliases have the same name as a flag
232 # argv = list(self.kernel_argv)
232 # argv = list(self.kernel_argv)
233 for a in argv:
233 for a in argv:
234 if swallow_next:
234 if swallow_next:
235 swallow_next = False
235 swallow_next = False
236 # last arg was an alias, remove the next one
236 # last arg was an alias, remove the next one
237 # *unless* the last alias has a no-arg flag version, in which
237 # *unless* the last alias has a no-arg flag version, in which
238 # case, don't swallow the next arg if it's also a flag:
238 # case, don't swallow the next arg if it's also a flag:
239 if not (was_flag and a.startswith('-')):
239 if not (was_flag and a.startswith('-')):
240 self.kernel_argv.remove(a)
240 self.kernel_argv.remove(a)
241 continue
241 continue
242 if a.startswith('-'):
242 if a.startswith('-'):
243 split = a.lstrip('-').split('=')
243 split = a.lstrip('-').split('=')
244 alias = split[0]
244 alias = split[0]
245 if alias in qt_aliases:
245 if alias in qt_aliases:
246 self.kernel_argv.remove(a)
246 self.kernel_argv.remove(a)
247 if len(split) == 1:
247 if len(split) == 1:
248 # alias passed with arg via space
248 # alias passed with arg via space
249 swallow_next = True
249 swallow_next = True
250 # could have been a flag that matches an alias, e.g. `existing`
250 # could have been a flag that matches an alias, e.g. `existing`
251 # in which case, we might not swallow the next arg
251 # in which case, we might not swallow the next arg
252 was_flag = alias in qt_flags
252 was_flag = alias in qt_flags
253 elif alias in qt_flags:
253 elif alias in qt_flags:
254 # strip flag, but don't swallow next, as flags don't take args
254 # strip flag, but don't swallow next, as flags don't take args
255 self.kernel_argv.remove(a)
255 self.kernel_argv.remove(a)
256
256
257 def init_connection_file(self):
257 def init_connection_file(self):
258 """find the connection file, and load the info if found.
258 """find the connection file, and load the info if found.
259
259
260 The current working directory and the current profile's security
260 The current working directory and the current profile's security
261 directory will be searched for the file if it is not given by
261 directory will be searched for the file if it is not given by
262 absolute path.
262 absolute path.
263
263
264 When attempting to connect to an existing kernel and the `--existing`
264 When attempting to connect to an existing kernel and the `--existing`
265 argument does not match an existing file, it will be interpreted as a
265 argument does not match an existing file, it will be interpreted as a
266 fileglob, and the matching file in the current profile's security dir
266 fileglob, and the matching file in the current profile's security dir
267 with the latest access time will be used.
267 with the latest access time will be used.
268 """
268 """
269 if self.existing:
269 if self.existing:
270 try:
270 try:
271 cf = find_connection_file(self.existing)
271 cf = find_connection_file(self.existing)
272 except Exception:
272 except Exception:
273 self.log.critical("Could not find existing kernel connection file %s", self.existing)
273 self.log.critical("Could not find existing kernel connection file %s", self.existing)
274 self.exit(1)
274 self.exit(1)
275 self.log.info("Connecting to existing kernel: %s" % cf)
275 self.log.info("Connecting to existing kernel: %s" % cf)
276 self.connection_file = cf
276 self.connection_file = cf
277 # should load_connection_file only be used for existing?
277 # should load_connection_file only be used for existing?
278 # as it is now, this allows reusing ports if an existing
278 # as it is now, this allows reusing ports if an existing
279 # file is requested
279 # file is requested
280 try:
280 try:
281 self.load_connection_file()
281 self.load_connection_file()
282 except Exception:
282 except Exception:
283 self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True)
283 self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True)
284 self.exit(1)
284 self.exit(1)
285
285
286 def load_connection_file(self):
286 def load_connection_file(self):
287 """load ip/port/hmac config from JSON connection file"""
287 """load ip/port/hmac config from JSON connection file"""
288 # this is identical to KernelApp.load_connection_file
288 # this is identical to KernelApp.load_connection_file
289 # perhaps it can be centralized somewhere?
289 # perhaps it can be centralized somewhere?
290 try:
290 try:
291 fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
291 fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
292 except IOError:
292 except IOError:
293 self.log.debug("Connection File not found: %s", self.connection_file)
293 self.log.debug("Connection File not found: %s", self.connection_file)
294 return
294 return
295 self.log.debug(u"Loading connection file %s", fname)
295 self.log.debug(u"Loading connection file %s", fname)
296 with open(fname) as f:
296 with open(fname) as f:
297 s = f.read()
297 s = f.read()
298 cfg = json.loads(s)
298 cfg = json.loads(s)
299 if self.ip == LOCALHOST and 'ip' in cfg:
299 if self.ip == LOCALHOST and 'ip' in cfg:
300 # not overridden by config or cl_args
300 # not overridden by config or cl_args
301 self.ip = cfg['ip']
301 self.ip = cfg['ip']
302 for channel in ('hb', 'shell', 'iopub', 'stdin'):
302 for channel in ('hb', 'shell', 'iopub', 'stdin'):
303 name = channel + '_port'
303 name = channel + '_port'
304 if getattr(self, name) == 0 and name in cfg:
304 if getattr(self, name) == 0 and name in cfg:
305 # not overridden by config or cl_args
305 # not overridden by config or cl_args
306 setattr(self, name, cfg[name])
306 setattr(self, name, cfg[name])
307 if 'key' in cfg:
307 if 'key' in cfg:
308 self.config.Session.key = str_to_bytes(cfg['key'])
308 self.config.Session.key = str_to_bytes(cfg['key'])
309
309
310 def init_ssh(self):
310 def init_ssh(self):
311 """set up ssh tunnels, if needed."""
311 """set up ssh tunnels, if needed."""
312 if not self.sshserver and not self.sshkey:
312 if not self.sshserver and not self.sshkey:
313 return
313 return
314
314
315 if self.sshkey and not self.sshserver:
315 if self.sshkey and not self.sshserver:
316 # specifying just the key implies that we are connecting directly
316 # specifying just the key implies that we are connecting directly
317 self.sshserver = self.ip
317 self.sshserver = self.ip
318 self.ip = LOCALHOST
318 self.ip = LOCALHOST
319
319
320 # build connection dict for tunnels:
320 # build connection dict for tunnels:
321 info = dict(ip=self.ip,
321 info = dict(ip=self.ip,
322 shell_port=self.shell_port,
322 shell_port=self.shell_port,
323 iopub_port=self.iopub_port,
323 iopub_port=self.iopub_port,
324 stdin_port=self.stdin_port,
324 stdin_port=self.stdin_port,
325 hb_port=self.hb_port
325 hb_port=self.hb_port
326 )
326 )
327
327
328 self.log.info("Forwarding connections to %s via %s"%(self.ip, self.sshserver))
328 self.log.info("Forwarding connections to %s via %s"%(self.ip, self.sshserver))
329
329
330 # tunnels return a new set of ports, which will be on localhost:
330 # tunnels return a new set of ports, which will be on localhost:
331 self.ip = LOCALHOST
331 self.ip = LOCALHOST
332 try:
332 try:
333 newports = tunnel_to_kernel(info, self.sshserver, self.sshkey)
333 newports = tunnel_to_kernel(info, self.sshserver, self.sshkey)
334 except:
334 except:
335 # even catch KeyboardInterrupt
335 # even catch KeyboardInterrupt
336 self.log.error("Could not setup tunnels", exc_info=True)
336 self.log.error("Could not setup tunnels", exc_info=True)
337 self.exit(1)
337 self.exit(1)
338
338
339 self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports
339 self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports
340
340
341 cf = self.connection_file
341 cf = self.connection_file
342 base,ext = os.path.splitext(cf)
342 base,ext = os.path.splitext(cf)
343 base = os.path.basename(base)
343 base = os.path.basename(base)
344 self.connection_file = os.path.basename(base)+'-ssh'+ext
344 self.connection_file = os.path.basename(base)+'-ssh'+ext
345 self.log.critical("To connect another client via this tunnel, use:")
345 self.log.critical("To connect another client via this tunnel, use:")
346 self.log.critical("--existing %s" % self.connection_file)
346 self.log.critical("--existing %s" % self.connection_file)
347
347
348 def _new_connection_file(self):
348 def _new_connection_file(self):
349 return os.path.join(self.profile_dir.security_dir, 'kernel-%s.json' % uuid.uuid4())
349 return os.path.join(self.profile_dir.security_dir, 'kernel-%s.json' % uuid.uuid4())
350
350
351 def init_kernel_manager(self):
351 def init_kernel_manager(self):
352 # Don't let Qt or ZMQ swallow KeyboardInterupts.
352 # Don't let Qt or ZMQ swallow KeyboardInterupts.
353 signal.signal(signal.SIGINT, signal.SIG_DFL)
353 signal.signal(signal.SIGINT, signal.SIG_DFL)
354 sec = self.profile_dir.security_dir
354 sec = self.profile_dir.security_dir
355 try:
355 try:
356 cf = filefind(self.connection_file, ['.', sec])
356 cf = filefind(self.connection_file, ['.', sec])
357 except IOError:
357 except IOError:
358 # file might not exist
358 # file might not exist
359 if self.connection_file == os.path.basename(self.connection_file):
359 if self.connection_file == os.path.basename(self.connection_file):
360 # just shortname, put it in security dir
360 # just shortname, put it in security dir
361 cf = os.path.join(sec, self.connection_file)
361 cf = os.path.join(sec, self.connection_file)
362 else:
362 else:
363 cf = self.connection_file
363 cf = self.connection_file
364
364
365 # Create a KernelManager and start a kernel.
365 # Create a KernelManager and start a kernel.
366 self.kernel_manager = QtKernelManager(
366 self.kernel_manager = QtKernelManager(
367 ip=self.ip,
367 ip=self.ip,
368 shell_port=self.shell_port,
368 shell_port=self.shell_port,
369 iopub_port=self.iopub_port,
369 iopub_port=self.iopub_port,
370 stdin_port=self.stdin_port,
370 stdin_port=self.stdin_port,
371 hb_port=self.hb_port,
371 hb_port=self.hb_port,
372 connection_file=cf,
372 connection_file=cf,
373 config=self.config,
373 config=self.config,
374 )
374 )
375 # start the kernel
375 # start the kernel
376 if not self.existing:
376 if not self.existing:
377 kwargs = dict(ipython=not self.pure)
377 kwargs = dict(ipython=not self.pure)
378 kwargs['extra_arguments'] = self.kernel_argv
378 kwargs['extra_arguments'] = self.kernel_argv
379 self.kernel_manager.start_kernel(**kwargs)
379 self.kernel_manager.start_kernel(**kwargs)
380 elif self.sshserver:
380 elif self.sshserver:
381 # ssh, write new connection file
381 # ssh, write new connection file
382 self.kernel_manager.write_connection_file()
382 self.kernel_manager.write_connection_file()
383 self.kernel_manager.start_channels()
383 self.kernel_manager.start_channels()
384
384
385 def new_frontend_master(self):
385 def new_frontend_master(self):
386 """ Create and return new frontend attached to new kernel, launched on localhost.
386 """ Create and return new frontend attached to new kernel, launched on localhost.
387 """
387 """
388 ip = self.ip if self.ip in LOCAL_IPS else LOCALHOST
388 ip = self.ip if self.ip in LOCAL_IPS else LOCALHOST
389 kernel_manager = QtKernelManager(
389 kernel_manager = QtKernelManager(
390 ip=ip,
390 ip=ip,
391 connection_file=self._new_connection_file(),
391 connection_file=self._new_connection_file(),
392 config=self.config,
392 config=self.config,
393 )
393 )
394 # start the kernel
394 # start the kernel
395 kwargs = dict(ipython=not self.pure)
395 kwargs = dict(ipython=not self.pure)
396 kwargs['extra_arguments'] = self.kernel_argv
396 kwargs['extra_arguments'] = self.kernel_argv
397 kernel_manager.start_kernel(**kwargs)
397 kernel_manager.start_kernel(**kwargs)
398 kernel_manager.start_channels()
398 kernel_manager.start_channels()
399 widget = self.widget_factory(config=self.config,
399 widget = self.widget_factory(config=self.config,
400 local_kernel=True)
400 local_kernel=True)
401 widget.kernel_manager = kernel_manager
401 widget.kernel_manager = kernel_manager
402 widget._existing = False
402 widget._existing = False
403 widget._may_close = True
403 widget._may_close = True
404 widget._confirm_exit = self.confirm_exit
404 widget._confirm_exit = self.confirm_exit
405 return widget
405 return widget
406
406
407 def new_frontend_slave(self, current_widget):
407 def new_frontend_slave(self, current_widget):
408 """Create and return a new frontend attached to an existing kernel.
408 """Create and return a new frontend attached to an existing kernel.
409
409
410 Parameters
410 Parameters
411 ----------
411 ----------
412 current_widget : IPythonWidget
412 current_widget : IPythonWidget
413 The IPythonWidget whose kernel this frontend is to share
413 The IPythonWidget whose kernel this frontend is to share
414 """
414 """
415 kernel_manager = QtKernelManager(
415 kernel_manager = QtKernelManager(
416 connection_file=current_widget.kernel_manager.connection_file,
416 connection_file=current_widget.kernel_manager.connection_file,
417 config = self.config,
417 config = self.config,
418 )
418 )
419 kernel_manager.load_connection_file()
419 kernel_manager.load_connection_file()
420 kernel_manager.start_channels()
420 kernel_manager.start_channels()
421 widget = self.widget_factory(config=self.config,
421 widget = self.widget_factory(config=self.config,
422 local_kernel=False)
422 local_kernel=False)
423 widget._existing = True
423 widget._existing = True
424 widget._may_close = False
424 widget._may_close = False
425 widget._confirm_exit = False
425 widget._confirm_exit = False
426 widget.kernel_manager = kernel_manager
426 widget.kernel_manager = kernel_manager
427 return widget
427 return widget
428
428
429 def init_qt_elements(self):
429 def init_qt_elements(self):
430 # Create the widget.
430 # Create the widget.
431 self.app = QtGui.QApplication([])
431 self.app = QtGui.QApplication([])
432
432
433 base_path = os.path.abspath(os.path.dirname(__file__))
433 base_path = os.path.abspath(os.path.dirname(__file__))
434 icon_path = os.path.join(base_path, 'resources', 'icon', 'IPythonConsole.svg')
434 icon_path = os.path.join(base_path, 'resources', 'icon', 'IPythonConsole.svg')
435 self.app.icon = QtGui.QIcon(icon_path)
435 self.app.icon = QtGui.QIcon(icon_path)
436 QtGui.QApplication.setWindowIcon(self.app.icon)
436 QtGui.QApplication.setWindowIcon(self.app.icon)
437
437
438 local_kernel = (not self.existing) or self.ip in LOCAL_IPS
438 local_kernel = (not self.existing) or self.ip in LOCAL_IPS
439 self.widget = self.widget_factory(config=self.config,
439 self.widget = self.widget_factory(config=self.config,
440 local_kernel=local_kernel)
440 local_kernel=local_kernel)
441 self.widget._existing = self.existing
441 self.widget._existing = self.existing
442 self.widget._may_close = not self.existing
442 self.widget._may_close = not self.existing
443 self.widget._confirm_exit = self.confirm_exit
443 self.widget._confirm_exit = self.confirm_exit
444
444
445 self.widget.kernel_manager = self.kernel_manager
445 self.widget.kernel_manager = self.kernel_manager
446 self.window = MainWindow(self.app,
446 self.window = MainWindow(self.app,
447 confirm_exit=self.confirm_exit,
447 confirm_exit=self.confirm_exit,
448 new_frontend_factory=self.new_frontend_master,
448 new_frontend_factory=self.new_frontend_master,
449 slave_frontend_factory=self.new_frontend_slave,
449 slave_frontend_factory=self.new_frontend_slave,
450 )
450 )
451 self.window.log = self.log
451 self.window.log = self.log
452 self.window.add_tab_with_frontend(self.widget)
452 self.window.add_tab_with_frontend(self.widget)
453 self.window.init_menu_bar()
453 self.window.init_menu_bar()
454 self.window.setWindowTitle('Python' if self.pure else 'IPython')
454 self.window.setWindowTitle('Python' if self.pure else 'IPython')
455
455
456 def init_colors(self):
456 def init_colors(self):
457 """Configure the coloring of the widget"""
457 """Configure the coloring of the widget"""
458 # Note: This will be dramatically simplified when colors
458 # Note: This will be dramatically simplified when colors
459 # are removed from the backend.
459 # are removed from the backend.
460
460
461 if self.pure:
461 if self.pure:
462 # only IPythonWidget supports styling
462 # only IPythonWidget supports styling
463 return
463 return
464
464
465 # parse the colors arg down to current known labels
465 # parse the colors arg down to current known labels
466 try:
466 try:
467 colors = self.config.ZMQInteractiveShell.colors
467 colors = self.config.ZMQInteractiveShell.colors
468 except AttributeError:
468 except AttributeError:
469 colors = None
469 colors = None
470 try:
470 try:
471 style = self.config.IPythonWidget.syntax_style
471 style = self.config.IPythonWidget.syntax_style
472 except AttributeError:
472 except AttributeError:
473 style = None
473 style = None
474
474
475 # find the value for colors:
475 # find the value for colors:
476 if colors:
476 if colors:
477 colors=colors.lower()
477 colors=colors.lower()
478 if colors in ('lightbg', 'light'):
478 if colors in ('lightbg', 'light'):
479 colors='lightbg'
479 colors='lightbg'
480 elif colors in ('dark', 'linux'):
480 elif colors in ('dark', 'linux'):
481 colors='linux'
481 colors='linux'
482 else:
482 else:
483 colors='nocolor'
483 colors='nocolor'
484 elif style:
484 elif style:
485 if style=='bw':
485 if style=='bw':
486 colors='nocolor'
486 colors='nocolor'
487 elif styles.dark_style(style):
487 elif styles.dark_style(style):
488 colors='linux'
488 colors='linux'
489 else:
489 else:
490 colors='lightbg'
490 colors='lightbg'
491 else:
491 else:
492 colors=None
492 colors=None
493
493
494 # Configure the style.
494 # Configure the style.
495 widget = self.widget
495 widget = self.widget
496 if style:
496 if style:
497 widget.style_sheet = styles.sheet_from_template(style, colors)
497 widget.style_sheet = styles.sheet_from_template(style, colors)
498 widget.syntax_style = style
498 widget.syntax_style = style
499 widget._syntax_style_changed()
499 widget._syntax_style_changed()
500 widget._style_sheet_changed()
500 widget._style_sheet_changed()
501 elif colors:
501 elif colors:
502 # use a default style
502 # use a default style
503 widget.set_default_style(colors=colors)
503 widget.set_default_style(colors=colors)
504 else:
504 else:
505 # this is redundant for now, but allows the widget's
505 # this is redundant for now, but allows the widget's
506 # defaults to change
506 # defaults to change
507 widget.set_default_style()
507 widget.set_default_style()
508
508
509 if self.stylesheet:
509 if self.stylesheet:
510 # we got an expicit stylesheet
510 # we got an expicit stylesheet
511 if os.path.isfile(self.stylesheet):
511 if os.path.isfile(self.stylesheet):
512 with open(self.stylesheet) as f:
512 with open(self.stylesheet) as f:
513 sheet = f.read()
513 sheet = f.read()
514 widget.style_sheet = sheet
514 widget.style_sheet = sheet
515 widget._style_sheet_changed()
515 widget._style_sheet_changed()
516 else:
516 else:
517 raise IOError("Stylesheet %r not found."%self.stylesheet)
517 raise IOError("Stylesheet %r not found."%self.stylesheet)
518
518
519 @catch_config
519 def initialize(self, argv=None):
520 def initialize(self, argv=None):
520 super(IPythonQtConsoleApp, self).initialize(argv)
521 super(IPythonQtConsoleApp, self).initialize(argv)
521 self.init_connection_file()
522 self.init_connection_file()
522 default_secure(self.config)
523 default_secure(self.config)
523 self.init_ssh()
524 self.init_ssh()
524 self.init_kernel_manager()
525 self.init_kernel_manager()
525 self.init_qt_elements()
526 self.init_qt_elements()
526 self.init_colors()
527 self.init_colors()
527
528
528 def start(self):
529 def start(self):
529
530
530 # draw the window
531 # draw the window
531 self.window.show()
532 self.window.show()
532
533
533 # Start the application main loop.
534 # Start the application main loop.
534 self.app.exec_()
535 self.app.exec_()
535
536
536 #-----------------------------------------------------------------------------
537 #-----------------------------------------------------------------------------
537 # Main entry point
538 # Main entry point
538 #-----------------------------------------------------------------------------
539 #-----------------------------------------------------------------------------
539
540
540 def main():
541 def main():
541 app = IPythonQtConsoleApp()
542 app = IPythonQtConsoleApp()
542 app.initialize()
543 app.initialize()
543 app.start()
544 app.start()
544
545
545
546
546 if __name__ == '__main__':
547 if __name__ == '__main__':
547 main()
548 main()
@@ -1,395 +1,396 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The :class:`~IPython.core.application.Application` object for the command
4 The :class:`~IPython.core.application.Application` object for the command
5 line :command:`ipython` program.
5 line :command:`ipython` program.
6
6
7 Authors
7 Authors
8 -------
8 -------
9
9
10 * Brian Granger
10 * Brian Granger
11 * Fernando Perez
11 * Fernando Perez
12 * Min Ragan-Kelley
12 * Min Ragan-Kelley
13 """
13 """
14
14
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16 # Copyright (C) 2008-2010 The IPython Development Team
16 # Copyright (C) 2008-2010 The IPython Development Team
17 #
17 #
18 # Distributed under the terms of the BSD License. The full license is in
18 # Distributed under the terms of the BSD License. The full license is in
19 # the file COPYING, distributed as part of this software.
19 # the file COPYING, distributed as part of this software.
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21
21
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23 # Imports
23 # Imports
24 #-----------------------------------------------------------------------------
24 #-----------------------------------------------------------------------------
25
25
26 from __future__ import absolute_import
26 from __future__ import absolute_import
27
27
28 import logging
28 import logging
29 import os
29 import os
30 import sys
30 import sys
31
31
32 from IPython.config.loader import (
32 from IPython.config.loader import (
33 Config, PyFileConfigLoader, ConfigFileNotFound
33 Config, PyFileConfigLoader, ConfigFileNotFound
34 )
34 )
35 from IPython.config.application import boolean_flag
35 from IPython.config.application import boolean_flag, catch_config
36 from IPython.core import release
36 from IPython.core import release
37 from IPython.core import usage
37 from IPython.core import usage
38 from IPython.core.completer import Completer
38 from IPython.core.completer import Completer
39 from IPython.core.crashhandler import CrashHandler
39 from IPython.core.crashhandler import CrashHandler
40 from IPython.core.formatters import PlainTextFormatter
40 from IPython.core.formatters import PlainTextFormatter
41 from IPython.core.application import (
41 from IPython.core.application import (
42 ProfileDir, BaseIPythonApplication, base_flags, base_aliases
42 ProfileDir, BaseIPythonApplication, base_flags, base_aliases
43 )
43 )
44 from IPython.core.shellapp import (
44 from IPython.core.shellapp import (
45 InteractiveShellApp, shell_flags, shell_aliases
45 InteractiveShellApp, shell_flags, shell_aliases
46 )
46 )
47 from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell
47 from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell
48 from IPython.lib import inputhook
48 from IPython.lib import inputhook
49 from IPython.utils import warn
49 from IPython.utils import warn
50 from IPython.utils.path import get_ipython_dir, check_for_old_config
50 from IPython.utils.path import get_ipython_dir, check_for_old_config
51 from IPython.utils.traitlets import (
51 from IPython.utils.traitlets import (
52 Bool, List, Dict, CaselessStrEnum
52 Bool, List, Dict, CaselessStrEnum
53 )
53 )
54
54
55 #-----------------------------------------------------------------------------
55 #-----------------------------------------------------------------------------
56 # Globals, utilities and helpers
56 # Globals, utilities and helpers
57 #-----------------------------------------------------------------------------
57 #-----------------------------------------------------------------------------
58
58
59 #: The default config file name for this application.
59 #: The default config file name for this application.
60 default_config_file_name = u'ipython_config.py'
60 default_config_file_name = u'ipython_config.py'
61
61
62 _examples = """
62 _examples = """
63 ipython --pylab # start in pylab mode
63 ipython --pylab # start in pylab mode
64 ipython --pylab=qt # start in pylab mode with the qt4 backend
64 ipython --pylab=qt # start in pylab mode with the qt4 backend
65 ipython --log-level=DEBUG # set logging to DEBUG
65 ipython --log-level=DEBUG # set logging to DEBUG
66 ipython --profile=foo # start with profile foo
66 ipython --profile=foo # start with profile foo
67
67
68 ipython qtconsole # start the qtconsole GUI application
68 ipython qtconsole # start the qtconsole GUI application
69 ipython qtconsole -h # show the help string for the qtconsole subcmd
69 ipython qtconsole -h # show the help string for the qtconsole subcmd
70
70
71 ipython profile create foo # create profile foo w/ default config files
71 ipython profile create foo # create profile foo w/ default config files
72 ipython profile -h # show the help string for the profile subcmd
72 ipython profile -h # show the help string for the profile subcmd
73 """
73 """
74
74
75 #-----------------------------------------------------------------------------
75 #-----------------------------------------------------------------------------
76 # Crash handler for this application
76 # Crash handler for this application
77 #-----------------------------------------------------------------------------
77 #-----------------------------------------------------------------------------
78
78
79 class IPAppCrashHandler(CrashHandler):
79 class IPAppCrashHandler(CrashHandler):
80 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
80 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
81
81
82 def __init__(self, app):
82 def __init__(self, app):
83 contact_name = release.authors['Fernando'][0]
83 contact_name = release.authors['Fernando'][0]
84 contact_email = release.authors['Fernando'][1]
84 contact_email = release.authors['Fernando'][1]
85 bug_tracker = 'http://github.com/ipython/ipython/issues'
85 bug_tracker = 'http://github.com/ipython/ipython/issues'
86 super(IPAppCrashHandler,self).__init__(
86 super(IPAppCrashHandler,self).__init__(
87 app, contact_name, contact_email, bug_tracker
87 app, contact_name, contact_email, bug_tracker
88 )
88 )
89
89
90 def make_report(self,traceback):
90 def make_report(self,traceback):
91 """Return a string containing a crash report."""
91 """Return a string containing a crash report."""
92
92
93 sec_sep = self.section_sep
93 sec_sep = self.section_sep
94 # Start with parent report
94 # Start with parent report
95 report = [super(IPAppCrashHandler, self).make_report(traceback)]
95 report = [super(IPAppCrashHandler, self).make_report(traceback)]
96 # Add interactive-specific info we may have
96 # Add interactive-specific info we may have
97 rpt_add = report.append
97 rpt_add = report.append
98 try:
98 try:
99 rpt_add(sec_sep+"History of session input:")
99 rpt_add(sec_sep+"History of session input:")
100 for line in self.app.shell.user_ns['_ih']:
100 for line in self.app.shell.user_ns['_ih']:
101 rpt_add(line)
101 rpt_add(line)
102 rpt_add('\n*** Last line of input (may not be in above history):\n')
102 rpt_add('\n*** Last line of input (may not be in above history):\n')
103 rpt_add(self.app.shell._last_input_line+'\n')
103 rpt_add(self.app.shell._last_input_line+'\n')
104 except:
104 except:
105 pass
105 pass
106
106
107 return ''.join(report)
107 return ''.join(report)
108
108
109 #-----------------------------------------------------------------------------
109 #-----------------------------------------------------------------------------
110 # Aliases and Flags
110 # Aliases and Flags
111 #-----------------------------------------------------------------------------
111 #-----------------------------------------------------------------------------
112 flags = dict(base_flags)
112 flags = dict(base_flags)
113 flags.update(shell_flags)
113 flags.update(shell_flags)
114 addflag = lambda *args: flags.update(boolean_flag(*args))
114 addflag = lambda *args: flags.update(boolean_flag(*args))
115 addflag('autoedit-syntax', 'TerminalInteractiveShell.autoedit_syntax',
115 addflag('autoedit-syntax', 'TerminalInteractiveShell.autoedit_syntax',
116 'Turn on auto editing of files with syntax errors.',
116 'Turn on auto editing of files with syntax errors.',
117 'Turn off auto editing of files with syntax errors.'
117 'Turn off auto editing of files with syntax errors.'
118 )
118 )
119 addflag('banner', 'TerminalIPythonApp.display_banner',
119 addflag('banner', 'TerminalIPythonApp.display_banner',
120 "Display a banner upon starting IPython.",
120 "Display a banner upon starting IPython.",
121 "Don't display a banner upon starting IPython."
121 "Don't display a banner upon starting IPython."
122 )
122 )
123 addflag('confirm-exit', 'TerminalInteractiveShell.confirm_exit',
123 addflag('confirm-exit', 'TerminalInteractiveShell.confirm_exit',
124 """Set to confirm when you try to exit IPython with an EOF (Control-D
124 """Set to confirm when you try to exit IPython with an EOF (Control-D
125 in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
125 in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
126 you can force a direct exit without any confirmation.""",
126 you can force a direct exit without any confirmation.""",
127 "Don't prompt the user when exiting."
127 "Don't prompt the user when exiting."
128 )
128 )
129 addflag('term-title', 'TerminalInteractiveShell.term_title',
129 addflag('term-title', 'TerminalInteractiveShell.term_title',
130 "Enable auto setting the terminal title.",
130 "Enable auto setting the terminal title.",
131 "Disable auto setting the terminal title."
131 "Disable auto setting the terminal title."
132 )
132 )
133 classic_config = Config()
133 classic_config = Config()
134 classic_config.InteractiveShell.cache_size = 0
134 classic_config.InteractiveShell.cache_size = 0
135 classic_config.PlainTextFormatter.pprint = False
135 classic_config.PlainTextFormatter.pprint = False
136 classic_config.InteractiveShell.prompt_in1 = '>>> '
136 classic_config.InteractiveShell.prompt_in1 = '>>> '
137 classic_config.InteractiveShell.prompt_in2 = '... '
137 classic_config.InteractiveShell.prompt_in2 = '... '
138 classic_config.InteractiveShell.prompt_out = ''
138 classic_config.InteractiveShell.prompt_out = ''
139 classic_config.InteractiveShell.separate_in = ''
139 classic_config.InteractiveShell.separate_in = ''
140 classic_config.InteractiveShell.separate_out = ''
140 classic_config.InteractiveShell.separate_out = ''
141 classic_config.InteractiveShell.separate_out2 = ''
141 classic_config.InteractiveShell.separate_out2 = ''
142 classic_config.InteractiveShell.colors = 'NoColor'
142 classic_config.InteractiveShell.colors = 'NoColor'
143 classic_config.InteractiveShell.xmode = 'Plain'
143 classic_config.InteractiveShell.xmode = 'Plain'
144
144
145 flags['classic']=(
145 flags['classic']=(
146 classic_config,
146 classic_config,
147 "Gives IPython a similar feel to the classic Python prompt."
147 "Gives IPython a similar feel to the classic Python prompt."
148 )
148 )
149 # # log doesn't make so much sense this way anymore
149 # # log doesn't make so much sense this way anymore
150 # paa('--log','-l',
150 # paa('--log','-l',
151 # action='store_true', dest='InteractiveShell.logstart',
151 # action='store_true', dest='InteractiveShell.logstart',
152 # help="Start logging to the default log file (./ipython_log.py).")
152 # help="Start logging to the default log file (./ipython_log.py).")
153 #
153 #
154 # # quick is harder to implement
154 # # quick is harder to implement
155 flags['quick']=(
155 flags['quick']=(
156 {'TerminalIPythonApp' : {'quick' : True}},
156 {'TerminalIPythonApp' : {'quick' : True}},
157 "Enable quick startup with no config files."
157 "Enable quick startup with no config files."
158 )
158 )
159
159
160 flags['i'] = (
160 flags['i'] = (
161 {'TerminalIPythonApp' : {'force_interact' : True}},
161 {'TerminalIPythonApp' : {'force_interact' : True}},
162 """If running code from the command line, become interactive afterwards.
162 """If running code from the command line, become interactive afterwards.
163 Note: can also be given simply as '-i.'"""
163 Note: can also be given simply as '-i.'"""
164 )
164 )
165 flags['pylab'] = (
165 flags['pylab'] = (
166 {'TerminalIPythonApp' : {'pylab' : 'auto'}},
166 {'TerminalIPythonApp' : {'pylab' : 'auto'}},
167 """Pre-load matplotlib and numpy for interactive use with
167 """Pre-load matplotlib and numpy for interactive use with
168 the default matplotlib backend."""
168 the default matplotlib backend."""
169 )
169 )
170
170
171 aliases = dict(base_aliases)
171 aliases = dict(base_aliases)
172 aliases.update(shell_aliases)
172 aliases.update(shell_aliases)
173
173
174 # it's possible we don't want short aliases for *all* of these:
174 # it's possible we don't want short aliases for *all* of these:
175 aliases.update(dict(
175 aliases.update(dict(
176 gui='TerminalIPythonApp.gui',
176 gui='TerminalIPythonApp.gui',
177 pylab='TerminalIPythonApp.pylab',
177 pylab='TerminalIPythonApp.pylab',
178 ))
178 ))
179
179
180 #-----------------------------------------------------------------------------
180 #-----------------------------------------------------------------------------
181 # Main classes and functions
181 # Main classes and functions
182 #-----------------------------------------------------------------------------
182 #-----------------------------------------------------------------------------
183
183
184 class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):
184 class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):
185 name = u'ipython'
185 name = u'ipython'
186 description = usage.cl_usage
186 description = usage.cl_usage
187 default_config_file_name = default_config_file_name
187 default_config_file_name = default_config_file_name
188 crash_handler_class = IPAppCrashHandler
188 crash_handler_class = IPAppCrashHandler
189 examples = _examples
189 examples = _examples
190
190
191 flags = Dict(flags)
191 flags = Dict(flags)
192 aliases = Dict(aliases)
192 aliases = Dict(aliases)
193 classes = List()
193 classes = List()
194 def _classes_default(self):
194 def _classes_default(self):
195 """This has to be in a method, for TerminalIPythonApp to be available."""
195 """This has to be in a method, for TerminalIPythonApp to be available."""
196 return [
196 return [
197 InteractiveShellApp, # ShellApp comes before TerminalApp, because
197 InteractiveShellApp, # ShellApp comes before TerminalApp, because
198 self.__class__, # it will also affect subclasses (e.g. QtConsole)
198 self.__class__, # it will also affect subclasses (e.g. QtConsole)
199 TerminalInteractiveShell,
199 TerminalInteractiveShell,
200 ProfileDir,
200 ProfileDir,
201 PlainTextFormatter,
201 PlainTextFormatter,
202 Completer,
202 Completer,
203 ]
203 ]
204
204
205 subcommands = Dict(dict(
205 subcommands = Dict(dict(
206 qtconsole=('IPython.frontend.qt.console.qtconsoleapp.IPythonQtConsoleApp',
206 qtconsole=('IPython.frontend.qt.console.qtconsoleapp.IPythonQtConsoleApp',
207 """Launch the IPython Qt Console."""
207 """Launch the IPython Qt Console."""
208 ),
208 ),
209 notebook=('IPython.frontend.html.notebook.notebookapp.NotebookApp',
209 notebook=('IPython.frontend.html.notebook.notebookapp.NotebookApp',
210 """Launch the IPython HTML Notebook Server"""
210 """Launch the IPython HTML Notebook Server"""
211 ),
211 ),
212 profile = ("IPython.core.profileapp.ProfileApp",
212 profile = ("IPython.core.profileapp.ProfileApp",
213 "Create and manage IPython profiles."
213 "Create and manage IPython profiles."
214 ),
214 ),
215 kernel = ("IPython.zmq.ipkernel.IPKernelApp",
215 kernel = ("IPython.zmq.ipkernel.IPKernelApp",
216 "Start a kernel without an attached frontend."
216 "Start a kernel without an attached frontend."
217 ),
217 ),
218 ))
218 ))
219
219
220 # *do* autocreate requested profile, but don't create the config file.
220 # *do* autocreate requested profile, but don't create the config file.
221 auto_create=Bool(True)
221 auto_create=Bool(True)
222 # configurables
222 # configurables
223 ignore_old_config=Bool(False, config=True,
223 ignore_old_config=Bool(False, config=True,
224 help="Suppress warning messages about legacy config files"
224 help="Suppress warning messages about legacy config files"
225 )
225 )
226 quick = Bool(False, config=True,
226 quick = Bool(False, config=True,
227 help="""Start IPython quickly by skipping the loading of config files."""
227 help="""Start IPython quickly by skipping the loading of config files."""
228 )
228 )
229 def _quick_changed(self, name, old, new):
229 def _quick_changed(self, name, old, new):
230 if new:
230 if new:
231 self.load_config_file = lambda *a, **kw: None
231 self.load_config_file = lambda *a, **kw: None
232 self.ignore_old_config=True
232 self.ignore_old_config=True
233
233
234 gui = CaselessStrEnum(('qt', 'wx', 'gtk', 'glut', 'pyglet'), config=True,
234 gui = CaselessStrEnum(('qt', 'wx', 'gtk', 'glut', 'pyglet'), config=True,
235 help="Enable GUI event loop integration ('qt', 'wx', 'gtk', 'glut', 'pyglet')."
235 help="Enable GUI event loop integration ('qt', 'wx', 'gtk', 'glut', 'pyglet')."
236 )
236 )
237 pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'auto'],
237 pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'auto'],
238 config=True,
238 config=True,
239 help="""Pre-load matplotlib and numpy for interactive use,
239 help="""Pre-load matplotlib and numpy for interactive use,
240 selecting a particular matplotlib backend and loop integration.
240 selecting a particular matplotlib backend and loop integration.
241 """
241 """
242 )
242 )
243 display_banner = Bool(True, config=True,
243 display_banner = Bool(True, config=True,
244 help="Whether to display a banner upon starting IPython."
244 help="Whether to display a banner upon starting IPython."
245 )
245 )
246
246
247 # if there is code of files to run from the cmd line, don't interact
247 # if there is code of files to run from the cmd line, don't interact
248 # unless the --i flag (App.force_interact) is true.
248 # unless the --i flag (App.force_interact) is true.
249 force_interact = Bool(False, config=True,
249 force_interact = Bool(False, config=True,
250 help="""If a command or file is given via the command-line,
250 help="""If a command or file is given via the command-line,
251 e.g. 'ipython foo.py"""
251 e.g. 'ipython foo.py"""
252 )
252 )
253 def _force_interact_changed(self, name, old, new):
253 def _force_interact_changed(self, name, old, new):
254 if new:
254 if new:
255 self.interact = True
255 self.interact = True
256
256
257 def _file_to_run_changed(self, name, old, new):
257 def _file_to_run_changed(self, name, old, new):
258 if new and not self.force_interact:
258 if new and not self.force_interact:
259 self.interact = False
259 self.interact = False
260 _code_to_run_changed = _file_to_run_changed
260 _code_to_run_changed = _file_to_run_changed
261
261
262 # internal, not-configurable
262 # internal, not-configurable
263 interact=Bool(True)
263 interact=Bool(True)
264
264
265
265
266 def parse_command_line(self, argv=None):
266 def parse_command_line(self, argv=None):
267 """override to allow old '-pylab' flag with deprecation warning"""
267 """override to allow old '-pylab' flag with deprecation warning"""
268
268
269 argv = sys.argv[1:] if argv is None else argv
269 argv = sys.argv[1:] if argv is None else argv
270
270
271 if '-pylab' in argv:
271 if '-pylab' in argv:
272 # deprecated `-pylab` given,
272 # deprecated `-pylab` given,
273 # warn and transform into current syntax
273 # warn and transform into current syntax
274 argv = argv[:] # copy, don't clobber
274 argv = argv[:] # copy, don't clobber
275 idx = argv.index('-pylab')
275 idx = argv.index('-pylab')
276 warn.warn("`-pylab` flag has been deprecated.\n"
276 warn.warn("`-pylab` flag has been deprecated.\n"
277 " Use `--pylab` instead, or `--pylab=foo` to specify a backend.")
277 " Use `--pylab` instead, or `--pylab=foo` to specify a backend.")
278 sub = '--pylab'
278 sub = '--pylab'
279 if len(argv) > idx+1:
279 if len(argv) > idx+1:
280 # check for gui arg, as in '-pylab qt'
280 # check for gui arg, as in '-pylab qt'
281 gui = argv[idx+1]
281 gui = argv[idx+1]
282 if gui in ('wx', 'qt', 'qt4', 'gtk', 'auto'):
282 if gui in ('wx', 'qt', 'qt4', 'gtk', 'auto'):
283 sub = '--pylab='+gui
283 sub = '--pylab='+gui
284 argv.pop(idx+1)
284 argv.pop(idx+1)
285 argv[idx] = sub
285 argv[idx] = sub
286
286
287 return super(TerminalIPythonApp, self).parse_command_line(argv)
287 return super(TerminalIPythonApp, self).parse_command_line(argv)
288
288
289 @catch_config
289 def initialize(self, argv=None):
290 def initialize(self, argv=None):
290 """Do actions after construct, but before starting the app."""
291 """Do actions after construct, but before starting the app."""
291 super(TerminalIPythonApp, self).initialize(argv)
292 super(TerminalIPythonApp, self).initialize(argv)
292 if self.subapp is not None:
293 if self.subapp is not None:
293 # don't bother initializing further, starting subapp
294 # don't bother initializing further, starting subapp
294 return
295 return
295 if not self.ignore_old_config:
296 if not self.ignore_old_config:
296 check_for_old_config(self.ipython_dir)
297 check_for_old_config(self.ipython_dir)
297 # print self.extra_args
298 # print self.extra_args
298 if self.extra_args:
299 if self.extra_args:
299 self.file_to_run = self.extra_args[0]
300 self.file_to_run = self.extra_args[0]
300 # create the shell
301 # create the shell
301 self.init_shell()
302 self.init_shell()
302 # and draw the banner
303 # and draw the banner
303 self.init_banner()
304 self.init_banner()
304 # Now a variety of things that happen after the banner is printed.
305 # Now a variety of things that happen after the banner is printed.
305 self.init_gui_pylab()
306 self.init_gui_pylab()
306 self.init_extensions()
307 self.init_extensions()
307 self.init_code()
308 self.init_code()
308
309
309 def init_shell(self):
310 def init_shell(self):
310 """initialize the InteractiveShell instance"""
311 """initialize the InteractiveShell instance"""
311 # I am a little hesitant to put these into InteractiveShell itself.
312 # I am a little hesitant to put these into InteractiveShell itself.
312 # But that might be the place for them
313 # But that might be the place for them
313 sys.path.insert(0, '')
314 sys.path.insert(0, '')
314
315
315 # Create an InteractiveShell instance.
316 # Create an InteractiveShell instance.
316 # shell.display_banner should always be False for the terminal
317 # shell.display_banner should always be False for the terminal
317 # based app, because we call shell.show_banner() by hand below
318 # based app, because we call shell.show_banner() by hand below
318 # so the banner shows *before* all extension loading stuff.
319 # so the banner shows *before* all extension loading stuff.
319 self.shell = TerminalInteractiveShell.instance(config=self.config,
320 self.shell = TerminalInteractiveShell.instance(config=self.config,
320 display_banner=False, profile_dir=self.profile_dir,
321 display_banner=False, profile_dir=self.profile_dir,
321 ipython_dir=self.ipython_dir)
322 ipython_dir=self.ipython_dir)
322
323
323 def init_banner(self):
324 def init_banner(self):
324 """optionally display the banner"""
325 """optionally display the banner"""
325 if self.display_banner and self.interact:
326 if self.display_banner and self.interact:
326 self.shell.show_banner()
327 self.shell.show_banner()
327 # Make sure there is a space below the banner.
328 # Make sure there is a space below the banner.
328 if self.log_level <= logging.INFO: print
329 if self.log_level <= logging.INFO: print
329
330
330
331
331 def init_gui_pylab(self):
332 def init_gui_pylab(self):
332 """Enable GUI event loop integration, taking pylab into account."""
333 """Enable GUI event loop integration, taking pylab into account."""
333 gui = self.gui
334 gui = self.gui
334
335
335 # Using `pylab` will also require gui activation, though which toolkit
336 # Using `pylab` will also require gui activation, though which toolkit
336 # to use may be chosen automatically based on mpl configuration.
337 # to use may be chosen automatically based on mpl configuration.
337 if self.pylab:
338 if self.pylab:
338 activate = self.shell.enable_pylab
339 activate = self.shell.enable_pylab
339 if self.pylab == 'auto':
340 if self.pylab == 'auto':
340 gui = None
341 gui = None
341 else:
342 else:
342 gui = self.pylab
343 gui = self.pylab
343 else:
344 else:
344 # Enable only GUI integration, no pylab
345 # Enable only GUI integration, no pylab
345 activate = inputhook.enable_gui
346 activate = inputhook.enable_gui
346
347
347 if gui or self.pylab:
348 if gui or self.pylab:
348 try:
349 try:
349 self.log.info("Enabling GUI event loop integration, "
350 self.log.info("Enabling GUI event loop integration, "
350 "toolkit=%s, pylab=%s" % (gui, self.pylab) )
351 "toolkit=%s, pylab=%s" % (gui, self.pylab) )
351 if self.pylab:
352 if self.pylab:
352 activate(gui, import_all=self.pylab_import_all)
353 activate(gui, import_all=self.pylab_import_all)
353 else:
354 else:
354 activate(gui)
355 activate(gui)
355 except:
356 except:
356 self.log.warn("Error in enabling GUI event loop integration:")
357 self.log.warn("Error in enabling GUI event loop integration:")
357 self.shell.showtraceback()
358 self.shell.showtraceback()
358
359
359 def start(self):
360 def start(self):
360 if self.subapp is not None:
361 if self.subapp is not None:
361 return self.subapp.start()
362 return self.subapp.start()
362 # perform any prexec steps:
363 # perform any prexec steps:
363 if self.interact:
364 if self.interact:
364 self.log.debug("Starting IPython's mainloop...")
365 self.log.debug("Starting IPython's mainloop...")
365 self.shell.mainloop()
366 self.shell.mainloop()
366 else:
367 else:
367 self.log.debug("IPython not interactive...")
368 self.log.debug("IPython not interactive...")
368
369
369
370
370 def load_default_config(ipython_dir=None):
371 def load_default_config(ipython_dir=None):
371 """Load the default config file from the default ipython_dir.
372 """Load the default config file from the default ipython_dir.
372
373
373 This is useful for embedded shells.
374 This is useful for embedded shells.
374 """
375 """
375 if ipython_dir is None:
376 if ipython_dir is None:
376 ipython_dir = get_ipython_dir()
377 ipython_dir = get_ipython_dir()
377 profile_dir = os.path.join(ipython_dir, 'profile_default')
378 profile_dir = os.path.join(ipython_dir, 'profile_default')
378 cl = PyFileConfigLoader(default_config_file_name, profile_dir)
379 cl = PyFileConfigLoader(default_config_file_name, profile_dir)
379 try:
380 try:
380 config = cl.load_config()
381 config = cl.load_config()
381 except ConfigFileNotFound:
382 except ConfigFileNotFound:
382 # no config found
383 # no config found
383 config = Config()
384 config = Config()
384 return config
385 return config
385
386
386
387
387 def launch_new_instance():
388 def launch_new_instance():
388 """Create and run a full blown IPython instance"""
389 """Create and run a full blown IPython instance"""
389 app = TerminalIPythonApp.instance()
390 app = TerminalIPythonApp.instance()
390 app.initialize()
391 app.initialize()
391 app.start()
392 app.start()
392
393
393
394
394 if __name__ == '__main__':
395 if __name__ == '__main__':
395 launch_new_instance()
396 launch_new_instance()
@@ -1,261 +1,263 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 The Base Application class for IPython.parallel apps
3 The Base Application class for IPython.parallel apps
4
4
5 Authors:
5 Authors:
6
6
7 * Brian Granger
7 * Brian Granger
8 * Min RK
8 * Min RK
9
9
10 """
10 """
11
11
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13 # Copyright (C) 2008-2011 The IPython Development Team
13 # Copyright (C) 2008-2011 The IPython Development Team
14 #
14 #
15 # Distributed under the terms of the BSD License. The full license is in
15 # Distributed under the terms of the BSD License. The full license is in
16 # the file COPYING, distributed as part of this software.
16 # the file COPYING, distributed as part of this software.
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20 # Imports
20 # Imports
21 #-----------------------------------------------------------------------------
21 #-----------------------------------------------------------------------------
22
22
23 from __future__ import with_statement
23 from __future__ import with_statement
24
24
25 import os
25 import os
26 import logging
26 import logging
27 import re
27 import re
28 import sys
28 import sys
29
29
30 from subprocess import Popen, PIPE
30 from subprocess import Popen, PIPE
31
31
32 from IPython.config.application import catch_config
32 from IPython.core import release
33 from IPython.core import release
33 from IPython.core.crashhandler import CrashHandler
34 from IPython.core.crashhandler import CrashHandler
34 from IPython.core.application import (
35 from IPython.core.application import (
35 BaseIPythonApplication,
36 BaseIPythonApplication,
36 base_aliases as base_ip_aliases,
37 base_aliases as base_ip_aliases,
37 base_flags as base_ip_flags
38 base_flags as base_ip_flags
38 )
39 )
39 from IPython.utils.path import expand_path
40 from IPython.utils.path import expand_path
40
41
41 from IPython.utils.traitlets import Unicode, Bool, Instance, Dict, List
42 from IPython.utils.traitlets import Unicode, Bool, Instance, Dict, List
42
43
43 #-----------------------------------------------------------------------------
44 #-----------------------------------------------------------------------------
44 # Module errors
45 # Module errors
45 #-----------------------------------------------------------------------------
46 #-----------------------------------------------------------------------------
46
47
47 class PIDFileError(Exception):
48 class PIDFileError(Exception):
48 pass
49 pass
49
50
50
51
51 #-----------------------------------------------------------------------------
52 #-----------------------------------------------------------------------------
52 # Crash handler for this application
53 # Crash handler for this application
53 #-----------------------------------------------------------------------------
54 #-----------------------------------------------------------------------------
54
55
55 class ParallelCrashHandler(CrashHandler):
56 class ParallelCrashHandler(CrashHandler):
56 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
57 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
57
58
58 def __init__(self, app):
59 def __init__(self, app):
59 contact_name = release.authors['Min'][0]
60 contact_name = release.authors['Min'][0]
60 contact_email = release.authors['Min'][1]
61 contact_email = release.authors['Min'][1]
61 bug_tracker = 'http://github.com/ipython/ipython/issues'
62 bug_tracker = 'http://github.com/ipython/ipython/issues'
62 super(ParallelCrashHandler,self).__init__(
63 super(ParallelCrashHandler,self).__init__(
63 app, contact_name, contact_email, bug_tracker
64 app, contact_name, contact_email, bug_tracker
64 )
65 )
65
66
66
67
67 #-----------------------------------------------------------------------------
68 #-----------------------------------------------------------------------------
68 # Main application
69 # Main application
69 #-----------------------------------------------------------------------------
70 #-----------------------------------------------------------------------------
70 base_aliases = {}
71 base_aliases = {}
71 base_aliases.update(base_ip_aliases)
72 base_aliases.update(base_ip_aliases)
72 base_aliases.update({
73 base_aliases.update({
73 'profile-dir' : 'ProfileDir.location',
74 'profile-dir' : 'ProfileDir.location',
74 'work-dir' : 'BaseParallelApplication.work_dir',
75 'work-dir' : 'BaseParallelApplication.work_dir',
75 'log-to-file' : 'BaseParallelApplication.log_to_file',
76 'log-to-file' : 'BaseParallelApplication.log_to_file',
76 'clean-logs' : 'BaseParallelApplication.clean_logs',
77 'clean-logs' : 'BaseParallelApplication.clean_logs',
77 'log-url' : 'BaseParallelApplication.log_url',
78 'log-url' : 'BaseParallelApplication.log_url',
78 'cluster-id' : 'BaseParallelApplication.cluster_id',
79 'cluster-id' : 'BaseParallelApplication.cluster_id',
79 })
80 })
80
81
81 base_flags = {
82 base_flags = {
82 'log-to-file' : (
83 'log-to-file' : (
83 {'BaseParallelApplication' : {'log_to_file' : True}},
84 {'BaseParallelApplication' : {'log_to_file' : True}},
84 "send log output to a file"
85 "send log output to a file"
85 )
86 )
86 }
87 }
87 base_flags.update(base_ip_flags)
88 base_flags.update(base_ip_flags)
88
89
89 class BaseParallelApplication(BaseIPythonApplication):
90 class BaseParallelApplication(BaseIPythonApplication):
90 """The base Application for IPython.parallel apps
91 """The base Application for IPython.parallel apps
91
92
92 Principle extensions to BaseIPyythonApplication:
93 Principle extensions to BaseIPyythonApplication:
93
94
94 * work_dir
95 * work_dir
95 * remote logging via pyzmq
96 * remote logging via pyzmq
96 * IOLoop instance
97 * IOLoop instance
97 """
98 """
98
99
99 crash_handler_class = ParallelCrashHandler
100 crash_handler_class = ParallelCrashHandler
100
101
101 def _log_level_default(self):
102 def _log_level_default(self):
102 # temporarily override default_log_level to INFO
103 # temporarily override default_log_level to INFO
103 return logging.INFO
104 return logging.INFO
104
105
105 work_dir = Unicode(os.getcwdu(), config=True,
106 work_dir = Unicode(os.getcwdu(), config=True,
106 help='Set the working dir for the process.'
107 help='Set the working dir for the process.'
107 )
108 )
108 def _work_dir_changed(self, name, old, new):
109 def _work_dir_changed(self, name, old, new):
109 self.work_dir = unicode(expand_path(new))
110 self.work_dir = unicode(expand_path(new))
110
111
111 log_to_file = Bool(config=True,
112 log_to_file = Bool(config=True,
112 help="whether to log to a file")
113 help="whether to log to a file")
113
114
114 clean_logs = Bool(False, config=True,
115 clean_logs = Bool(False, config=True,
115 help="whether to cleanup old logfiles before starting")
116 help="whether to cleanup old logfiles before starting")
116
117
117 log_url = Unicode('', config=True,
118 log_url = Unicode('', config=True,
118 help="The ZMQ URL of the iplogger to aggregate logging.")
119 help="The ZMQ URL of the iplogger to aggregate logging.")
119
120
120 cluster_id = Unicode('', config=True,
121 cluster_id = Unicode('', config=True,
121 help="""String id to add to runtime files, to prevent name collisions when
122 help="""String id to add to runtime files, to prevent name collisions when
122 using multiple clusters with a single profile simultaneously.
123 using multiple clusters with a single profile simultaneously.
123
124
124 When set, files will be named like: 'ipcontroller-<cluster_id>-engine.json'
125 When set, files will be named like: 'ipcontroller-<cluster_id>-engine.json'
125
126
126 Since this is text inserted into filenames, typical recommendations apply:
127 Since this is text inserted into filenames, typical recommendations apply:
127 Simple character strings are ideal, and spaces are not recommended (but should
128 Simple character strings are ideal, and spaces are not recommended (but should
128 generally work).
129 generally work).
129 """
130 """
130 )
131 )
131 def _cluster_id_changed(self, name, old, new):
132 def _cluster_id_changed(self, name, old, new):
132 self.name = self.__class__.name
133 self.name = self.__class__.name
133 if new:
134 if new:
134 self.name += '-%s'%new
135 self.name += '-%s'%new
135
136
136 def _config_files_default(self):
137 def _config_files_default(self):
137 return ['ipcontroller_config.py', 'ipengine_config.py', 'ipcluster_config.py']
138 return ['ipcontroller_config.py', 'ipengine_config.py', 'ipcluster_config.py']
138
139
139 loop = Instance('zmq.eventloop.ioloop.IOLoop')
140 loop = Instance('zmq.eventloop.ioloop.IOLoop')
140 def _loop_default(self):
141 def _loop_default(self):
141 from zmq.eventloop.ioloop import IOLoop
142 from zmq.eventloop.ioloop import IOLoop
142 return IOLoop.instance()
143 return IOLoop.instance()
143
144
144 aliases = Dict(base_aliases)
145 aliases = Dict(base_aliases)
145 flags = Dict(base_flags)
146 flags = Dict(base_flags)
146
147
148 @catch_config
147 def initialize(self, argv=None):
149 def initialize(self, argv=None):
148 """initialize the app"""
150 """initialize the app"""
149 super(BaseParallelApplication, self).initialize(argv)
151 super(BaseParallelApplication, self).initialize(argv)
150 self.to_work_dir()
152 self.to_work_dir()
151 self.reinit_logging()
153 self.reinit_logging()
152
154
153 def to_work_dir(self):
155 def to_work_dir(self):
154 wd = self.work_dir
156 wd = self.work_dir
155 if unicode(wd) != os.getcwdu():
157 if unicode(wd) != os.getcwdu():
156 os.chdir(wd)
158 os.chdir(wd)
157 self.log.info("Changing to working dir: %s" % wd)
159 self.log.info("Changing to working dir: %s" % wd)
158 # This is the working dir by now.
160 # This is the working dir by now.
159 sys.path.insert(0, '')
161 sys.path.insert(0, '')
160
162
161 def reinit_logging(self):
163 def reinit_logging(self):
162 # Remove old log files
164 # Remove old log files
163 log_dir = self.profile_dir.log_dir
165 log_dir = self.profile_dir.log_dir
164 if self.clean_logs:
166 if self.clean_logs:
165 for f in os.listdir(log_dir):
167 for f in os.listdir(log_dir):
166 if re.match(r'%s-\d+\.(log|err|out)'%self.name,f):
168 if re.match(r'%s-\d+\.(log|err|out)'%self.name,f):
167 os.remove(os.path.join(log_dir, f))
169 os.remove(os.path.join(log_dir, f))
168 if self.log_to_file:
170 if self.log_to_file:
169 # Start logging to the new log file
171 # Start logging to the new log file
170 log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
172 log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
171 logfile = os.path.join(log_dir, log_filename)
173 logfile = os.path.join(log_dir, log_filename)
172 open_log_file = open(logfile, 'w')
174 open_log_file = open(logfile, 'w')
173 else:
175 else:
174 open_log_file = None
176 open_log_file = None
175 if open_log_file is not None:
177 if open_log_file is not None:
176 self.log.removeHandler(self._log_handler)
178 self.log.removeHandler(self._log_handler)
177 self._log_handler = logging.StreamHandler(open_log_file)
179 self._log_handler = logging.StreamHandler(open_log_file)
178 self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
180 self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
179 self._log_handler.setFormatter(self._log_formatter)
181 self._log_handler.setFormatter(self._log_formatter)
180 self.log.addHandler(self._log_handler)
182 self.log.addHandler(self._log_handler)
181 # do not propagate log messages to root logger
183 # do not propagate log messages to root logger
182 # ipcluster app will sometimes print duplicate messages during shutdown
184 # ipcluster app will sometimes print duplicate messages during shutdown
183 # if this is 1 (default):
185 # if this is 1 (default):
184 self.log.propagate = False
186 self.log.propagate = False
185
187
186 def write_pid_file(self, overwrite=False):
188 def write_pid_file(self, overwrite=False):
187 """Create a .pid file in the pid_dir with my pid.
189 """Create a .pid file in the pid_dir with my pid.
188
190
189 This must be called after pre_construct, which sets `self.pid_dir`.
191 This must be called after pre_construct, which sets `self.pid_dir`.
190 This raises :exc:`PIDFileError` if the pid file exists already.
192 This raises :exc:`PIDFileError` if the pid file exists already.
191 """
193 """
192 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
194 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
193 if os.path.isfile(pid_file):
195 if os.path.isfile(pid_file):
194 pid = self.get_pid_from_file()
196 pid = self.get_pid_from_file()
195 if not overwrite:
197 if not overwrite:
196 raise PIDFileError(
198 raise PIDFileError(
197 'The pid file [%s] already exists. \nThis could mean that this '
199 'The pid file [%s] already exists. \nThis could mean that this '
198 'server is already running with [pid=%s].' % (pid_file, pid)
200 'server is already running with [pid=%s].' % (pid_file, pid)
199 )
201 )
200 with open(pid_file, 'w') as f:
202 with open(pid_file, 'w') as f:
201 self.log.info("Creating pid file: %s" % pid_file)
203 self.log.info("Creating pid file: %s" % pid_file)
202 f.write(repr(os.getpid())+'\n')
204 f.write(repr(os.getpid())+'\n')
203
205
204 def remove_pid_file(self):
206 def remove_pid_file(self):
205 """Remove the pid file.
207 """Remove the pid file.
206
208
207 This should be called at shutdown by registering a callback with
209 This should be called at shutdown by registering a callback with
208 :func:`reactor.addSystemEventTrigger`. This needs to return
210 :func:`reactor.addSystemEventTrigger`. This needs to return
209 ``None``.
211 ``None``.
210 """
212 """
211 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
213 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
212 if os.path.isfile(pid_file):
214 if os.path.isfile(pid_file):
213 try:
215 try:
214 self.log.info("Removing pid file: %s" % pid_file)
216 self.log.info("Removing pid file: %s" % pid_file)
215 os.remove(pid_file)
217 os.remove(pid_file)
216 except:
218 except:
217 self.log.warn("Error removing the pid file: %s" % pid_file)
219 self.log.warn("Error removing the pid file: %s" % pid_file)
218
220
219 def get_pid_from_file(self):
221 def get_pid_from_file(self):
220 """Get the pid from the pid file.
222 """Get the pid from the pid file.
221
223
222 If the pid file doesn't exist a :exc:`PIDFileError` is raised.
224 If the pid file doesn't exist a :exc:`PIDFileError` is raised.
223 """
225 """
224 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
226 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
225 if os.path.isfile(pid_file):
227 if os.path.isfile(pid_file):
226 with open(pid_file, 'r') as f:
228 with open(pid_file, 'r') as f:
227 s = f.read().strip()
229 s = f.read().strip()
228 try:
230 try:
229 pid = int(s)
231 pid = int(s)
230 except:
232 except:
231 raise PIDFileError("invalid pid file: %s (contents: %r)"%(pid_file, s))
233 raise PIDFileError("invalid pid file: %s (contents: %r)"%(pid_file, s))
232 return pid
234 return pid
233 else:
235 else:
234 raise PIDFileError('pid file not found: %s' % pid_file)
236 raise PIDFileError('pid file not found: %s' % pid_file)
235
237
236 def check_pid(self, pid):
238 def check_pid(self, pid):
237 if os.name == 'nt':
239 if os.name == 'nt':
238 try:
240 try:
239 import ctypes
241 import ctypes
240 # returns 0 if no such process (of ours) exists
242 # returns 0 if no such process (of ours) exists
241 # positive int otherwise
243 # positive int otherwise
242 p = ctypes.windll.kernel32.OpenProcess(1,0,pid)
244 p = ctypes.windll.kernel32.OpenProcess(1,0,pid)
243 except Exception:
245 except Exception:
244 self.log.warn(
246 self.log.warn(
245 "Could not determine whether pid %i is running via `OpenProcess`. "
247 "Could not determine whether pid %i is running via `OpenProcess`. "
246 " Making the likely assumption that it is."%pid
248 " Making the likely assumption that it is."%pid
247 )
249 )
248 return True
250 return True
249 return bool(p)
251 return bool(p)
250 else:
252 else:
251 try:
253 try:
252 p = Popen(['ps','x'], stdout=PIPE, stderr=PIPE)
254 p = Popen(['ps','x'], stdout=PIPE, stderr=PIPE)
253 output,_ = p.communicate()
255 output,_ = p.communicate()
254 except OSError:
256 except OSError:
255 self.log.warn(
257 self.log.warn(
256 "Could not determine whether pid %i is running via `ps x`. "
258 "Could not determine whether pid %i is running via `ps x`. "
257 " Making the likely assumption that it is."%pid
259 " Making the likely assumption that it is."%pid
258 )
260 )
259 return True
261 return True
260 pids = map(int, re.findall(r'^\W*\d+', output, re.MULTILINE))
262 pids = map(int, re.findall(r'^\W*\d+', output, re.MULTILINE))
261 return pid in pids
263 return pid in pids
@@ -1,522 +1,523 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The ipcluster application.
4 The ipcluster application.
5
5
6 Authors:
6 Authors:
7
7
8 * Brian Granger
8 * Brian Granger
9 * MinRK
9 * MinRK
10
10
11 """
11 """
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Copyright (C) 2008-2011 The IPython Development Team
14 # Copyright (C) 2008-2011 The IPython Development Team
15 #
15 #
16 # Distributed under the terms of the BSD License. The full license is in
16 # Distributed under the terms of the BSD License. The full license is in
17 # the file COPYING, distributed as part of this software.
17 # the file COPYING, distributed as part of this software.
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Imports
21 # Imports
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 import errno
24 import errno
25 import logging
25 import logging
26 import os
26 import os
27 import re
27 import re
28 import signal
28 import signal
29
29
30 from subprocess import check_call, CalledProcessError, PIPE
30 from subprocess import check_call, CalledProcessError, PIPE
31 import zmq
31 import zmq
32 from zmq.eventloop import ioloop
32 from zmq.eventloop import ioloop
33
33
34 from IPython.config.application import Application, boolean_flag
34 from IPython.config.application import Application, boolean_flag, catch_config
35 from IPython.config.loader import Config
35 from IPython.config.loader import Config
36 from IPython.core.application import BaseIPythonApplication
36 from IPython.core.application import BaseIPythonApplication
37 from IPython.core.profiledir import ProfileDir
37 from IPython.core.profiledir import ProfileDir
38 from IPython.utils.daemonize import daemonize
38 from IPython.utils.daemonize import daemonize
39 from IPython.utils.importstring import import_item
39 from IPython.utils.importstring import import_item
40 from IPython.utils.sysinfo import num_cpus
40 from IPython.utils.sysinfo import num_cpus
41 from IPython.utils.traitlets import (Int, Unicode, Bool, CFloat, Dict, List,
41 from IPython.utils.traitlets import (Int, Unicode, Bool, CFloat, Dict, List,
42 DottedObjectName)
42 DottedObjectName)
43
43
44 from IPython.parallel.apps.baseapp import (
44 from IPython.parallel.apps.baseapp import (
45 BaseParallelApplication,
45 BaseParallelApplication,
46 PIDFileError,
46 PIDFileError,
47 base_flags, base_aliases
47 base_flags, base_aliases
48 )
48 )
49
49
50
50
51 #-----------------------------------------------------------------------------
51 #-----------------------------------------------------------------------------
52 # Module level variables
52 # Module level variables
53 #-----------------------------------------------------------------------------
53 #-----------------------------------------------------------------------------
54
54
55
55
56 default_config_file_name = u'ipcluster_config.py'
56 default_config_file_name = u'ipcluster_config.py'
57
57
58
58
59 _description = """Start an IPython cluster for parallel computing.
59 _description = """Start an IPython cluster for parallel computing.
60
60
61 An IPython cluster consists of 1 controller and 1 or more engines.
61 An IPython cluster consists of 1 controller and 1 or more engines.
62 This command automates the startup of these processes using a wide
62 This command automates the startup of these processes using a wide
63 range of startup methods (SSH, local processes, PBS, mpiexec,
63 range of startup methods (SSH, local processes, PBS, mpiexec,
64 Windows HPC Server 2008). To start a cluster with 4 engines on your
64 Windows HPC Server 2008). To start a cluster with 4 engines on your
65 local host simply do 'ipcluster start --n=4'. For more complex usage
65 local host simply do 'ipcluster start --n=4'. For more complex usage
66 you will typically do 'ipython profile create mycluster --parallel', then edit
66 you will typically do 'ipython profile create mycluster --parallel', then edit
67 configuration files, followed by 'ipcluster start --profile=mycluster --n=4'.
67 configuration files, followed by 'ipcluster start --profile=mycluster --n=4'.
68 """
68 """
69
69
70 _main_examples = """
70 _main_examples = """
71 ipcluster start --n=4 # start a 4 node cluster on localhost
71 ipcluster start --n=4 # start a 4 node cluster on localhost
72 ipcluster start -h # show the help string for the start subcmd
72 ipcluster start -h # show the help string for the start subcmd
73
73
74 ipcluster stop -h # show the help string for the stop subcmd
74 ipcluster stop -h # show the help string for the stop subcmd
75 ipcluster engines -h # show the help string for the engines subcmd
75 ipcluster engines -h # show the help string for the engines subcmd
76 """
76 """
77
77
78 _start_examples = """
78 _start_examples = """
79 ipython profile create mycluster --parallel # create mycluster profile
79 ipython profile create mycluster --parallel # create mycluster profile
80 ipcluster start --profile=mycluster --n=4 # start mycluster with 4 nodes
80 ipcluster start --profile=mycluster --n=4 # start mycluster with 4 nodes
81 """
81 """
82
82
83 _stop_examples = """
83 _stop_examples = """
84 ipcluster stop --profile=mycluster # stop a running cluster by profile name
84 ipcluster stop --profile=mycluster # stop a running cluster by profile name
85 """
85 """
86
86
87 _engines_examples = """
87 _engines_examples = """
88 ipcluster engines --profile=mycluster --n=4 # start 4 engines only
88 ipcluster engines --profile=mycluster --n=4 # start 4 engines only
89 """
89 """
90
90
91
91
92 # Exit codes for ipcluster
92 # Exit codes for ipcluster
93
93
94 # This will be the exit code if the ipcluster appears to be running because
94 # This will be the exit code if the ipcluster appears to be running because
95 # a .pid file exists
95 # a .pid file exists
96 ALREADY_STARTED = 10
96 ALREADY_STARTED = 10
97
97
98
98
99 # This will be the exit code if ipcluster stop is run, but there is not .pid
99 # This will be the exit code if ipcluster stop is run, but there is not .pid
100 # file to be found.
100 # file to be found.
101 ALREADY_STOPPED = 11
101 ALREADY_STOPPED = 11
102
102
103 # This will be the exit code if ipcluster engines is run, but there is not .pid
103 # This will be the exit code if ipcluster engines is run, but there is not .pid
104 # file to be found.
104 # file to be found.
105 NO_CLUSTER = 12
105 NO_CLUSTER = 12
106
106
107
107
108 #-----------------------------------------------------------------------------
108 #-----------------------------------------------------------------------------
109 # Main application
109 # Main application
110 #-----------------------------------------------------------------------------
110 #-----------------------------------------------------------------------------
111 start_help = """Start an IPython cluster for parallel computing
111 start_help = """Start an IPython cluster for parallel computing
112
112
113 Start an ipython cluster by its profile name or cluster
113 Start an ipython cluster by its profile name or cluster
114 directory. Cluster directories contain configuration, log and
114 directory. Cluster directories contain configuration, log and
115 security related files and are named using the convention
115 security related files and are named using the convention
116 'profile_<name>' and should be creating using the 'start'
116 'profile_<name>' and should be creating using the 'start'
117 subcommand of 'ipcluster'. If your cluster directory is in
117 subcommand of 'ipcluster'. If your cluster directory is in
118 the cwd or the ipython directory, you can simply refer to it
118 the cwd or the ipython directory, you can simply refer to it
119 using its profile name, 'ipcluster start --n=4 --profile=<profile>`,
119 using its profile name, 'ipcluster start --n=4 --profile=<profile>`,
120 otherwise use the 'profile-dir' option.
120 otherwise use the 'profile-dir' option.
121 """
121 """
122 stop_help = """Stop a running IPython cluster
122 stop_help = """Stop a running IPython cluster
123
123
124 Stop a running ipython cluster by its profile name or cluster
124 Stop a running ipython cluster by its profile name or cluster
125 directory. Cluster directories are named using the convention
125 directory. Cluster directories are named using the convention
126 'profile_<name>'. If your cluster directory is in
126 'profile_<name>'. If your cluster directory is in
127 the cwd or the ipython directory, you can simply refer to it
127 the cwd or the ipython directory, you can simply refer to it
128 using its profile name, 'ipcluster stop --profile=<profile>`, otherwise
128 using its profile name, 'ipcluster stop --profile=<profile>`, otherwise
129 use the '--profile-dir' option.
129 use the '--profile-dir' option.
130 """
130 """
131 engines_help = """Start engines connected to an existing IPython cluster
131 engines_help = """Start engines connected to an existing IPython cluster
132
132
133 Start one or more engines to connect to an existing Cluster
133 Start one or more engines to connect to an existing Cluster
134 by profile name or cluster directory.
134 by profile name or cluster directory.
135 Cluster directories contain configuration, log and
135 Cluster directories contain configuration, log and
136 security related files and are named using the convention
136 security related files and are named using the convention
137 'profile_<name>' and should be creating using the 'start'
137 'profile_<name>' and should be creating using the 'start'
138 subcommand of 'ipcluster'. If your cluster directory is in
138 subcommand of 'ipcluster'. If your cluster directory is in
139 the cwd or the ipython directory, you can simply refer to it
139 the cwd or the ipython directory, you can simply refer to it
140 using its profile name, 'ipcluster engines --n=4 --profile=<profile>`,
140 using its profile name, 'ipcluster engines --n=4 --profile=<profile>`,
141 otherwise use the 'profile-dir' option.
141 otherwise use the 'profile-dir' option.
142 """
142 """
143 stop_aliases = dict(
143 stop_aliases = dict(
144 signal='IPClusterStop.signal',
144 signal='IPClusterStop.signal',
145 )
145 )
146 stop_aliases.update(base_aliases)
146 stop_aliases.update(base_aliases)
147
147
148 class IPClusterStop(BaseParallelApplication):
148 class IPClusterStop(BaseParallelApplication):
149 name = u'ipcluster'
149 name = u'ipcluster'
150 description = stop_help
150 description = stop_help
151 examples = _stop_examples
151 examples = _stop_examples
152 config_file_name = Unicode(default_config_file_name)
152 config_file_name = Unicode(default_config_file_name)
153
153
154 signal = Int(signal.SIGINT, config=True,
154 signal = Int(signal.SIGINT, config=True,
155 help="signal to use for stopping processes.")
155 help="signal to use for stopping processes.")
156
156
157 aliases = Dict(stop_aliases)
157 aliases = Dict(stop_aliases)
158
158
159 def start(self):
159 def start(self):
160 """Start the app for the stop subcommand."""
160 """Start the app for the stop subcommand."""
161 try:
161 try:
162 pid = self.get_pid_from_file()
162 pid = self.get_pid_from_file()
163 except PIDFileError:
163 except PIDFileError:
164 self.log.critical(
164 self.log.critical(
165 'Could not read pid file, cluster is probably not running.'
165 'Could not read pid file, cluster is probably not running.'
166 )
166 )
167 # Here I exit with a unusual exit status that other processes
167 # Here I exit with a unusual exit status that other processes
168 # can watch for to learn how I existed.
168 # can watch for to learn how I existed.
169 self.remove_pid_file()
169 self.remove_pid_file()
170 self.exit(ALREADY_STOPPED)
170 self.exit(ALREADY_STOPPED)
171
171
172 if not self.check_pid(pid):
172 if not self.check_pid(pid):
173 self.log.critical(
173 self.log.critical(
174 'Cluster [pid=%r] is not running.' % pid
174 'Cluster [pid=%r] is not running.' % pid
175 )
175 )
176 self.remove_pid_file()
176 self.remove_pid_file()
177 # Here I exit with a unusual exit status that other processes
177 # Here I exit with a unusual exit status that other processes
178 # can watch for to learn how I existed.
178 # can watch for to learn how I existed.
179 self.exit(ALREADY_STOPPED)
179 self.exit(ALREADY_STOPPED)
180
180
181 elif os.name=='posix':
181 elif os.name=='posix':
182 sig = self.signal
182 sig = self.signal
183 self.log.info(
183 self.log.info(
184 "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
184 "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
185 )
185 )
186 try:
186 try:
187 os.kill(pid, sig)
187 os.kill(pid, sig)
188 except OSError:
188 except OSError:
189 self.log.error("Stopping cluster failed, assuming already dead.",
189 self.log.error("Stopping cluster failed, assuming already dead.",
190 exc_info=True)
190 exc_info=True)
191 self.remove_pid_file()
191 self.remove_pid_file()
192 elif os.name=='nt':
192 elif os.name=='nt':
193 try:
193 try:
194 # kill the whole tree
194 # kill the whole tree
195 p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE)
195 p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE)
196 except (CalledProcessError, OSError):
196 except (CalledProcessError, OSError):
197 self.log.error("Stopping cluster failed, assuming already dead.",
197 self.log.error("Stopping cluster failed, assuming already dead.",
198 exc_info=True)
198 exc_info=True)
199 self.remove_pid_file()
199 self.remove_pid_file()
200
200
201 engine_aliases = {}
201 engine_aliases = {}
202 engine_aliases.update(base_aliases)
202 engine_aliases.update(base_aliases)
203 engine_aliases.update(dict(
203 engine_aliases.update(dict(
204 n='IPClusterEngines.n',
204 n='IPClusterEngines.n',
205 engines = 'IPClusterEngines.engine_launcher_class',
205 engines = 'IPClusterEngines.engine_launcher_class',
206 daemonize = 'IPClusterEngines.daemonize',
206 daemonize = 'IPClusterEngines.daemonize',
207 ))
207 ))
208 engine_flags = {}
208 engine_flags = {}
209 engine_flags.update(base_flags)
209 engine_flags.update(base_flags)
210
210
211 engine_flags.update(dict(
211 engine_flags.update(dict(
212 daemonize=(
212 daemonize=(
213 {'IPClusterEngines' : {'daemonize' : True}},
213 {'IPClusterEngines' : {'daemonize' : True}},
214 """run the cluster into the background (not available on Windows)""",
214 """run the cluster into the background (not available on Windows)""",
215 )
215 )
216 ))
216 ))
217 class IPClusterEngines(BaseParallelApplication):
217 class IPClusterEngines(BaseParallelApplication):
218
218
219 name = u'ipcluster'
219 name = u'ipcluster'
220 description = engines_help
220 description = engines_help
221 examples = _engines_examples
221 examples = _engines_examples
222 usage = None
222 usage = None
223 config_file_name = Unicode(default_config_file_name)
223 config_file_name = Unicode(default_config_file_name)
224 default_log_level = logging.INFO
224 default_log_level = logging.INFO
225 classes = List()
225 classes = List()
226 def _classes_default(self):
226 def _classes_default(self):
227 from IPython.parallel.apps import launcher
227 from IPython.parallel.apps import launcher
228 launchers = launcher.all_launchers
228 launchers = launcher.all_launchers
229 eslaunchers = [ l for l in launchers if 'EngineSet' in l.__name__]
229 eslaunchers = [ l for l in launchers if 'EngineSet' in l.__name__]
230 return [ProfileDir]+eslaunchers
230 return [ProfileDir]+eslaunchers
231
231
232 n = Int(num_cpus(), config=True,
232 n = Int(num_cpus(), config=True,
233 help="""The number of engines to start. The default is to use one for each
233 help="""The number of engines to start. The default is to use one for each
234 CPU on your machine""")
234 CPU on your machine""")
235
235
236 engine_launcher_class = DottedObjectName('LocalEngineSetLauncher',
236 engine_launcher_class = DottedObjectName('LocalEngineSetLauncher',
237 config=True,
237 config=True,
238 help="""The class for launching a set of Engines. Change this value
238 help="""The class for launching a set of Engines. Change this value
239 to use various batch systems to launch your engines, such as PBS,SGE,MPIExec,etc.
239 to use various batch systems to launch your engines, such as PBS,SGE,MPIExec,etc.
240 Each launcher class has its own set of configuration options, for making sure
240 Each launcher class has its own set of configuration options, for making sure
241 it will work in your environment.
241 it will work in your environment.
242
242
243 You can also write your own launcher, and specify it's absolute import path,
243 You can also write your own launcher, and specify it's absolute import path,
244 as in 'mymodule.launcher.FTLEnginesLauncher`.
244 as in 'mymodule.launcher.FTLEnginesLauncher`.
245
245
246 Examples include:
246 Examples include:
247
247
248 LocalEngineSetLauncher : start engines locally as subprocesses [default]
248 LocalEngineSetLauncher : start engines locally as subprocesses [default]
249 MPIExecEngineSetLauncher : use mpiexec to launch in an MPI environment
249 MPIExecEngineSetLauncher : use mpiexec to launch in an MPI environment
250 PBSEngineSetLauncher : use PBS (qsub) to submit engines to a batch queue
250 PBSEngineSetLauncher : use PBS (qsub) to submit engines to a batch queue
251 SGEEngineSetLauncher : use SGE (qsub) to submit engines to a batch queue
251 SGEEngineSetLauncher : use SGE (qsub) to submit engines to a batch queue
252 SSHEngineSetLauncher : use SSH to start the controller
252 SSHEngineSetLauncher : use SSH to start the controller
253 Note that SSH does *not* move the connection files
253 Note that SSH does *not* move the connection files
254 around, so you will likely have to do this manually
254 around, so you will likely have to do this manually
255 unless the machines are on a shared file system.
255 unless the machines are on a shared file system.
256 WindowsHPCEngineSetLauncher : use Windows HPC
256 WindowsHPCEngineSetLauncher : use Windows HPC
257 """
257 """
258 )
258 )
259 daemonize = Bool(False, config=True,
259 daemonize = Bool(False, config=True,
260 help="""Daemonize the ipcluster program. This implies --log-to-file.
260 help="""Daemonize the ipcluster program. This implies --log-to-file.
261 Not available on Windows.
261 Not available on Windows.
262 """)
262 """)
263
263
264 def _daemonize_changed(self, name, old, new):
264 def _daemonize_changed(self, name, old, new):
265 if new:
265 if new:
266 self.log_to_file = True
266 self.log_to_file = True
267
267
268 aliases = Dict(engine_aliases)
268 aliases = Dict(engine_aliases)
269 flags = Dict(engine_flags)
269 flags = Dict(engine_flags)
270 _stopping = False
270 _stopping = False
271
271
272 @catch_config
272 def initialize(self, argv=None):
273 def initialize(self, argv=None):
273 super(IPClusterEngines, self).initialize(argv)
274 super(IPClusterEngines, self).initialize(argv)
274 self.init_signal()
275 self.init_signal()
275 self.init_launchers()
276 self.init_launchers()
276
277
277 def init_launchers(self):
278 def init_launchers(self):
278 self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet')
279 self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet')
279 self.engine_launcher.on_stop(lambda r: self.loop.stop())
280 self.engine_launcher.on_stop(lambda r: self.loop.stop())
280
281
281 def init_signal(self):
282 def init_signal(self):
282 # Setup signals
283 # Setup signals
283 signal.signal(signal.SIGINT, self.sigint_handler)
284 signal.signal(signal.SIGINT, self.sigint_handler)
284
285
285 def build_launcher(self, clsname, kind=None):
286 def build_launcher(self, clsname, kind=None):
286 """import and instantiate a Launcher based on importstring"""
287 """import and instantiate a Launcher based on importstring"""
287 if '.' not in clsname:
288 if '.' not in clsname:
288 # not a module, presume it's the raw name in apps.launcher
289 # not a module, presume it's the raw name in apps.launcher
289 if kind and kind not in clsname:
290 if kind and kind not in clsname:
290 # doesn't match necessary full class name, assume it's
291 # doesn't match necessary full class name, assume it's
291 # just 'PBS' or 'MPIExec' prefix:
292 # just 'PBS' or 'MPIExec' prefix:
292 clsname = clsname + kind + 'Launcher'
293 clsname = clsname + kind + 'Launcher'
293 clsname = 'IPython.parallel.apps.launcher.'+clsname
294 clsname = 'IPython.parallel.apps.launcher.'+clsname
294 try:
295 try:
295 klass = import_item(clsname)
296 klass = import_item(clsname)
296 except (ImportError, KeyError):
297 except (ImportError, KeyError):
297 self.log.fatal("Could not import launcher class: %r"%clsname)
298 self.log.fatal("Could not import launcher class: %r"%clsname)
298 self.exit(1)
299 self.exit(1)
299
300
300 launcher = klass(
301 launcher = klass(
301 work_dir=u'.', config=self.config, log=self.log,
302 work_dir=u'.', config=self.config, log=self.log,
302 profile_dir=self.profile_dir.location, cluster_id=self.cluster_id,
303 profile_dir=self.profile_dir.location, cluster_id=self.cluster_id,
303 )
304 )
304 return launcher
305 return launcher
305
306
306 def start_engines(self):
307 def start_engines(self):
307 self.log.info("Starting %i engines"%self.n)
308 self.log.info("Starting %i engines"%self.n)
308 self.engine_launcher.start(self.n)
309 self.engine_launcher.start(self.n)
309
310
310 def stop_engines(self):
311 def stop_engines(self):
311 self.log.info("Stopping Engines...")
312 self.log.info("Stopping Engines...")
312 if self.engine_launcher.running:
313 if self.engine_launcher.running:
313 d = self.engine_launcher.stop()
314 d = self.engine_launcher.stop()
314 return d
315 return d
315 else:
316 else:
316 return None
317 return None
317
318
318 def stop_launchers(self, r=None):
319 def stop_launchers(self, r=None):
319 if not self._stopping:
320 if not self._stopping:
320 self._stopping = True
321 self._stopping = True
321 self.log.error("IPython cluster: stopping")
322 self.log.error("IPython cluster: stopping")
322 self.stop_engines()
323 self.stop_engines()
323 # Wait a few seconds to let things shut down.
324 # Wait a few seconds to let things shut down.
324 dc = ioloop.DelayedCallback(self.loop.stop, 4000, self.loop)
325 dc = ioloop.DelayedCallback(self.loop.stop, 4000, self.loop)
325 dc.start()
326 dc.start()
326
327
327 def sigint_handler(self, signum, frame):
328 def sigint_handler(self, signum, frame):
328 self.log.debug("SIGINT received, stopping launchers...")
329 self.log.debug("SIGINT received, stopping launchers...")
329 self.stop_launchers()
330 self.stop_launchers()
330
331
331 def start_logging(self):
332 def start_logging(self):
332 # Remove old log files of the controller and engine
333 # Remove old log files of the controller and engine
333 if self.clean_logs:
334 if self.clean_logs:
334 log_dir = self.profile_dir.log_dir
335 log_dir = self.profile_dir.log_dir
335 for f in os.listdir(log_dir):
336 for f in os.listdir(log_dir):
336 if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f):
337 if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f):
337 os.remove(os.path.join(log_dir, f))
338 os.remove(os.path.join(log_dir, f))
338 # This will remove old log files for ipcluster itself
339 # This will remove old log files for ipcluster itself
339 # super(IPBaseParallelApplication, self).start_logging()
340 # super(IPBaseParallelApplication, self).start_logging()
340
341
341 def start(self):
342 def start(self):
342 """Start the app for the engines subcommand."""
343 """Start the app for the engines subcommand."""
343 self.log.info("IPython cluster: started")
344 self.log.info("IPython cluster: started")
344 # First see if the cluster is already running
345 # First see if the cluster is already running
345
346
346 # Now log and daemonize
347 # Now log and daemonize
347 self.log.info(
348 self.log.info(
348 'Starting engines with [daemon=%r]' % self.daemonize
349 'Starting engines with [daemon=%r]' % self.daemonize
349 )
350 )
350 # TODO: Get daemonize working on Windows or as a Windows Server.
351 # TODO: Get daemonize working on Windows or as a Windows Server.
351 if self.daemonize:
352 if self.daemonize:
352 if os.name=='posix':
353 if os.name=='posix':
353 daemonize()
354 daemonize()
354
355
355 dc = ioloop.DelayedCallback(self.start_engines, 0, self.loop)
356 dc = ioloop.DelayedCallback(self.start_engines, 0, self.loop)
356 dc.start()
357 dc.start()
357 # Now write the new pid file AFTER our new forked pid is active.
358 # Now write the new pid file AFTER our new forked pid is active.
358 # self.write_pid_file()
359 # self.write_pid_file()
359 try:
360 try:
360 self.loop.start()
361 self.loop.start()
361 except KeyboardInterrupt:
362 except KeyboardInterrupt:
362 pass
363 pass
363 except zmq.ZMQError as e:
364 except zmq.ZMQError as e:
364 if e.errno == errno.EINTR:
365 if e.errno == errno.EINTR:
365 pass
366 pass
366 else:
367 else:
367 raise
368 raise
368
369
369 start_aliases = {}
370 start_aliases = {}
370 start_aliases.update(engine_aliases)
371 start_aliases.update(engine_aliases)
371 start_aliases.update(dict(
372 start_aliases.update(dict(
372 delay='IPClusterStart.delay',
373 delay='IPClusterStart.delay',
373 controller = 'IPClusterStart.controller_launcher_class',
374 controller = 'IPClusterStart.controller_launcher_class',
374 ))
375 ))
375 start_aliases['clean-logs'] = 'IPClusterStart.clean_logs'
376 start_aliases['clean-logs'] = 'IPClusterStart.clean_logs'
376
377
377 class IPClusterStart(IPClusterEngines):
378 class IPClusterStart(IPClusterEngines):
378
379
379 name = u'ipcluster'
380 name = u'ipcluster'
380 description = start_help
381 description = start_help
381 examples = _start_examples
382 examples = _start_examples
382 default_log_level = logging.INFO
383 default_log_level = logging.INFO
383 auto_create = Bool(True, config=True,
384 auto_create = Bool(True, config=True,
384 help="whether to create the profile_dir if it doesn't exist")
385 help="whether to create the profile_dir if it doesn't exist")
385 classes = List()
386 classes = List()
386 def _classes_default(self,):
387 def _classes_default(self,):
387 from IPython.parallel.apps import launcher
388 from IPython.parallel.apps import launcher
388 return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers
389 return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers
389
390
390 clean_logs = Bool(True, config=True,
391 clean_logs = Bool(True, config=True,
391 help="whether to cleanup old logs before starting")
392 help="whether to cleanup old logs before starting")
392
393
393 delay = CFloat(1., config=True,
394 delay = CFloat(1., config=True,
394 help="delay (in s) between starting the controller and the engines")
395 help="delay (in s) between starting the controller and the engines")
395
396
396 controller_launcher_class = DottedObjectName('LocalControllerLauncher',
397 controller_launcher_class = DottedObjectName('LocalControllerLauncher',
397 config=True,
398 config=True,
398 help="""The class for launching a Controller. Change this value if you want
399 help="""The class for launching a Controller. Change this value if you want
399 your controller to also be launched by a batch system, such as PBS,SGE,MPIExec,etc.
400 your controller to also be launched by a batch system, such as PBS,SGE,MPIExec,etc.
400
401
401 Each launcher class has its own set of configuration options, for making sure
402 Each launcher class has its own set of configuration options, for making sure
402 it will work in your environment.
403 it will work in your environment.
403
404
404 Examples include:
405 Examples include:
405
406
406 LocalControllerLauncher : start engines locally as subprocesses
407 LocalControllerLauncher : start engines locally as subprocesses
407 MPIExecControllerLauncher : use mpiexec to launch engines in an MPI universe
408 MPIExecControllerLauncher : use mpiexec to launch engines in an MPI universe
408 PBSControllerLauncher : use PBS (qsub) to submit engines to a batch queue
409 PBSControllerLauncher : use PBS (qsub) to submit engines to a batch queue
409 SGEControllerLauncher : use SGE (qsub) to submit engines to a batch queue
410 SGEControllerLauncher : use SGE (qsub) to submit engines to a batch queue
410 SSHControllerLauncher : use SSH to start the controller
411 SSHControllerLauncher : use SSH to start the controller
411 WindowsHPCControllerLauncher : use Windows HPC
412 WindowsHPCControllerLauncher : use Windows HPC
412 """
413 """
413 )
414 )
414 reset = Bool(False, config=True,
415 reset = Bool(False, config=True,
415 help="Whether to reset config files as part of '--create'."
416 help="Whether to reset config files as part of '--create'."
416 )
417 )
417
418
418 # flags = Dict(flags)
419 # flags = Dict(flags)
419 aliases = Dict(start_aliases)
420 aliases = Dict(start_aliases)
420
421
421 def init_launchers(self):
422 def init_launchers(self):
422 self.controller_launcher = self.build_launcher(self.controller_launcher_class, 'Controller')
423 self.controller_launcher = self.build_launcher(self.controller_launcher_class, 'Controller')
423 self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet')
424 self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet')
424 self.controller_launcher.on_stop(self.stop_launchers)
425 self.controller_launcher.on_stop(self.stop_launchers)
425
426
426 def start_controller(self):
427 def start_controller(self):
427 self.controller_launcher.start()
428 self.controller_launcher.start()
428
429
429 def stop_controller(self):
430 def stop_controller(self):
430 # self.log.info("In stop_controller")
431 # self.log.info("In stop_controller")
431 if self.controller_launcher and self.controller_launcher.running:
432 if self.controller_launcher and self.controller_launcher.running:
432 return self.controller_launcher.stop()
433 return self.controller_launcher.stop()
433
434
434 def stop_launchers(self, r=None):
435 def stop_launchers(self, r=None):
435 if not self._stopping:
436 if not self._stopping:
436 self.stop_controller()
437 self.stop_controller()
437 super(IPClusterStart, self).stop_launchers()
438 super(IPClusterStart, self).stop_launchers()
438
439
439 def start(self):
440 def start(self):
440 """Start the app for the start subcommand."""
441 """Start the app for the start subcommand."""
441 # First see if the cluster is already running
442 # First see if the cluster is already running
442 try:
443 try:
443 pid = self.get_pid_from_file()
444 pid = self.get_pid_from_file()
444 except PIDFileError:
445 except PIDFileError:
445 pass
446 pass
446 else:
447 else:
447 if self.check_pid(pid):
448 if self.check_pid(pid):
448 self.log.critical(
449 self.log.critical(
449 'Cluster is already running with [pid=%s]. '
450 'Cluster is already running with [pid=%s]. '
450 'use "ipcluster stop" to stop the cluster.' % pid
451 'use "ipcluster stop" to stop the cluster.' % pid
451 )
452 )
452 # Here I exit with a unusual exit status that other processes
453 # Here I exit with a unusual exit status that other processes
453 # can watch for to learn how I existed.
454 # can watch for to learn how I existed.
454 self.exit(ALREADY_STARTED)
455 self.exit(ALREADY_STARTED)
455 else:
456 else:
456 self.remove_pid_file()
457 self.remove_pid_file()
457
458
458
459
459 # Now log and daemonize
460 # Now log and daemonize
460 self.log.info(
461 self.log.info(
461 'Starting ipcluster with [daemon=%r]' % self.daemonize
462 'Starting ipcluster with [daemon=%r]' % self.daemonize
462 )
463 )
463 # TODO: Get daemonize working on Windows or as a Windows Server.
464 # TODO: Get daemonize working on Windows or as a Windows Server.
464 if self.daemonize:
465 if self.daemonize:
465 if os.name=='posix':
466 if os.name=='posix':
466 daemonize()
467 daemonize()
467
468
468 dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop)
469 dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop)
469 dc.start()
470 dc.start()
470 dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop)
471 dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop)
471 dc.start()
472 dc.start()
472 # Now write the new pid file AFTER our new forked pid is active.
473 # Now write the new pid file AFTER our new forked pid is active.
473 self.write_pid_file()
474 self.write_pid_file()
474 try:
475 try:
475 self.loop.start()
476 self.loop.start()
476 except KeyboardInterrupt:
477 except KeyboardInterrupt:
477 pass
478 pass
478 except zmq.ZMQError as e:
479 except zmq.ZMQError as e:
479 if e.errno == errno.EINTR:
480 if e.errno == errno.EINTR:
480 pass
481 pass
481 else:
482 else:
482 raise
483 raise
483 finally:
484 finally:
484 self.remove_pid_file()
485 self.remove_pid_file()
485
486
486 base='IPython.parallel.apps.ipclusterapp.IPCluster'
487 base='IPython.parallel.apps.ipclusterapp.IPCluster'
487
488
488 class IPClusterApp(Application):
489 class IPClusterApp(Application):
489 name = u'ipcluster'
490 name = u'ipcluster'
490 description = _description
491 description = _description
491 examples = _main_examples
492 examples = _main_examples
492
493
493 subcommands = {
494 subcommands = {
494 'start' : (base+'Start', start_help),
495 'start' : (base+'Start', start_help),
495 'stop' : (base+'Stop', stop_help),
496 'stop' : (base+'Stop', stop_help),
496 'engines' : (base+'Engines', engines_help),
497 'engines' : (base+'Engines', engines_help),
497 }
498 }
498
499
499 # no aliases or flags for parent App
500 # no aliases or flags for parent App
500 aliases = Dict()
501 aliases = Dict()
501 flags = Dict()
502 flags = Dict()
502
503
503 def start(self):
504 def start(self):
504 if self.subapp is None:
505 if self.subapp is None:
505 print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
506 print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
506 print
507 print
507 self.print_description()
508 self.print_description()
508 self.print_subcommands()
509 self.print_subcommands()
509 self.exit(1)
510 self.exit(1)
510 else:
511 else:
511 return self.subapp.start()
512 return self.subapp.start()
512
513
513 def launch_new_instance():
514 def launch_new_instance():
514 """Create and run the IPython cluster."""
515 """Create and run the IPython cluster."""
515 app = IPClusterApp.instance()
516 app = IPClusterApp.instance()
516 app.initialize()
517 app.initialize()
517 app.start()
518 app.start()
518
519
519
520
520 if __name__ == '__main__':
521 if __name__ == '__main__':
521 launch_new_instance()
522 launch_new_instance()
522
523
@@ -1,439 +1,443 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The IPython controller application.
4 The IPython controller application.
5
5
6 Authors:
6 Authors:
7
7
8 * Brian Granger
8 * Brian Granger
9 * MinRK
9 * MinRK
10
10
11 """
11 """
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Copyright (C) 2008-2011 The IPython Development Team
14 # Copyright (C) 2008-2011 The IPython Development Team
15 #
15 #
16 # Distributed under the terms of the BSD License. The full license is in
16 # Distributed under the terms of the BSD License. The full license is in
17 # the file COPYING, distributed as part of this software.
17 # the file COPYING, distributed as part of this software.
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Imports
21 # Imports
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 from __future__ import with_statement
24 from __future__ import with_statement
25
25
26 import os
26 import os
27 import socket
27 import socket
28 import stat
28 import stat
29 import sys
29 import sys
30
30
31 from multiprocessing import Process
31 from multiprocessing import Process
32
32
33 import zmq
33 import zmq
34 from zmq.devices import ProcessMonitoredQueue
34 from zmq.devices import ProcessMonitoredQueue
35 from zmq.log.handlers import PUBHandler
35 from zmq.log.handlers import PUBHandler
36 from zmq.utils import jsonapi as json
36 from zmq.utils import jsonapi as json
37
37
38 from IPython.core.profiledir import ProfileDir
38 from IPython.core.profiledir import ProfileDir
39
39
40 from IPython.parallel.apps.baseapp import (
40 from IPython.parallel.apps.baseapp import (
41 BaseParallelApplication,
41 BaseParallelApplication,
42 base_aliases,
42 base_aliases,
43 base_flags,
43 base_flags,
44 catch_config,
44 )
45 )
45 from IPython.utils.importstring import import_item
46 from IPython.utils.importstring import import_item
46 from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict
47 from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict, TraitError
47
48
48 from IPython.zmq.session import (
49 from IPython.zmq.session import (
49 Session, session_aliases, session_flags, default_secure
50 Session, session_aliases, session_flags, default_secure
50 )
51 )
51
52
52 from IPython.parallel.controller.heartmonitor import HeartMonitor
53 from IPython.parallel.controller.heartmonitor import HeartMonitor
53 from IPython.parallel.controller.hub import HubFactory
54 from IPython.parallel.controller.hub import HubFactory
54 from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
55 from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
55 from IPython.parallel.controller.sqlitedb import SQLiteDB
56 from IPython.parallel.controller.sqlitedb import SQLiteDB
56
57
57 from IPython.parallel.util import signal_children, split_url, asbytes
58 from IPython.parallel.util import signal_children, split_url, asbytes
58
59
59 # conditional import of MongoDB backend class
60 # conditional import of MongoDB backend class
60
61
61 try:
62 try:
62 from IPython.parallel.controller.mongodb import MongoDB
63 from IPython.parallel.controller.mongodb import MongoDB
63 except ImportError:
64 except ImportError:
64 maybe_mongo = []
65 maybe_mongo = []
65 else:
66 else:
66 maybe_mongo = [MongoDB]
67 maybe_mongo = [MongoDB]
67
68
68
69
69 #-----------------------------------------------------------------------------
70 #-----------------------------------------------------------------------------
70 # Module level variables
71 # Module level variables
71 #-----------------------------------------------------------------------------
72 #-----------------------------------------------------------------------------
72
73
73
74
74 #: The default config file name for this application
75 #: The default config file name for this application
75 default_config_file_name = u'ipcontroller_config.py'
76 default_config_file_name = u'ipcontroller_config.py'
76
77
77
78
78 _description = """Start the IPython controller for parallel computing.
79 _description = """Start the IPython controller for parallel computing.
79
80
80 The IPython controller provides a gateway between the IPython engines and
81 The IPython controller provides a gateway between the IPython engines and
81 clients. The controller needs to be started before the engines and can be
82 clients. The controller needs to be started before the engines and can be
82 configured using command line options or using a cluster directory. Cluster
83 configured using command line options or using a cluster directory. Cluster
83 directories contain config, log and security files and are usually located in
84 directories contain config, log and security files and are usually located in
84 your ipython directory and named as "profile_name". See the `profile`
85 your ipython directory and named as "profile_name". See the `profile`
85 and `profile-dir` options for details.
86 and `profile-dir` options for details.
86 """
87 """
87
88
88 _examples = """
89 _examples = """
89 ipcontroller --ip=192.168.0.1 --port=1000 # listen on ip, port for engines
90 ipcontroller --ip=192.168.0.1 --port=1000 # listen on ip, port for engines
90 ipcontroller --scheme=pure # use the pure zeromq scheduler
91 ipcontroller --scheme=pure # use the pure zeromq scheduler
91 """
92 """
92
93
93
94
94 #-----------------------------------------------------------------------------
95 #-----------------------------------------------------------------------------
95 # The main application
96 # The main application
96 #-----------------------------------------------------------------------------
97 #-----------------------------------------------------------------------------
97 flags = {}
98 flags = {}
98 flags.update(base_flags)
99 flags.update(base_flags)
99 flags.update({
100 flags.update({
100 'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
101 'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
101 'Use threads instead of processes for the schedulers'),
102 'Use threads instead of processes for the schedulers'),
102 'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
103 'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
103 'use the SQLiteDB backend'),
104 'use the SQLiteDB backend'),
104 'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
105 'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
105 'use the MongoDB backend'),
106 'use the MongoDB backend'),
106 'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
107 'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
107 'use the in-memory DictDB backend'),
108 'use the in-memory DictDB backend'),
108 'reuse' : ({'IPControllerApp' : {'reuse_files' : True}},
109 'reuse' : ({'IPControllerApp' : {'reuse_files' : True}},
109 'reuse existing json connection files')
110 'reuse existing json connection files')
110 })
111 })
111
112
112 flags.update(session_flags)
113 flags.update(session_flags)
113
114
114 aliases = dict(
115 aliases = dict(
115 ssh = 'IPControllerApp.ssh_server',
116 ssh = 'IPControllerApp.ssh_server',
116 enginessh = 'IPControllerApp.engine_ssh_server',
117 enginessh = 'IPControllerApp.engine_ssh_server',
117 location = 'IPControllerApp.location',
118 location = 'IPControllerApp.location',
118
119
119 url = 'HubFactory.url',
120 url = 'HubFactory.url',
120 ip = 'HubFactory.ip',
121 ip = 'HubFactory.ip',
121 transport = 'HubFactory.transport',
122 transport = 'HubFactory.transport',
122 port = 'HubFactory.regport',
123 port = 'HubFactory.regport',
123
124
124 ping = 'HeartMonitor.period',
125 ping = 'HeartMonitor.period',
125
126
126 scheme = 'TaskScheduler.scheme_name',
127 scheme = 'TaskScheduler.scheme_name',
127 hwm = 'TaskScheduler.hwm',
128 hwm = 'TaskScheduler.hwm',
128 )
129 )
129 aliases.update(base_aliases)
130 aliases.update(base_aliases)
130 aliases.update(session_aliases)
131 aliases.update(session_aliases)
131
132
132
133
133 class IPControllerApp(BaseParallelApplication):
134 class IPControllerApp(BaseParallelApplication):
134
135
135 name = u'ipcontroller'
136 name = u'ipcontroller'
136 description = _description
137 description = _description
137 examples = _examples
138 examples = _examples
138 config_file_name = Unicode(default_config_file_name)
139 config_file_name = Unicode(default_config_file_name)
139 classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo
140 classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo
140
141
141 # change default to True
142 # change default to True
142 auto_create = Bool(True, config=True,
143 auto_create = Bool(True, config=True,
143 help="""Whether to create profile dir if it doesn't exist.""")
144 help="""Whether to create profile dir if it doesn't exist.""")
144
145
145 reuse_files = Bool(False, config=True,
146 reuse_files = Bool(False, config=True,
146 help='Whether to reuse existing json connection files.'
147 help='Whether to reuse existing json connection files.'
147 )
148 )
148 ssh_server = Unicode(u'', config=True,
149 ssh_server = Unicode(u'', config=True,
149 help="""ssh url for clients to use when connecting to the Controller
150 help="""ssh url for clients to use when connecting to the Controller
150 processes. It should be of the form: [user@]server[:port]. The
151 processes. It should be of the form: [user@]server[:port]. The
151 Controller's listening addresses must be accessible from the ssh server""",
152 Controller's listening addresses must be accessible from the ssh server""",
152 )
153 )
153 engine_ssh_server = Unicode(u'', config=True,
154 engine_ssh_server = Unicode(u'', config=True,
154 help="""ssh url for engines to use when connecting to the Controller
155 help="""ssh url for engines to use when connecting to the Controller
155 processes. It should be of the form: [user@]server[:port]. The
156 processes. It should be of the form: [user@]server[:port]. The
156 Controller's listening addresses must be accessible from the ssh server""",
157 Controller's listening addresses must be accessible from the ssh server""",
157 )
158 )
158 location = Unicode(u'', config=True,
159 location = Unicode(u'', config=True,
159 help="""The external IP or domain name of the Controller, used for disambiguating
160 help="""The external IP or domain name of the Controller, used for disambiguating
160 engine and client connections.""",
161 engine and client connections.""",
161 )
162 )
162 import_statements = List([], config=True,
163 import_statements = List([], config=True,
163 help="import statements to be run at startup. Necessary in some environments"
164 help="import statements to be run at startup. Necessary in some environments"
164 )
165 )
165
166
166 use_threads = Bool(False, config=True,
167 use_threads = Bool(False, config=True,
167 help='Use threads instead of processes for the schedulers',
168 help='Use threads instead of processes for the schedulers',
168 )
169 )
169
170
170 engine_json_file = Unicode('ipcontroller-engine.json', config=True,
171 engine_json_file = Unicode('ipcontroller-engine.json', config=True,
171 help="JSON filename where engine connection info will be stored.")
172 help="JSON filename where engine connection info will be stored.")
172 client_json_file = Unicode('ipcontroller-client.json', config=True,
173 client_json_file = Unicode('ipcontroller-client.json', config=True,
173 help="JSON filename where client connection info will be stored.")
174 help="JSON filename where client connection info will be stored.")
174
175
175 def _cluster_id_changed(self, name, old, new):
176 def _cluster_id_changed(self, name, old, new):
176 super(IPControllerApp, self)._cluster_id_changed(name, old, new)
177 super(IPControllerApp, self)._cluster_id_changed(name, old, new)
177 self.engine_json_file = "%s-engine.json" % self.name
178 self.engine_json_file = "%s-engine.json" % self.name
178 self.client_json_file = "%s-client.json" % self.name
179 self.client_json_file = "%s-client.json" % self.name
179
180
180
181
181 # internal
182 # internal
182 children = List()
183 children = List()
183 mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')
184 mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')
184
185
185 def _use_threads_changed(self, name, old, new):
186 def _use_threads_changed(self, name, old, new):
186 self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
187 self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
187
188
188 aliases = Dict(aliases)
189 aliases = Dict(aliases)
189 flags = Dict(flags)
190 flags = Dict(flags)
190
191
191
192
192 def save_connection_dict(self, fname, cdict):
193 def save_connection_dict(self, fname, cdict):
193 """save a connection dict to json file."""
194 """save a connection dict to json file."""
194 c = self.config
195 c = self.config
195 url = cdict['url']
196 url = cdict['url']
196 location = cdict['location']
197 location = cdict['location']
197 if not location:
198 if not location:
198 try:
199 try:
199 proto,ip,port = split_url(url)
200 proto,ip,port = split_url(url)
200 except AssertionError:
201 except AssertionError:
201 pass
202 pass
202 else:
203 else:
203 try:
204 try:
204 location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
205 location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
205 except (socket.gaierror, IndexError):
206 except (socket.gaierror, IndexError):
206 self.log.warn("Could not identify this machine's IP, assuming 127.0.0.1."
207 self.log.warn("Could not identify this machine's IP, assuming 127.0.0.1."
207 " You may need to specify '--location=<external_ip_address>' to help"
208 " You may need to specify '--location=<external_ip_address>' to help"
208 " IPython decide when to connect via loopback.")
209 " IPython decide when to connect via loopback.")
209 location = '127.0.0.1'
210 location = '127.0.0.1'
210 cdict['location'] = location
211 cdict['location'] = location
211 fname = os.path.join(self.profile_dir.security_dir, fname)
212 fname = os.path.join(self.profile_dir.security_dir, fname)
212 with open(fname, 'wb') as f:
213 with open(fname, 'wb') as f:
213 f.write(json.dumps(cdict, indent=2))
214 f.write(json.dumps(cdict, indent=2))
214 os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
215 os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
215
216
216 def load_config_from_json(self):
217 def load_config_from_json(self):
217 """load config from existing json connector files."""
218 """load config from existing json connector files."""
218 c = self.config
219 c = self.config
219 self.log.debug("loading config from JSON")
220 self.log.debug("loading config from JSON")
220 # load from engine config
221 # load from engine config
221 with open(os.path.join(self.profile_dir.security_dir, self.engine_json_file)) as f:
222 with open(os.path.join(self.profile_dir.security_dir, self.engine_json_file)) as f:
222 cfg = json.loads(f.read())
223 cfg = json.loads(f.read())
223 key = c.Session.key = asbytes(cfg['exec_key'])
224 key = c.Session.key = asbytes(cfg['exec_key'])
224 xport,addr = cfg['url'].split('://')
225 xport,addr = cfg['url'].split('://')
225 c.HubFactory.engine_transport = xport
226 c.HubFactory.engine_transport = xport
226 ip,ports = addr.split(':')
227 ip,ports = addr.split(':')
227 c.HubFactory.engine_ip = ip
228 c.HubFactory.engine_ip = ip
228 c.HubFactory.regport = int(ports)
229 c.HubFactory.regport = int(ports)
229 self.location = cfg['location']
230 self.location = cfg['location']
230 if not self.engine_ssh_server:
231 if not self.engine_ssh_server:
231 self.engine_ssh_server = cfg['ssh']
232 self.engine_ssh_server = cfg['ssh']
232 # load client config
233 # load client config
233 with open(os.path.join(self.profile_dir.security_dir, self.client_json_file)) as f:
234 with open(os.path.join(self.profile_dir.security_dir, self.client_json_file)) as f:
234 cfg = json.loads(f.read())
235 cfg = json.loads(f.read())
235 assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
236 assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
236 xport,addr = cfg['url'].split('://')
237 xport,addr = cfg['url'].split('://')
237 c.HubFactory.client_transport = xport
238 c.HubFactory.client_transport = xport
238 ip,ports = addr.split(':')
239 ip,ports = addr.split(':')
239 c.HubFactory.client_ip = ip
240 c.HubFactory.client_ip = ip
240 if not self.ssh_server:
241 if not self.ssh_server:
241 self.ssh_server = cfg['ssh']
242 self.ssh_server = cfg['ssh']
242 assert int(ports) == c.HubFactory.regport, "regport mismatch"
243 assert int(ports) == c.HubFactory.regport, "regport mismatch"
243
244
244 def load_secondary_config(self):
245 def load_secondary_config(self):
245 """secondary config, loading from JSON and setting defaults"""
246 """secondary config, loading from JSON and setting defaults"""
246 if self.reuse_files:
247 if self.reuse_files:
247 try:
248 try:
248 self.load_config_from_json()
249 self.load_config_from_json()
249 except (AssertionError,IOError) as e:
250 except (AssertionError,IOError) as e:
250 self.log.error("Could not load config from JSON: %s" % e)
251 self.log.error("Could not load config from JSON: %s" % e)
251 self.reuse_files=False
252 self.reuse_files=False
252 # switch Session.key default to secure
253 # switch Session.key default to secure
253 default_secure(self.config)
254 default_secure(self.config)
254 self.log.debug("Config changed")
255 self.log.debug("Config changed")
255 self.log.debug(repr(self.config))
256 self.log.debug(repr(self.config))
256
257
257 def init_hub(self):
258 def init_hub(self):
258 c = self.config
259 c = self.config
259
260
260 self.do_import_statements()
261 self.do_import_statements()
261
262
262 try:
263 try:
263 self.factory = HubFactory(config=c, log=self.log)
264 self.factory = HubFactory(config=c, log=self.log)
264 # self.start_logging()
265 # self.start_logging()
265 self.factory.init_hub()
266 self.factory.init_hub()
266 except:
267 except TraitError:
268 raise
269 except Exception:
267 self.log.error("Couldn't construct the Controller", exc_info=True)
270 self.log.error("Couldn't construct the Controller", exc_info=True)
268 self.exit(1)
271 self.exit(1)
269
272
270 if not self.reuse_files:
273 if not self.reuse_files:
271 # save to new json config files
274 # save to new json config files
272 f = self.factory
275 f = self.factory
273 cdict = {'exec_key' : f.session.key,
276 cdict = {'exec_key' : f.session.key,
274 'ssh' : self.ssh_server,
277 'ssh' : self.ssh_server,
275 'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
278 'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
276 'location' : self.location
279 'location' : self.location
277 }
280 }
278 self.save_connection_dict(self.client_json_file, cdict)
281 self.save_connection_dict(self.client_json_file, cdict)
279 edict = cdict
282 edict = cdict
280 edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
283 edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
281 edict['ssh'] = self.engine_ssh_server
284 edict['ssh'] = self.engine_ssh_server
282 self.save_connection_dict(self.engine_json_file, edict)
285 self.save_connection_dict(self.engine_json_file, edict)
283
286
284 #
287 #
285 def init_schedulers(self):
288 def init_schedulers(self):
286 children = self.children
289 children = self.children
287 mq = import_item(str(self.mq_class))
290 mq = import_item(str(self.mq_class))
288
291
289 hub = self.factory
292 hub = self.factory
290 # maybe_inproc = 'inproc://monitor' if self.use_threads else self.monitor_url
293 # maybe_inproc = 'inproc://monitor' if self.use_threads else self.monitor_url
291 # IOPub relay (in a Process)
294 # IOPub relay (in a Process)
292 q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub')
295 q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub')
293 q.bind_in(hub.client_info['iopub'])
296 q.bind_in(hub.client_info['iopub'])
294 q.bind_out(hub.engine_info['iopub'])
297 q.bind_out(hub.engine_info['iopub'])
295 q.setsockopt_out(zmq.SUBSCRIBE, b'')
298 q.setsockopt_out(zmq.SUBSCRIBE, b'')
296 q.connect_mon(hub.monitor_url)
299 q.connect_mon(hub.monitor_url)
297 q.daemon=True
300 q.daemon=True
298 children.append(q)
301 children.append(q)
299
302
300 # Multiplexer Queue (in a Process)
303 # Multiplexer Queue (in a Process)
301 q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out')
304 q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out')
302 q.bind_in(hub.client_info['mux'])
305 q.bind_in(hub.client_info['mux'])
303 q.setsockopt_in(zmq.IDENTITY, b'mux')
306 q.setsockopt_in(zmq.IDENTITY, b'mux')
304 q.bind_out(hub.engine_info['mux'])
307 q.bind_out(hub.engine_info['mux'])
305 q.connect_mon(hub.monitor_url)
308 q.connect_mon(hub.monitor_url)
306 q.daemon=True
309 q.daemon=True
307 children.append(q)
310 children.append(q)
308
311
309 # Control Queue (in a Process)
312 # Control Queue (in a Process)
310 q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol')
313 q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol')
311 q.bind_in(hub.client_info['control'])
314 q.bind_in(hub.client_info['control'])
312 q.setsockopt_in(zmq.IDENTITY, b'control')
315 q.setsockopt_in(zmq.IDENTITY, b'control')
313 q.bind_out(hub.engine_info['control'])
316 q.bind_out(hub.engine_info['control'])
314 q.connect_mon(hub.monitor_url)
317 q.connect_mon(hub.monitor_url)
315 q.daemon=True
318 q.daemon=True
316 children.append(q)
319 children.append(q)
317 try:
320 try:
318 scheme = self.config.TaskScheduler.scheme_name
321 scheme = self.config.TaskScheduler.scheme_name
319 except AttributeError:
322 except AttributeError:
320 scheme = TaskScheduler.scheme_name.get_default_value()
323 scheme = TaskScheduler.scheme_name.get_default_value()
321 # Task Queue (in a Process)
324 # Task Queue (in a Process)
322 if scheme == 'pure':
325 if scheme == 'pure':
323 self.log.warn("task::using pure XREQ Task scheduler")
326 self.log.warn("task::using pure XREQ Task scheduler")
324 q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask')
327 q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask')
325 # q.setsockopt_out(zmq.HWM, hub.hwm)
328 # q.setsockopt_out(zmq.HWM, hub.hwm)
326 q.bind_in(hub.client_info['task'][1])
329 q.bind_in(hub.client_info['task'][1])
327 q.setsockopt_in(zmq.IDENTITY, b'task')
330 q.setsockopt_in(zmq.IDENTITY, b'task')
328 q.bind_out(hub.engine_info['task'])
331 q.bind_out(hub.engine_info['task'])
329 q.connect_mon(hub.monitor_url)
332 q.connect_mon(hub.monitor_url)
330 q.daemon=True
333 q.daemon=True
331 children.append(q)
334 children.append(q)
332 elif scheme == 'none':
335 elif scheme == 'none':
333 self.log.warn("task::using no Task scheduler")
336 self.log.warn("task::using no Task scheduler")
334
337
335 else:
338 else:
336 self.log.info("task::using Python %s Task scheduler"%scheme)
339 self.log.info("task::using Python %s Task scheduler"%scheme)
337 sargs = (hub.client_info['task'][1], hub.engine_info['task'],
340 sargs = (hub.client_info['task'][1], hub.engine_info['task'],
338 hub.monitor_url, hub.client_info['notification'])
341 hub.monitor_url, hub.client_info['notification'])
339 kwargs = dict(logname='scheduler', loglevel=self.log_level,
342 kwargs = dict(logname='scheduler', loglevel=self.log_level,
340 log_url = self.log_url, config=dict(self.config))
343 log_url = self.log_url, config=dict(self.config))
341 if 'Process' in self.mq_class:
344 if 'Process' in self.mq_class:
342 # run the Python scheduler in a Process
345 # run the Python scheduler in a Process
343 q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
346 q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
344 q.daemon=True
347 q.daemon=True
345 children.append(q)
348 children.append(q)
346 else:
349 else:
347 # single-threaded Controller
350 # single-threaded Controller
348 kwargs['in_thread'] = True
351 kwargs['in_thread'] = True
349 launch_scheduler(*sargs, **kwargs)
352 launch_scheduler(*sargs, **kwargs)
350
353
351
354
352 def save_urls(self):
355 def save_urls(self):
353 """save the registration urls to files."""
356 """save the registration urls to files."""
354 c = self.config
357 c = self.config
355
358
356 sec_dir = self.profile_dir.security_dir
359 sec_dir = self.profile_dir.security_dir
357 cf = self.factory
360 cf = self.factory
358
361
359 with open(os.path.join(sec_dir, 'ipcontroller-engine.url'), 'w') as f:
362 with open(os.path.join(sec_dir, 'ipcontroller-engine.url'), 'w') as f:
360 f.write("%s://%s:%s"%(cf.engine_transport, cf.engine_ip, cf.regport))
363 f.write("%s://%s:%s"%(cf.engine_transport, cf.engine_ip, cf.regport))
361
364
362 with open(os.path.join(sec_dir, 'ipcontroller-client.url'), 'w') as f:
365 with open(os.path.join(sec_dir, 'ipcontroller-client.url'), 'w') as f:
363 f.write("%s://%s:%s"%(cf.client_transport, cf.client_ip, cf.regport))
366 f.write("%s://%s:%s"%(cf.client_transport, cf.client_ip, cf.regport))
364
367
365
368
366 def do_import_statements(self):
369 def do_import_statements(self):
367 statements = self.import_statements
370 statements = self.import_statements
368 for s in statements:
371 for s in statements:
369 try:
372 try:
370 self.log.msg("Executing statement: '%s'" % s)
373 self.log.msg("Executing statement: '%s'" % s)
371 exec s in globals(), locals()
374 exec s in globals(), locals()
372 except:
375 except:
373 self.log.msg("Error running statement: %s" % s)
376 self.log.msg("Error running statement: %s" % s)
374
377
375 def forward_logging(self):
378 def forward_logging(self):
376 if self.log_url:
379 if self.log_url:
377 self.log.info("Forwarding logging to %s"%self.log_url)
380 self.log.info("Forwarding logging to %s"%self.log_url)
378 context = zmq.Context.instance()
381 context = zmq.Context.instance()
379 lsock = context.socket(zmq.PUB)
382 lsock = context.socket(zmq.PUB)
380 lsock.connect(self.log_url)
383 lsock.connect(self.log_url)
381 handler = PUBHandler(lsock)
384 handler = PUBHandler(lsock)
382 self.log.removeHandler(self._log_handler)
385 self.log.removeHandler(self._log_handler)
383 handler.root_topic = 'controller'
386 handler.root_topic = 'controller'
384 handler.setLevel(self.log_level)
387 handler.setLevel(self.log_level)
385 self.log.addHandler(handler)
388 self.log.addHandler(handler)
386 self._log_handler = handler
389 self._log_handler = handler
387
390
391 @catch_config
388 def initialize(self, argv=None):
392 def initialize(self, argv=None):
389 super(IPControllerApp, self).initialize(argv)
393 super(IPControllerApp, self).initialize(argv)
390 self.forward_logging()
394 self.forward_logging()
391 self.load_secondary_config()
395 self.load_secondary_config()
392 self.init_hub()
396 self.init_hub()
393 self.init_schedulers()
397 self.init_schedulers()
394
398
395 def start(self):
399 def start(self):
396 # Start the subprocesses:
400 # Start the subprocesses:
397 self.factory.start()
401 self.factory.start()
398 child_procs = []
402 child_procs = []
399 for child in self.children:
403 for child in self.children:
400 child.start()
404 child.start()
401 if isinstance(child, ProcessMonitoredQueue):
405 if isinstance(child, ProcessMonitoredQueue):
402 child_procs.append(child.launcher)
406 child_procs.append(child.launcher)
403 elif isinstance(child, Process):
407 elif isinstance(child, Process):
404 child_procs.append(child)
408 child_procs.append(child)
405 if child_procs:
409 if child_procs:
406 signal_children(child_procs)
410 signal_children(child_procs)
407
411
408 self.write_pid_file(overwrite=True)
412 self.write_pid_file(overwrite=True)
409
413
410 try:
414 try:
411 self.factory.loop.start()
415 self.factory.loop.start()
412 except KeyboardInterrupt:
416 except KeyboardInterrupt:
413 self.log.critical("Interrupted, Exiting...\n")
417 self.log.critical("Interrupted, Exiting...\n")
414
418
415
419
416
420
417 def launch_new_instance():
421 def launch_new_instance():
418 """Create and run the IPython controller"""
422 """Create and run the IPython controller"""
419 if sys.platform == 'win32':
423 if sys.platform == 'win32':
420 # make sure we don't get called from a multiprocessing subprocess
424 # make sure we don't get called from a multiprocessing subprocess
421 # this can result in infinite Controllers being started on Windows
425 # this can result in infinite Controllers being started on Windows
422 # which doesn't have a proper fork, so multiprocessing is wonky
426 # which doesn't have a proper fork, so multiprocessing is wonky
423
427
424 # this only comes up when IPython has been installed using vanilla
428 # this only comes up when IPython has been installed using vanilla
425 # setuptools, and *not* distribute.
429 # setuptools, and *not* distribute.
426 import multiprocessing
430 import multiprocessing
427 p = multiprocessing.current_process()
431 p = multiprocessing.current_process()
428 # the main process has name 'MainProcess'
432 # the main process has name 'MainProcess'
429 # subprocesses will have names like 'Process-1'
433 # subprocesses will have names like 'Process-1'
430 if p.name != 'MainProcess':
434 if p.name != 'MainProcess':
431 # we are a subprocess, don't start another Controller!
435 # we are a subprocess, don't start another Controller!
432 return
436 return
433 app = IPControllerApp.instance()
437 app = IPControllerApp.instance()
434 app.initialize()
438 app.initialize()
435 app.start()
439 app.start()
436
440
437
441
438 if __name__ == '__main__':
442 if __name__ == '__main__':
439 launch_new_instance()
443 launch_new_instance()
@@ -1,344 +1,346 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The IPython engine application
4 The IPython engine application
5
5
6 Authors:
6 Authors:
7
7
8 * Brian Granger
8 * Brian Granger
9 * MinRK
9 * MinRK
10
10
11 """
11 """
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Copyright (C) 2008-2011 The IPython Development Team
14 # Copyright (C) 2008-2011 The IPython Development Team
15 #
15 #
16 # Distributed under the terms of the BSD License. The full license is in
16 # Distributed under the terms of the BSD License. The full license is in
17 # the file COPYING, distributed as part of this software.
17 # the file COPYING, distributed as part of this software.
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Imports
21 # Imports
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 import json
24 import json
25 import os
25 import os
26 import sys
26 import sys
27 import time
27 import time
28
28
29 import zmq
29 import zmq
30 from zmq.eventloop import ioloop
30 from zmq.eventloop import ioloop
31
31
32 from IPython.core.profiledir import ProfileDir
32 from IPython.core.profiledir import ProfileDir
33 from IPython.parallel.apps.baseapp import (
33 from IPython.parallel.apps.baseapp import (
34 BaseParallelApplication,
34 BaseParallelApplication,
35 base_aliases,
35 base_aliases,
36 base_flags,
36 base_flags,
37 catch_config,
37 )
38 )
38 from IPython.zmq.log import EnginePUBHandler
39 from IPython.zmq.log import EnginePUBHandler
39 from IPython.zmq.session import (
40 from IPython.zmq.session import (
40 Session, session_aliases, session_flags
41 Session, session_aliases, session_flags
41 )
42 )
42
43
43 from IPython.config.configurable import Configurable
44 from IPython.config.configurable import Configurable
44
45
45 from IPython.parallel.engine.engine import EngineFactory
46 from IPython.parallel.engine.engine import EngineFactory
46 from IPython.parallel.engine.streamkernel import Kernel
47 from IPython.parallel.engine.streamkernel import Kernel
47 from IPython.parallel.util import disambiguate_url, asbytes
48 from IPython.parallel.util import disambiguate_url, asbytes
48
49
49 from IPython.utils.importstring import import_item
50 from IPython.utils.importstring import import_item
50 from IPython.utils.traitlets import Bool, Unicode, Dict, List, Float
51 from IPython.utils.traitlets import Bool, Unicode, Dict, List, Float
51
52
52
53
53 #-----------------------------------------------------------------------------
54 #-----------------------------------------------------------------------------
54 # Module level variables
55 # Module level variables
55 #-----------------------------------------------------------------------------
56 #-----------------------------------------------------------------------------
56
57
57 #: The default config file name for this application
58 #: The default config file name for this application
58 default_config_file_name = u'ipengine_config.py'
59 default_config_file_name = u'ipengine_config.py'
59
60
60 _description = """Start an IPython engine for parallel computing.
61 _description = """Start an IPython engine for parallel computing.
61
62
62 IPython engines run in parallel and perform computations on behalf of a client
63 IPython engines run in parallel and perform computations on behalf of a client
63 and controller. A controller needs to be started before the engines. The
64 and controller. A controller needs to be started before the engines. The
64 engine can be configured using command line options or using a cluster
65 engine can be configured using command line options or using a cluster
65 directory. Cluster directories contain config, log and security files and are
66 directory. Cluster directories contain config, log and security files and are
66 usually located in your ipython directory and named as "profile_name".
67 usually located in your ipython directory and named as "profile_name".
67 See the `profile` and `profile-dir` options for details.
68 See the `profile` and `profile-dir` options for details.
68 """
69 """
69
70
70 _examples = """
71 _examples = """
71 ipengine --ip=192.168.0.1 --port=1000 # connect to hub at ip and port
72 ipengine --ip=192.168.0.1 --port=1000 # connect to hub at ip and port
72 ipengine --log-to-file --log-level=DEBUG # log to a file with DEBUG verbosity
73 ipengine --log-to-file --log-level=DEBUG # log to a file with DEBUG verbosity
73 """
74 """
74
75
75 #-----------------------------------------------------------------------------
76 #-----------------------------------------------------------------------------
76 # MPI configuration
77 # MPI configuration
77 #-----------------------------------------------------------------------------
78 #-----------------------------------------------------------------------------
78
79
79 mpi4py_init = """from mpi4py import MPI as mpi
80 mpi4py_init = """from mpi4py import MPI as mpi
80 mpi.size = mpi.COMM_WORLD.Get_size()
81 mpi.size = mpi.COMM_WORLD.Get_size()
81 mpi.rank = mpi.COMM_WORLD.Get_rank()
82 mpi.rank = mpi.COMM_WORLD.Get_rank()
82 """
83 """
83
84
84
85
85 pytrilinos_init = """from PyTrilinos import Epetra
86 pytrilinos_init = """from PyTrilinos import Epetra
86 class SimpleStruct:
87 class SimpleStruct:
87 pass
88 pass
88 mpi = SimpleStruct()
89 mpi = SimpleStruct()
89 mpi.rank = 0
90 mpi.rank = 0
90 mpi.size = 0
91 mpi.size = 0
91 """
92 """
92
93
93 class MPI(Configurable):
94 class MPI(Configurable):
94 """Configurable for MPI initialization"""
95 """Configurable for MPI initialization"""
95 use = Unicode('', config=True,
96 use = Unicode('', config=True,
96 help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).'
97 help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).'
97 )
98 )
98
99
99 def _use_changed(self, name, old, new):
100 def _use_changed(self, name, old, new):
100 # load default init script if it's not set
101 # load default init script if it's not set
101 if not self.init_script:
102 if not self.init_script:
102 self.init_script = self.default_inits.get(new, '')
103 self.init_script = self.default_inits.get(new, '')
103
104
104 init_script = Unicode('', config=True,
105 init_script = Unicode('', config=True,
105 help="Initialization code for MPI")
106 help="Initialization code for MPI")
106
107
107 default_inits = Dict({'mpi4py' : mpi4py_init, 'pytrilinos':pytrilinos_init},
108 default_inits = Dict({'mpi4py' : mpi4py_init, 'pytrilinos':pytrilinos_init},
108 config=True)
109 config=True)
109
110
110
111
111 #-----------------------------------------------------------------------------
112 #-----------------------------------------------------------------------------
112 # Main application
113 # Main application
113 #-----------------------------------------------------------------------------
114 #-----------------------------------------------------------------------------
114 aliases = dict(
115 aliases = dict(
115 file = 'IPEngineApp.url_file',
116 file = 'IPEngineApp.url_file',
116 c = 'IPEngineApp.startup_command',
117 c = 'IPEngineApp.startup_command',
117 s = 'IPEngineApp.startup_script',
118 s = 'IPEngineApp.startup_script',
118
119
119 url = 'EngineFactory.url',
120 url = 'EngineFactory.url',
120 ssh = 'EngineFactory.sshserver',
121 ssh = 'EngineFactory.sshserver',
121 sshkey = 'EngineFactory.sshkey',
122 sshkey = 'EngineFactory.sshkey',
122 ip = 'EngineFactory.ip',
123 ip = 'EngineFactory.ip',
123 transport = 'EngineFactory.transport',
124 transport = 'EngineFactory.transport',
124 port = 'EngineFactory.regport',
125 port = 'EngineFactory.regport',
125 location = 'EngineFactory.location',
126 location = 'EngineFactory.location',
126
127
127 timeout = 'EngineFactory.timeout',
128 timeout = 'EngineFactory.timeout',
128
129
129 mpi = 'MPI.use',
130 mpi = 'MPI.use',
130
131
131 )
132 )
132 aliases.update(base_aliases)
133 aliases.update(base_aliases)
133 aliases.update(session_aliases)
134 aliases.update(session_aliases)
134 flags = {}
135 flags = {}
135 flags.update(base_flags)
136 flags.update(base_flags)
136 flags.update(session_flags)
137 flags.update(session_flags)
137
138
138 class IPEngineApp(BaseParallelApplication):
139 class IPEngineApp(BaseParallelApplication):
139
140
140 name = 'ipengine'
141 name = 'ipengine'
141 description = _description
142 description = _description
142 examples = _examples
143 examples = _examples
143 config_file_name = Unicode(default_config_file_name)
144 config_file_name = Unicode(default_config_file_name)
144 classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI])
145 classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI])
145
146
146 startup_script = Unicode(u'', config=True,
147 startup_script = Unicode(u'', config=True,
147 help='specify a script to be run at startup')
148 help='specify a script to be run at startup')
148 startup_command = Unicode('', config=True,
149 startup_command = Unicode('', config=True,
149 help='specify a command to be run at startup')
150 help='specify a command to be run at startup')
150
151
151 url_file = Unicode(u'', config=True,
152 url_file = Unicode(u'', config=True,
152 help="""The full location of the file containing the connection information for
153 help="""The full location of the file containing the connection information for
153 the controller. If this is not given, the file must be in the
154 the controller. If this is not given, the file must be in the
154 security directory of the cluster directory. This location is
155 security directory of the cluster directory. This location is
155 resolved using the `profile` or `profile_dir` options.""",
156 resolved using the `profile` or `profile_dir` options.""",
156 )
157 )
157 wait_for_url_file = Float(5, config=True,
158 wait_for_url_file = Float(5, config=True,
158 help="""The maximum number of seconds to wait for url_file to exist.
159 help="""The maximum number of seconds to wait for url_file to exist.
159 This is useful for batch-systems and shared-filesystems where the
160 This is useful for batch-systems and shared-filesystems where the
160 controller and engine are started at the same time and it
161 controller and engine are started at the same time and it
161 may take a moment for the controller to write the connector files.""")
162 may take a moment for the controller to write the connector files.""")
162
163
163 url_file_name = Unicode(u'ipcontroller-engine.json', config=True)
164 url_file_name = Unicode(u'ipcontroller-engine.json', config=True)
164
165
165 def _cluster_id_changed(self, name, old, new):
166 def _cluster_id_changed(self, name, old, new):
166 if new:
167 if new:
167 base = 'ipcontroller-%s' % new
168 base = 'ipcontroller-%s' % new
168 else:
169 else:
169 base = 'ipcontroller'
170 base = 'ipcontroller'
170 self.url_file_name = "%s-engine.json" % base
171 self.url_file_name = "%s-engine.json" % base
171
172
172 log_url = Unicode('', config=True,
173 log_url = Unicode('', config=True,
173 help="""The URL for the iploggerapp instance, for forwarding
174 help="""The URL for the iploggerapp instance, for forwarding
174 logging to a central location.""")
175 logging to a central location.""")
175
176
176 aliases = Dict(aliases)
177 aliases = Dict(aliases)
177 flags = Dict(flags)
178 flags = Dict(flags)
178
179
179 # def find_key_file(self):
180 # def find_key_file(self):
180 # """Set the key file.
181 # """Set the key file.
181 #
182 #
182 # Here we don't try to actually see if it exists for is valid as that
183 # Here we don't try to actually see if it exists for is valid as that
183 # is hadled by the connection logic.
184 # is hadled by the connection logic.
184 # """
185 # """
185 # config = self.master_config
186 # config = self.master_config
186 # # Find the actual controller key file
187 # # Find the actual controller key file
187 # if not config.Global.key_file:
188 # if not config.Global.key_file:
188 # try_this = os.path.join(
189 # try_this = os.path.join(
189 # config.Global.profile_dir,
190 # config.Global.profile_dir,
190 # config.Global.security_dir,
191 # config.Global.security_dir,
191 # config.Global.key_file_name
192 # config.Global.key_file_name
192 # )
193 # )
193 # config.Global.key_file = try_this
194 # config.Global.key_file = try_this
194
195
195 def find_url_file(self):
196 def find_url_file(self):
196 """Set the url file.
197 """Set the url file.
197
198
198 Here we don't try to actually see if it exists for is valid as that
199 Here we don't try to actually see if it exists for is valid as that
199 is hadled by the connection logic.
200 is hadled by the connection logic.
200 """
201 """
201 config = self.config
202 config = self.config
202 # Find the actual controller key file
203 # Find the actual controller key file
203 if not self.url_file:
204 if not self.url_file:
204 self.url_file = os.path.join(
205 self.url_file = os.path.join(
205 self.profile_dir.security_dir,
206 self.profile_dir.security_dir,
206 self.url_file_name
207 self.url_file_name
207 )
208 )
208
209
209 def load_connector_file(self):
210 def load_connector_file(self):
210 """load config from a JSON connector file,
211 """load config from a JSON connector file,
211 at a *lower* priority than command-line/config files.
212 at a *lower* priority than command-line/config files.
212 """
213 """
213
214
214 self.log.info("Loading url_file %r"%self.url_file)
215 self.log.info("Loading url_file %r"%self.url_file)
215 config = self.config
216 config = self.config
216
217
217 with open(self.url_file) as f:
218 with open(self.url_file) as f:
218 d = json.loads(f.read())
219 d = json.loads(f.read())
219
220
220 if 'exec_key' in d:
221 if 'exec_key' in d:
221 config.Session.key = asbytes(d['exec_key'])
222 config.Session.key = asbytes(d['exec_key'])
222
223
223 try:
224 try:
224 config.EngineFactory.location
225 config.EngineFactory.location
225 except AttributeError:
226 except AttributeError:
226 config.EngineFactory.location = d['location']
227 config.EngineFactory.location = d['location']
227
228
228 d['url'] = disambiguate_url(d['url'], config.EngineFactory.location)
229 d['url'] = disambiguate_url(d['url'], config.EngineFactory.location)
229 try:
230 try:
230 config.EngineFactory.url
231 config.EngineFactory.url
231 except AttributeError:
232 except AttributeError:
232 config.EngineFactory.url = d['url']
233 config.EngineFactory.url = d['url']
233
234
234 try:
235 try:
235 config.EngineFactory.sshserver
236 config.EngineFactory.sshserver
236 except AttributeError:
237 except AttributeError:
237 config.EngineFactory.sshserver = d['ssh']
238 config.EngineFactory.sshserver = d['ssh']
238
239
239 def init_engine(self):
240 def init_engine(self):
240 # This is the working dir by now.
241 # This is the working dir by now.
241 sys.path.insert(0, '')
242 sys.path.insert(0, '')
242 config = self.config
243 config = self.config
243 # print config
244 # print config
244 self.find_url_file()
245 self.find_url_file()
245
246
246 # was the url manually specified?
247 # was the url manually specified?
247 keys = set(self.config.EngineFactory.keys())
248 keys = set(self.config.EngineFactory.keys())
248 keys = keys.union(set(self.config.RegistrationFactory.keys()))
249 keys = keys.union(set(self.config.RegistrationFactory.keys()))
249
250
250 if keys.intersection(set(['ip', 'url', 'port'])):
251 if keys.intersection(set(['ip', 'url', 'port'])):
251 # Connection info was specified, don't wait for the file
252 # Connection info was specified, don't wait for the file
252 url_specified = True
253 url_specified = True
253 self.wait_for_url_file = 0
254 self.wait_for_url_file = 0
254 else:
255 else:
255 url_specified = False
256 url_specified = False
256
257
257 if self.wait_for_url_file and not os.path.exists(self.url_file):
258 if self.wait_for_url_file and not os.path.exists(self.url_file):
258 self.log.warn("url_file %r not found"%self.url_file)
259 self.log.warn("url_file %r not found"%self.url_file)
259 self.log.warn("Waiting up to %.1f seconds for it to arrive."%self.wait_for_url_file)
260 self.log.warn("Waiting up to %.1f seconds for it to arrive."%self.wait_for_url_file)
260 tic = time.time()
261 tic = time.time()
261 while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file):
262 while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file):
262 # wait for url_file to exist, for up to 10 seconds
263 # wait for url_file to exist, for up to 10 seconds
263 time.sleep(0.1)
264 time.sleep(0.1)
264
265
265 if os.path.exists(self.url_file):
266 if os.path.exists(self.url_file):
266 self.load_connector_file()
267 self.load_connector_file()
267 elif not url_specified:
268 elif not url_specified:
268 self.log.critical("Fatal: url file never arrived: %s"%self.url_file)
269 self.log.critical("Fatal: url file never arrived: %s"%self.url_file)
269 self.exit(1)
270 self.exit(1)
270
271
271
272
272 try:
273 try:
273 exec_lines = config.Kernel.exec_lines
274 exec_lines = config.Kernel.exec_lines
274 except AttributeError:
275 except AttributeError:
275 config.Kernel.exec_lines = []
276 config.Kernel.exec_lines = []
276 exec_lines = config.Kernel.exec_lines
277 exec_lines = config.Kernel.exec_lines
277
278
278 if self.startup_script:
279 if self.startup_script:
279 enc = sys.getfilesystemencoding() or 'utf8'
280 enc = sys.getfilesystemencoding() or 'utf8'
280 cmd="execfile(%r)"%self.startup_script.encode(enc)
281 cmd="execfile(%r)"%self.startup_script.encode(enc)
281 exec_lines.append(cmd)
282 exec_lines.append(cmd)
282 if self.startup_command:
283 if self.startup_command:
283 exec_lines.append(self.startup_command)
284 exec_lines.append(self.startup_command)
284
285
285 # Create the underlying shell class and Engine
286 # Create the underlying shell class and Engine
286 # shell_class = import_item(self.master_config.Global.shell_class)
287 # shell_class = import_item(self.master_config.Global.shell_class)
287 # print self.config
288 # print self.config
288 try:
289 try:
289 self.engine = EngineFactory(config=config, log=self.log)
290 self.engine = EngineFactory(config=config, log=self.log)
290 except:
291 except:
291 self.log.error("Couldn't start the Engine", exc_info=True)
292 self.log.error("Couldn't start the Engine", exc_info=True)
292 self.exit(1)
293 self.exit(1)
293
294
294 def forward_logging(self):
295 def forward_logging(self):
295 if self.log_url:
296 if self.log_url:
296 self.log.info("Forwarding logging to %s"%self.log_url)
297 self.log.info("Forwarding logging to %s"%self.log_url)
297 context = self.engine.context
298 context = self.engine.context
298 lsock = context.socket(zmq.PUB)
299 lsock = context.socket(zmq.PUB)
299 lsock.connect(self.log_url)
300 lsock.connect(self.log_url)
300 self.log.removeHandler(self._log_handler)
301 self.log.removeHandler(self._log_handler)
301 handler = EnginePUBHandler(self.engine, lsock)
302 handler = EnginePUBHandler(self.engine, lsock)
302 handler.setLevel(self.log_level)
303 handler.setLevel(self.log_level)
303 self.log.addHandler(handler)
304 self.log.addHandler(handler)
304 self._log_handler = handler
305 self._log_handler = handler
305
306
306 def init_mpi(self):
307 def init_mpi(self):
307 global mpi
308 global mpi
308 self.mpi = MPI(config=self.config)
309 self.mpi = MPI(config=self.config)
309
310
310 mpi_import_statement = self.mpi.init_script
311 mpi_import_statement = self.mpi.init_script
311 if mpi_import_statement:
312 if mpi_import_statement:
312 try:
313 try:
313 self.log.info("Initializing MPI:")
314 self.log.info("Initializing MPI:")
314 self.log.info(mpi_import_statement)
315 self.log.info(mpi_import_statement)
315 exec mpi_import_statement in globals()
316 exec mpi_import_statement in globals()
316 except:
317 except:
317 mpi = None
318 mpi = None
318 else:
319 else:
319 mpi = None
320 mpi = None
320
321
322 @catch_config
321 def initialize(self, argv=None):
323 def initialize(self, argv=None):
322 super(IPEngineApp, self).initialize(argv)
324 super(IPEngineApp, self).initialize(argv)
323 self.init_mpi()
325 self.init_mpi()
324 self.init_engine()
326 self.init_engine()
325 self.forward_logging()
327 self.forward_logging()
326
328
327 def start(self):
329 def start(self):
328 self.engine.start()
330 self.engine.start()
329 try:
331 try:
330 self.engine.loop.start()
332 self.engine.loop.start()
331 except KeyboardInterrupt:
333 except KeyboardInterrupt:
332 self.log.critical("Engine Interrupted, shutting down...\n")
334 self.log.critical("Engine Interrupted, shutting down...\n")
333
335
334
336
335 def launch_new_instance():
337 def launch_new_instance():
336 """Create and run the IPython engine"""
338 """Create and run the IPython engine"""
337 app = IPEngineApp.instance()
339 app = IPEngineApp.instance()
338 app.initialize()
340 app.initialize()
339 app.start()
341 app.start()
340
342
341
343
342 if __name__ == '__main__':
344 if __name__ == '__main__':
343 launch_new_instance()
345 launch_new_instance()
344
346
@@ -1,101 +1,103 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 A simple IPython logger application
4 A simple IPython logger application
5
5
6 Authors:
6 Authors:
7
7
8 * MinRK
8 * MinRK
9
9
10 """
10 """
11
11
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13 # Copyright (C) 2011 The IPython Development Team
13 # Copyright (C) 2011 The IPython Development Team
14 #
14 #
15 # Distributed under the terms of the BSD License. The full license is in
15 # Distributed under the terms of the BSD License. The full license is in
16 # the file COPYING, distributed as part of this software.
16 # the file COPYING, distributed as part of this software.
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20 # Imports
20 # Imports
21 #-----------------------------------------------------------------------------
21 #-----------------------------------------------------------------------------
22
22
23 import os
23 import os
24 import sys
24 import sys
25
25
26 import zmq
26 import zmq
27
27
28 from IPython.core.profiledir import ProfileDir
28 from IPython.core.profiledir import ProfileDir
29 from IPython.utils.traitlets import Bool, Dict, Unicode
29 from IPython.utils.traitlets import Bool, Dict, Unicode
30
30
31 from IPython.parallel.apps.baseapp import (
31 from IPython.parallel.apps.baseapp import (
32 BaseParallelApplication,
32 BaseParallelApplication,
33 base_aliases
33 base_aliases,
34 catch_config,
34 )
35 )
35 from IPython.parallel.apps.logwatcher import LogWatcher
36 from IPython.parallel.apps.logwatcher import LogWatcher
36
37
37 #-----------------------------------------------------------------------------
38 #-----------------------------------------------------------------------------
38 # Module level variables
39 # Module level variables
39 #-----------------------------------------------------------------------------
40 #-----------------------------------------------------------------------------
40
41
41 #: The default config file name for this application
42 #: The default config file name for this application
42 default_config_file_name = u'iplogger_config.py'
43 default_config_file_name = u'iplogger_config.py'
43
44
44 _description = """Start an IPython logger for parallel computing.
45 _description = """Start an IPython logger for parallel computing.
45
46
46 IPython controllers and engines (and your own processes) can broadcast log messages
47 IPython controllers and engines (and your own processes) can broadcast log messages
47 by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The
48 by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The
48 logger can be configured using command line options or using a cluster
49 logger can be configured using command line options or using a cluster
49 directory. Cluster directories contain config, log and security files and are
50 directory. Cluster directories contain config, log and security files and are
50 usually located in your ipython directory and named as "profile_name".
51 usually located in your ipython directory and named as "profile_name".
51 See the `profile` and `profile-dir` options for details.
52 See the `profile` and `profile-dir` options for details.
52 """
53 """
53
54
54
55
55 #-----------------------------------------------------------------------------
56 #-----------------------------------------------------------------------------
56 # Main application
57 # Main application
57 #-----------------------------------------------------------------------------
58 #-----------------------------------------------------------------------------
58 aliases = {}
59 aliases = {}
59 aliases.update(base_aliases)
60 aliases.update(base_aliases)
60 aliases.update(dict(url='LogWatcher.url', topics='LogWatcher.topics'))
61 aliases.update(dict(url='LogWatcher.url', topics='LogWatcher.topics'))
61
62
62 class IPLoggerApp(BaseParallelApplication):
63 class IPLoggerApp(BaseParallelApplication):
63
64
64 name = u'iplogger'
65 name = u'iplogger'
65 description = _description
66 description = _description
66 config_file_name = Unicode(default_config_file_name)
67 config_file_name = Unicode(default_config_file_name)
67
68
68 classes = [LogWatcher, ProfileDir]
69 classes = [LogWatcher, ProfileDir]
69 aliases = Dict(aliases)
70 aliases = Dict(aliases)
70
71
72 @catch_config
71 def initialize(self, argv=None):
73 def initialize(self, argv=None):
72 super(IPLoggerApp, self).initialize(argv)
74 super(IPLoggerApp, self).initialize(argv)
73 self.init_watcher()
75 self.init_watcher()
74
76
75 def init_watcher(self):
77 def init_watcher(self):
76 try:
78 try:
77 self.watcher = LogWatcher(config=self.config, log=self.log)
79 self.watcher = LogWatcher(config=self.config, log=self.log)
78 except:
80 except:
79 self.log.error("Couldn't start the LogWatcher", exc_info=True)
81 self.log.error("Couldn't start the LogWatcher", exc_info=True)
80 self.exit(1)
82 self.exit(1)
81 self.log.info("Listening for log messages on %r"%self.watcher.url)
83 self.log.info("Listening for log messages on %r"%self.watcher.url)
82
84
83
85
84 def start(self):
86 def start(self):
85 self.watcher.start()
87 self.watcher.start()
86 try:
88 try:
87 self.watcher.loop.start()
89 self.watcher.loop.start()
88 except KeyboardInterrupt:
90 except KeyboardInterrupt:
89 self.log.critical("Logging Interrupted, shutting down...\n")
91 self.log.critical("Logging Interrupted, shutting down...\n")
90
92
91
93
92 def launch_new_instance():
94 def launch_new_instance():
93 """Create and run the IPython LogWatcher"""
95 """Create and run the IPython LogWatcher"""
94 app = IPLoggerApp.instance()
96 app = IPLoggerApp.instance()
95 app.initialize()
97 app.initialize()
96 app.start()
98 app.start()
97
99
98
100
99 if __name__ == '__main__':
101 if __name__ == '__main__':
100 launch_new_instance()
102 launch_new_instance()
101
103
@@ -1,789 +1,791 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 """A simple interactive kernel that talks to a frontend over 0MQ.
2 """A simple interactive kernel that talks to a frontend over 0MQ.
3
3
4 Things to do:
4 Things to do:
5
5
6 * Implement `set_parent` logic. Right before doing exec, the Kernel should
6 * Implement `set_parent` logic. Right before doing exec, the Kernel should
7 call set_parent on all the PUB objects with the message about to be executed.
7 call set_parent on all the PUB objects with the message about to be executed.
8 * Implement random port and security key logic.
8 * Implement random port and security key logic.
9 * Implement control messages.
9 * Implement control messages.
10 * Implement event loop and poll version.
10 * Implement event loop and poll version.
11 """
11 """
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Imports
14 # Imports
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16 from __future__ import print_function
16 from __future__ import print_function
17
17
18 # Standard library imports.
18 # Standard library imports.
19 import __builtin__
19 import __builtin__
20 import atexit
20 import atexit
21 import sys
21 import sys
22 import time
22 import time
23 import traceback
23 import traceback
24 import logging
24 import logging
25
25
26 # System library imports.
26 # System library imports.
27 import zmq
27 import zmq
28
28
29 # Local imports.
29 # Local imports.
30 from IPython.config.configurable import Configurable
30 from IPython.config.configurable import Configurable
31 from IPython.config.application import boolean_flag
31 from IPython.config.application import boolean_flag, catch_config
32 from IPython.core.application import ProfileDir
32 from IPython.core.application import ProfileDir
33 from IPython.core.error import StdinNotImplementedError
33 from IPython.core.error import StdinNotImplementedError
34 from IPython.core.shellapp import (
34 from IPython.core.shellapp import (
35 InteractiveShellApp, shell_flags, shell_aliases
35 InteractiveShellApp, shell_flags, shell_aliases
36 )
36 )
37 from IPython.utils import io
37 from IPython.utils import io
38 from IPython.utils import py3compat
38 from IPython.utils import py3compat
39 from IPython.utils.jsonutil import json_clean
39 from IPython.utils.jsonutil import json_clean
40 from IPython.lib import pylabtools
40 from IPython.lib import pylabtools
41 from IPython.utils.traitlets import (
41 from IPython.utils.traitlets import (
42 Any, List, Instance, Float, Dict, Bool, Int, Unicode, CaselessStrEnum
42 Any, List, Instance, Float, Dict, Bool, Int, Unicode, CaselessStrEnum
43 )
43 )
44
44
45 from entry_point import base_launch_kernel
45 from entry_point import base_launch_kernel
46 from kernelapp import KernelApp, kernel_flags, kernel_aliases
46 from kernelapp import KernelApp, kernel_flags, kernel_aliases
47 from iostream import OutStream
47 from iostream import OutStream
48 from session import Session, Message
48 from session import Session, Message
49 from zmqshell import ZMQInteractiveShell
49 from zmqshell import ZMQInteractiveShell
50
50
51
51
52 #-----------------------------------------------------------------------------
52 #-----------------------------------------------------------------------------
53 # Main kernel class
53 # Main kernel class
54 #-----------------------------------------------------------------------------
54 #-----------------------------------------------------------------------------
55
55
56 class Kernel(Configurable):
56 class Kernel(Configurable):
57
57
58 #---------------------------------------------------------------------------
58 #---------------------------------------------------------------------------
59 # Kernel interface
59 # Kernel interface
60 #---------------------------------------------------------------------------
60 #---------------------------------------------------------------------------
61
61
62 # attribute to override with a GUI
62 # attribute to override with a GUI
63 eventloop = Any(None)
63 eventloop = Any(None)
64
64
65 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
65 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
66 session = Instance(Session)
66 session = Instance(Session)
67 shell_socket = Instance('zmq.Socket')
67 shell_socket = Instance('zmq.Socket')
68 iopub_socket = Instance('zmq.Socket')
68 iopub_socket = Instance('zmq.Socket')
69 stdin_socket = Instance('zmq.Socket')
69 stdin_socket = Instance('zmq.Socket')
70 log = Instance(logging.Logger)
70 log = Instance(logging.Logger)
71
71
72 # Private interface
72 # Private interface
73
73
74 # Time to sleep after flushing the stdout/err buffers in each execute
74 # Time to sleep after flushing the stdout/err buffers in each execute
75 # cycle. While this introduces a hard limit on the minimal latency of the
75 # cycle. While this introduces a hard limit on the minimal latency of the
76 # execute cycle, it helps prevent output synchronization problems for
76 # execute cycle, it helps prevent output synchronization problems for
77 # clients.
77 # clients.
78 # Units are in seconds. The minimum zmq latency on local host is probably
78 # Units are in seconds. The minimum zmq latency on local host is probably
79 # ~150 microseconds, set this to 500us for now. We may need to increase it
79 # ~150 microseconds, set this to 500us for now. We may need to increase it
80 # a little if it's not enough after more interactive testing.
80 # a little if it's not enough after more interactive testing.
81 _execute_sleep = Float(0.0005, config=True)
81 _execute_sleep = Float(0.0005, config=True)
82
82
83 # Frequency of the kernel's event loop.
83 # Frequency of the kernel's event loop.
84 # Units are in seconds, kernel subclasses for GUI toolkits may need to
84 # Units are in seconds, kernel subclasses for GUI toolkits may need to
85 # adapt to milliseconds.
85 # adapt to milliseconds.
86 _poll_interval = Float(0.05, config=True)
86 _poll_interval = Float(0.05, config=True)
87
87
88 # If the shutdown was requested over the network, we leave here the
88 # If the shutdown was requested over the network, we leave here the
89 # necessary reply message so it can be sent by our registered atexit
89 # necessary reply message so it can be sent by our registered atexit
90 # handler. This ensures that the reply is only sent to clients truly at
90 # handler. This ensures that the reply is only sent to clients truly at
91 # the end of our shutdown process (which happens after the underlying
91 # the end of our shutdown process (which happens after the underlying
92 # IPython shell's own shutdown).
92 # IPython shell's own shutdown).
93 _shutdown_message = None
93 _shutdown_message = None
94
94
95 # This is a dict of port number that the kernel is listening on. It is set
95 # This is a dict of port number that the kernel is listening on. It is set
96 # by record_ports and used by connect_request.
96 # by record_ports and used by connect_request.
97 _recorded_ports = Dict()
97 _recorded_ports = Dict()
98
98
99
99
100
100
101 def __init__(self, **kwargs):
101 def __init__(self, **kwargs):
102 super(Kernel, self).__init__(**kwargs)
102 super(Kernel, self).__init__(**kwargs)
103
103
104 # Before we even start up the shell, register *first* our exit handlers
104 # Before we even start up the shell, register *first* our exit handlers
105 # so they come before the shell's
105 # so they come before the shell's
106 atexit.register(self._at_shutdown)
106 atexit.register(self._at_shutdown)
107
107
108 # Initialize the InteractiveShell subclass
108 # Initialize the InteractiveShell subclass
109 self.shell = ZMQInteractiveShell.instance(config=self.config)
109 self.shell = ZMQInteractiveShell.instance(config=self.config)
110 self.shell.displayhook.session = self.session
110 self.shell.displayhook.session = self.session
111 self.shell.displayhook.pub_socket = self.iopub_socket
111 self.shell.displayhook.pub_socket = self.iopub_socket
112 self.shell.display_pub.session = self.session
112 self.shell.display_pub.session = self.session
113 self.shell.display_pub.pub_socket = self.iopub_socket
113 self.shell.display_pub.pub_socket = self.iopub_socket
114
114
115 # TMP - hack while developing
115 # TMP - hack while developing
116 self.shell._reply_content = None
116 self.shell._reply_content = None
117
117
118 # Build dict of handlers for message types
118 # Build dict of handlers for message types
119 msg_types = [ 'execute_request', 'complete_request',
119 msg_types = [ 'execute_request', 'complete_request',
120 'object_info_request', 'history_request',
120 'object_info_request', 'history_request',
121 'connect_request', 'shutdown_request']
121 'connect_request', 'shutdown_request']
122 self.handlers = {}
122 self.handlers = {}
123 for msg_type in msg_types:
123 for msg_type in msg_types:
124 self.handlers[msg_type] = getattr(self, msg_type)
124 self.handlers[msg_type] = getattr(self, msg_type)
125
125
126 def do_one_iteration(self):
126 def do_one_iteration(self):
127 """Do one iteration of the kernel's evaluation loop.
127 """Do one iteration of the kernel's evaluation loop.
128 """
128 """
129 try:
129 try:
130 ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK)
130 ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK)
131 except Exception:
131 except Exception:
132 self.log.warn("Invalid Message:", exc_info=True)
132 self.log.warn("Invalid Message:", exc_info=True)
133 return
133 return
134 if msg is None:
134 if msg is None:
135 return
135 return
136
136
137 msg_type = msg['header']['msg_type']
137 msg_type = msg['header']['msg_type']
138
138
139 # This assert will raise in versions of zeromq 2.0.7 and lesser.
139 # This assert will raise in versions of zeromq 2.0.7 and lesser.
140 # We now require 2.0.8 or above, so we can uncomment for safety.
140 # We now require 2.0.8 or above, so we can uncomment for safety.
141 # print(ident,msg, file=sys.__stdout__)
141 # print(ident,msg, file=sys.__stdout__)
142 assert ident is not None, "Missing message part."
142 assert ident is not None, "Missing message part."
143
143
144 # Print some info about this message and leave a '--->' marker, so it's
144 # Print some info about this message and leave a '--->' marker, so it's
145 # easier to trace visually the message chain when debugging. Each
145 # easier to trace visually the message chain when debugging. Each
146 # handler prints its message at the end.
146 # handler prints its message at the end.
147 self.log.debug('\n*** MESSAGE TYPE:'+str(msg_type)+'***')
147 self.log.debug('\n*** MESSAGE TYPE:'+str(msg_type)+'***')
148 self.log.debug(' Content: '+str(msg['content'])+'\n --->\n ')
148 self.log.debug(' Content: '+str(msg['content'])+'\n --->\n ')
149
149
150 # Find and call actual handler for message
150 # Find and call actual handler for message
151 handler = self.handlers.get(msg_type, None)
151 handler = self.handlers.get(msg_type, None)
152 if handler is None:
152 if handler is None:
153 self.log.error("UNKNOWN MESSAGE TYPE:" +str(msg))
153 self.log.error("UNKNOWN MESSAGE TYPE:" +str(msg))
154 else:
154 else:
155 handler(ident, msg)
155 handler(ident, msg)
156
156
157 # Check whether we should exit, in case the incoming message set the
157 # Check whether we should exit, in case the incoming message set the
158 # exit flag on
158 # exit flag on
159 if self.shell.exit_now:
159 if self.shell.exit_now:
160 self.log.debug('\nExiting IPython kernel...')
160 self.log.debug('\nExiting IPython kernel...')
161 # We do a normal, clean exit, which allows any actions registered
161 # We do a normal, clean exit, which allows any actions registered
162 # via atexit (such as history saving) to take place.
162 # via atexit (such as history saving) to take place.
163 sys.exit(0)
163 sys.exit(0)
164
164
165
165
166 def start(self):
166 def start(self):
167 """ Start the kernel main loop.
167 """ Start the kernel main loop.
168 """
168 """
169 poller = zmq.Poller()
169 poller = zmq.Poller()
170 poller.register(self.shell_socket, zmq.POLLIN)
170 poller.register(self.shell_socket, zmq.POLLIN)
171 # loop while self.eventloop has not been overridden
171 # loop while self.eventloop has not been overridden
172 while self.eventloop is None:
172 while self.eventloop is None:
173 try:
173 try:
174 # scale by extra factor of 10, because there is no
174 # scale by extra factor of 10, because there is no
175 # reason for this to be anything less than ~ 0.1s
175 # reason for this to be anything less than ~ 0.1s
176 # since it is a real poller and will respond
176 # since it is a real poller and will respond
177 # to events immediately
177 # to events immediately
178
178
179 # double nested try/except, to properly catch KeyboardInterrupt
179 # double nested try/except, to properly catch KeyboardInterrupt
180 # due to pyzmq Issue #130
180 # due to pyzmq Issue #130
181 try:
181 try:
182 poller.poll(10*1000*self._poll_interval)
182 poller.poll(10*1000*self._poll_interval)
183 self.do_one_iteration()
183 self.do_one_iteration()
184 except:
184 except:
185 raise
185 raise
186 except KeyboardInterrupt:
186 except KeyboardInterrupt:
187 # Ctrl-C shouldn't crash the kernel
187 # Ctrl-C shouldn't crash the kernel
188 io.raw_print("KeyboardInterrupt caught in kernel")
188 io.raw_print("KeyboardInterrupt caught in kernel")
189 if self.eventloop is not None:
189 if self.eventloop is not None:
190 try:
190 try:
191 self.eventloop(self)
191 self.eventloop(self)
192 except KeyboardInterrupt:
192 except KeyboardInterrupt:
193 # Ctrl-C shouldn't crash the kernel
193 # Ctrl-C shouldn't crash the kernel
194 io.raw_print("KeyboardInterrupt caught in kernel")
194 io.raw_print("KeyboardInterrupt caught in kernel")
195
195
196
196
197 def record_ports(self, ports):
197 def record_ports(self, ports):
198 """Record the ports that this kernel is using.
198 """Record the ports that this kernel is using.
199
199
200 The creator of the Kernel instance must call this methods if they
200 The creator of the Kernel instance must call this methods if they
201 want the :meth:`connect_request` method to return the port numbers.
201 want the :meth:`connect_request` method to return the port numbers.
202 """
202 """
203 self._recorded_ports = ports
203 self._recorded_ports = ports
204
204
205 #---------------------------------------------------------------------------
205 #---------------------------------------------------------------------------
206 # Kernel request handlers
206 # Kernel request handlers
207 #---------------------------------------------------------------------------
207 #---------------------------------------------------------------------------
208
208
209 def _publish_pyin(self, code, parent):
209 def _publish_pyin(self, code, parent):
210 """Publish the code request on the pyin stream."""
210 """Publish the code request on the pyin stream."""
211
211
212 pyin_msg = self.session.send(self.iopub_socket, u'pyin',{u'code':code}, parent=parent)
212 pyin_msg = self.session.send(self.iopub_socket, u'pyin',{u'code':code}, parent=parent)
213
213
214 def execute_request(self, ident, parent):
214 def execute_request(self, ident, parent):
215
215
216 status_msg = self.session.send(self.iopub_socket,
216 status_msg = self.session.send(self.iopub_socket,
217 u'status',
217 u'status',
218 {u'execution_state':u'busy'},
218 {u'execution_state':u'busy'},
219 parent=parent
219 parent=parent
220 )
220 )
221
221
222 try:
222 try:
223 content = parent[u'content']
223 content = parent[u'content']
224 code = content[u'code']
224 code = content[u'code']
225 silent = content[u'silent']
225 silent = content[u'silent']
226 except:
226 except:
227 self.log.error("Got bad msg: ")
227 self.log.error("Got bad msg: ")
228 self.log.error(str(Message(parent)))
228 self.log.error(str(Message(parent)))
229 return
229 return
230
230
231 shell = self.shell # we'll need this a lot here
231 shell = self.shell # we'll need this a lot here
232
232
233 # Replace raw_input. Note that is not sufficient to replace
233 # Replace raw_input. Note that is not sufficient to replace
234 # raw_input in the user namespace.
234 # raw_input in the user namespace.
235 if content.get('allow_stdin', False):
235 if content.get('allow_stdin', False):
236 raw_input = lambda prompt='': self._raw_input(prompt, ident, parent)
236 raw_input = lambda prompt='': self._raw_input(prompt, ident, parent)
237 else:
237 else:
238 raw_input = lambda prompt='' : self._no_raw_input()
238 raw_input = lambda prompt='' : self._no_raw_input()
239
239
240 if py3compat.PY3:
240 if py3compat.PY3:
241 __builtin__.input = raw_input
241 __builtin__.input = raw_input
242 else:
242 else:
243 __builtin__.raw_input = raw_input
243 __builtin__.raw_input = raw_input
244
244
245 # Set the parent message of the display hook and out streams.
245 # Set the parent message of the display hook and out streams.
246 shell.displayhook.set_parent(parent)
246 shell.displayhook.set_parent(parent)
247 shell.display_pub.set_parent(parent)
247 shell.display_pub.set_parent(parent)
248 sys.stdout.set_parent(parent)
248 sys.stdout.set_parent(parent)
249 sys.stderr.set_parent(parent)
249 sys.stderr.set_parent(parent)
250
250
251 # Re-broadcast our input for the benefit of listening clients, and
251 # Re-broadcast our input for the benefit of listening clients, and
252 # start computing output
252 # start computing output
253 if not silent:
253 if not silent:
254 self._publish_pyin(code, parent)
254 self._publish_pyin(code, parent)
255
255
256 reply_content = {}
256 reply_content = {}
257 try:
257 try:
258 if silent:
258 if silent:
259 # run_code uses 'exec' mode, so no displayhook will fire, and it
259 # run_code uses 'exec' mode, so no displayhook will fire, and it
260 # doesn't call logging or history manipulations. Print
260 # doesn't call logging or history manipulations. Print
261 # statements in that code will obviously still execute.
261 # statements in that code will obviously still execute.
262 shell.run_code(code)
262 shell.run_code(code)
263 else:
263 else:
264 # FIXME: the shell calls the exception handler itself.
264 # FIXME: the shell calls the exception handler itself.
265 shell.run_cell(code, store_history=True)
265 shell.run_cell(code, store_history=True)
266 except:
266 except:
267 status = u'error'
267 status = u'error'
268 # FIXME: this code right now isn't being used yet by default,
268 # FIXME: this code right now isn't being used yet by default,
269 # because the run_cell() call above directly fires off exception
269 # because the run_cell() call above directly fires off exception
270 # reporting. This code, therefore, is only active in the scenario
270 # reporting. This code, therefore, is only active in the scenario
271 # where runlines itself has an unhandled exception. We need to
271 # where runlines itself has an unhandled exception. We need to
272 # uniformize this, for all exception construction to come from a
272 # uniformize this, for all exception construction to come from a
273 # single location in the codbase.
273 # single location in the codbase.
274 etype, evalue, tb = sys.exc_info()
274 etype, evalue, tb = sys.exc_info()
275 tb_list = traceback.format_exception(etype, evalue, tb)
275 tb_list = traceback.format_exception(etype, evalue, tb)
276 reply_content.update(shell._showtraceback(etype, evalue, tb_list))
276 reply_content.update(shell._showtraceback(etype, evalue, tb_list))
277 else:
277 else:
278 status = u'ok'
278 status = u'ok'
279
279
280 reply_content[u'status'] = status
280 reply_content[u'status'] = status
281
281
282 # Return the execution counter so clients can display prompts
282 # Return the execution counter so clients can display prompts
283 reply_content['execution_count'] = shell.execution_count -1
283 reply_content['execution_count'] = shell.execution_count -1
284
284
285 # FIXME - fish exception info out of shell, possibly left there by
285 # FIXME - fish exception info out of shell, possibly left there by
286 # runlines. We'll need to clean up this logic later.
286 # runlines. We'll need to clean up this logic later.
287 if shell._reply_content is not None:
287 if shell._reply_content is not None:
288 reply_content.update(shell._reply_content)
288 reply_content.update(shell._reply_content)
289 # reset after use
289 # reset after use
290 shell._reply_content = None
290 shell._reply_content = None
291
291
292 # At this point, we can tell whether the main code execution succeeded
292 # At this point, we can tell whether the main code execution succeeded
293 # or not. If it did, we proceed to evaluate user_variables/expressions
293 # or not. If it did, we proceed to evaluate user_variables/expressions
294 if reply_content['status'] == 'ok':
294 if reply_content['status'] == 'ok':
295 reply_content[u'user_variables'] = \
295 reply_content[u'user_variables'] = \
296 shell.user_variables(content[u'user_variables'])
296 shell.user_variables(content[u'user_variables'])
297 reply_content[u'user_expressions'] = \
297 reply_content[u'user_expressions'] = \
298 shell.user_expressions(content[u'user_expressions'])
298 shell.user_expressions(content[u'user_expressions'])
299 else:
299 else:
300 # If there was an error, don't even try to compute variables or
300 # If there was an error, don't even try to compute variables or
301 # expressions
301 # expressions
302 reply_content[u'user_variables'] = {}
302 reply_content[u'user_variables'] = {}
303 reply_content[u'user_expressions'] = {}
303 reply_content[u'user_expressions'] = {}
304
304
305 # Payloads should be retrieved regardless of outcome, so we can both
305 # Payloads should be retrieved regardless of outcome, so we can both
306 # recover partial output (that could have been generated early in a
306 # recover partial output (that could have been generated early in a
307 # block, before an error) and clear the payload system always.
307 # block, before an error) and clear the payload system always.
308 reply_content[u'payload'] = shell.payload_manager.read_payload()
308 reply_content[u'payload'] = shell.payload_manager.read_payload()
309 # Be agressive about clearing the payload because we don't want
309 # Be agressive about clearing the payload because we don't want
310 # it to sit in memory until the next execute_request comes in.
310 # it to sit in memory until the next execute_request comes in.
311 shell.payload_manager.clear_payload()
311 shell.payload_manager.clear_payload()
312
312
313 # Flush output before sending the reply.
313 # Flush output before sending the reply.
314 sys.stdout.flush()
314 sys.stdout.flush()
315 sys.stderr.flush()
315 sys.stderr.flush()
316 # FIXME: on rare occasions, the flush doesn't seem to make it to the
316 # FIXME: on rare occasions, the flush doesn't seem to make it to the
317 # clients... This seems to mitigate the problem, but we definitely need
317 # clients... This seems to mitigate the problem, but we definitely need
318 # to better understand what's going on.
318 # to better understand what's going on.
319 if self._execute_sleep:
319 if self._execute_sleep:
320 time.sleep(self._execute_sleep)
320 time.sleep(self._execute_sleep)
321
321
322 # Send the reply.
322 # Send the reply.
323 reply_content = json_clean(reply_content)
323 reply_content = json_clean(reply_content)
324 reply_msg = self.session.send(self.shell_socket, u'execute_reply',
324 reply_msg = self.session.send(self.shell_socket, u'execute_reply',
325 reply_content, parent, ident=ident)
325 reply_content, parent, ident=ident)
326 self.log.debug(str(reply_msg))
326 self.log.debug(str(reply_msg))
327
327
328 if reply_msg['content']['status'] == u'error':
328 if reply_msg['content']['status'] == u'error':
329 self._abort_queue()
329 self._abort_queue()
330
330
331 status_msg = self.session.send(self.iopub_socket,
331 status_msg = self.session.send(self.iopub_socket,
332 u'status',
332 u'status',
333 {u'execution_state':u'idle'},
333 {u'execution_state':u'idle'},
334 parent=parent
334 parent=parent
335 )
335 )
336
336
337 def complete_request(self, ident, parent):
337 def complete_request(self, ident, parent):
338 txt, matches = self._complete(parent)
338 txt, matches = self._complete(parent)
339 matches = {'matches' : matches,
339 matches = {'matches' : matches,
340 'matched_text' : txt,
340 'matched_text' : txt,
341 'status' : 'ok'}
341 'status' : 'ok'}
342 matches = json_clean(matches)
342 matches = json_clean(matches)
343 completion_msg = self.session.send(self.shell_socket, 'complete_reply',
343 completion_msg = self.session.send(self.shell_socket, 'complete_reply',
344 matches, parent, ident)
344 matches, parent, ident)
345 self.log.debug(str(completion_msg))
345 self.log.debug(str(completion_msg))
346
346
347 def object_info_request(self, ident, parent):
347 def object_info_request(self, ident, parent):
348 object_info = self.shell.object_inspect(parent['content']['oname'])
348 object_info = self.shell.object_inspect(parent['content']['oname'])
349 # Before we send this object over, we scrub it for JSON usage
349 # Before we send this object over, we scrub it for JSON usage
350 oinfo = json_clean(object_info)
350 oinfo = json_clean(object_info)
351 msg = self.session.send(self.shell_socket, 'object_info_reply',
351 msg = self.session.send(self.shell_socket, 'object_info_reply',
352 oinfo, parent, ident)
352 oinfo, parent, ident)
353 self.log.debug(msg)
353 self.log.debug(msg)
354
354
355 def history_request(self, ident, parent):
355 def history_request(self, ident, parent):
356 # We need to pull these out, as passing **kwargs doesn't work with
356 # We need to pull these out, as passing **kwargs doesn't work with
357 # unicode keys before Python 2.6.5.
357 # unicode keys before Python 2.6.5.
358 hist_access_type = parent['content']['hist_access_type']
358 hist_access_type = parent['content']['hist_access_type']
359 raw = parent['content']['raw']
359 raw = parent['content']['raw']
360 output = parent['content']['output']
360 output = parent['content']['output']
361 if hist_access_type == 'tail':
361 if hist_access_type == 'tail':
362 n = parent['content']['n']
362 n = parent['content']['n']
363 hist = self.shell.history_manager.get_tail(n, raw=raw, output=output,
363 hist = self.shell.history_manager.get_tail(n, raw=raw, output=output,
364 include_latest=True)
364 include_latest=True)
365
365
366 elif hist_access_type == 'range':
366 elif hist_access_type == 'range':
367 session = parent['content']['session']
367 session = parent['content']['session']
368 start = parent['content']['start']
368 start = parent['content']['start']
369 stop = parent['content']['stop']
369 stop = parent['content']['stop']
370 hist = self.shell.history_manager.get_range(session, start, stop,
370 hist = self.shell.history_manager.get_range(session, start, stop,
371 raw=raw, output=output)
371 raw=raw, output=output)
372
372
373 elif hist_access_type == 'search':
373 elif hist_access_type == 'search':
374 pattern = parent['content']['pattern']
374 pattern = parent['content']['pattern']
375 hist = self.shell.history_manager.search(pattern, raw=raw, output=output)
375 hist = self.shell.history_manager.search(pattern, raw=raw, output=output)
376
376
377 else:
377 else:
378 hist = []
378 hist = []
379 content = {'history' : list(hist)}
379 content = {'history' : list(hist)}
380 content = json_clean(content)
380 content = json_clean(content)
381 msg = self.session.send(self.shell_socket, 'history_reply',
381 msg = self.session.send(self.shell_socket, 'history_reply',
382 content, parent, ident)
382 content, parent, ident)
383 self.log.debug(str(msg))
383 self.log.debug(str(msg))
384
384
385 def connect_request(self, ident, parent):
385 def connect_request(self, ident, parent):
386 if self._recorded_ports is not None:
386 if self._recorded_ports is not None:
387 content = self._recorded_ports.copy()
387 content = self._recorded_ports.copy()
388 else:
388 else:
389 content = {}
389 content = {}
390 msg = self.session.send(self.shell_socket, 'connect_reply',
390 msg = self.session.send(self.shell_socket, 'connect_reply',
391 content, parent, ident)
391 content, parent, ident)
392 self.log.debug(msg)
392 self.log.debug(msg)
393
393
394 def shutdown_request(self, ident, parent):
394 def shutdown_request(self, ident, parent):
395 self.shell.exit_now = True
395 self.shell.exit_now = True
396 self._shutdown_message = self.session.msg(u'shutdown_reply', parent['content'], parent)
396 self._shutdown_message = self.session.msg(u'shutdown_reply', parent['content'], parent)
397 sys.exit(0)
397 sys.exit(0)
398
398
399 #---------------------------------------------------------------------------
399 #---------------------------------------------------------------------------
400 # Protected interface
400 # Protected interface
401 #---------------------------------------------------------------------------
401 #---------------------------------------------------------------------------
402
402
403 def _abort_queue(self):
403 def _abort_queue(self):
404 while True:
404 while True:
405 try:
405 try:
406 ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK)
406 ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK)
407 except Exception:
407 except Exception:
408 self.log.warn("Invalid Message:", exc_info=True)
408 self.log.warn("Invalid Message:", exc_info=True)
409 continue
409 continue
410 if msg is None:
410 if msg is None:
411 break
411 break
412 else:
412 else:
413 assert ident is not None, \
413 assert ident is not None, \
414 "Unexpected missing message part."
414 "Unexpected missing message part."
415
415
416 self.log.debug("Aborting:\n"+str(Message(msg)))
416 self.log.debug("Aborting:\n"+str(Message(msg)))
417 msg_type = msg['header']['msg_type']
417 msg_type = msg['header']['msg_type']
418 reply_type = msg_type.split('_')[0] + '_reply'
418 reply_type = msg_type.split('_')[0] + '_reply'
419 reply_msg = self.session.send(self.shell_socket, reply_type,
419 reply_msg = self.session.send(self.shell_socket, reply_type,
420 {'status' : 'aborted'}, msg, ident=ident)
420 {'status' : 'aborted'}, msg, ident=ident)
421 self.log.debug(reply_msg)
421 self.log.debug(reply_msg)
422 # We need to wait a bit for requests to come in. This can probably
422 # We need to wait a bit for requests to come in. This can probably
423 # be set shorter for true asynchronous clients.
423 # be set shorter for true asynchronous clients.
424 time.sleep(0.1)
424 time.sleep(0.1)
425
425
426 def _no_raw_input(self):
426 def _no_raw_input(self):
427 """Raise StdinNotImplentedError if active frontend doesn't support stdin."""
427 """Raise StdinNotImplentedError if active frontend doesn't support stdin."""
428 raise StdinNotImplementedError("raw_input was called, but this frontend does not support stdin.")
428 raise StdinNotImplementedError("raw_input was called, but this frontend does not support stdin.")
429
429
430 def _raw_input(self, prompt, ident, parent):
430 def _raw_input(self, prompt, ident, parent):
431 # Flush output before making the request.
431 # Flush output before making the request.
432 sys.stderr.flush()
432 sys.stderr.flush()
433 sys.stdout.flush()
433 sys.stdout.flush()
434
434
435 # Send the input request.
435 # Send the input request.
436 content = json_clean(dict(prompt=prompt))
436 content = json_clean(dict(prompt=prompt))
437 msg = self.session.send(self.stdin_socket, u'input_request', content, parent, ident=ident)
437 msg = self.session.send(self.stdin_socket, u'input_request', content, parent, ident=ident)
438
438
439 # Await a response.
439 # Await a response.
440 while True:
440 while True:
441 try:
441 try:
442 ident, reply = self.session.recv(self.stdin_socket, 0)
442 ident, reply = self.session.recv(self.stdin_socket, 0)
443 except Exception:
443 except Exception:
444 self.log.warn("Invalid Message:", exc_info=True)
444 self.log.warn("Invalid Message:", exc_info=True)
445 else:
445 else:
446 break
446 break
447 try:
447 try:
448 value = reply['content']['value']
448 value = reply['content']['value']
449 except:
449 except:
450 self.log.error("Got bad raw_input reply: ")
450 self.log.error("Got bad raw_input reply: ")
451 self.log.error(str(Message(parent)))
451 self.log.error(str(Message(parent)))
452 value = ''
452 value = ''
453 return value
453 return value
454
454
455 def _complete(self, msg):
455 def _complete(self, msg):
456 c = msg['content']
456 c = msg['content']
457 try:
457 try:
458 cpos = int(c['cursor_pos'])
458 cpos = int(c['cursor_pos'])
459 except:
459 except:
460 # If we don't get something that we can convert to an integer, at
460 # If we don't get something that we can convert to an integer, at
461 # least attempt the completion guessing the cursor is at the end of
461 # least attempt the completion guessing the cursor is at the end of
462 # the text, if there's any, and otherwise of the line
462 # the text, if there's any, and otherwise of the line
463 cpos = len(c['text'])
463 cpos = len(c['text'])
464 if cpos==0:
464 if cpos==0:
465 cpos = len(c['line'])
465 cpos = len(c['line'])
466 return self.shell.complete(c['text'], c['line'], cpos)
466 return self.shell.complete(c['text'], c['line'], cpos)
467
467
468 def _object_info(self, context):
468 def _object_info(self, context):
469 symbol, leftover = self._symbol_from_context(context)
469 symbol, leftover = self._symbol_from_context(context)
470 if symbol is not None and not leftover:
470 if symbol is not None and not leftover:
471 doc = getattr(symbol, '__doc__', '')
471 doc = getattr(symbol, '__doc__', '')
472 else:
472 else:
473 doc = ''
473 doc = ''
474 object_info = dict(docstring = doc)
474 object_info = dict(docstring = doc)
475 return object_info
475 return object_info
476
476
477 def _symbol_from_context(self, context):
477 def _symbol_from_context(self, context):
478 if not context:
478 if not context:
479 return None, context
479 return None, context
480
480
481 base_symbol_string = context[0]
481 base_symbol_string = context[0]
482 symbol = self.shell.user_ns.get(base_symbol_string, None)
482 symbol = self.shell.user_ns.get(base_symbol_string, None)
483 if symbol is None:
483 if symbol is None:
484 symbol = __builtin__.__dict__.get(base_symbol_string, None)
484 symbol = __builtin__.__dict__.get(base_symbol_string, None)
485 if symbol is None:
485 if symbol is None:
486 return None, context
486 return None, context
487
487
488 context = context[1:]
488 context = context[1:]
489 for i, name in enumerate(context):
489 for i, name in enumerate(context):
490 new_symbol = getattr(symbol, name, None)
490 new_symbol = getattr(symbol, name, None)
491 if new_symbol is None:
491 if new_symbol is None:
492 return symbol, context[i:]
492 return symbol, context[i:]
493 else:
493 else:
494 symbol = new_symbol
494 symbol = new_symbol
495
495
496 return symbol, []
496 return symbol, []
497
497
498 def _at_shutdown(self):
498 def _at_shutdown(self):
499 """Actions taken at shutdown by the kernel, called by python's atexit.
499 """Actions taken at shutdown by the kernel, called by python's atexit.
500 """
500 """
501 # io.rprint("Kernel at_shutdown") # dbg
501 # io.rprint("Kernel at_shutdown") # dbg
502 if self._shutdown_message is not None:
502 if self._shutdown_message is not None:
503 self.session.send(self.shell_socket, self._shutdown_message)
503 self.session.send(self.shell_socket, self._shutdown_message)
504 self.session.send(self.iopub_socket, self._shutdown_message)
504 self.session.send(self.iopub_socket, self._shutdown_message)
505 self.log.debug(str(self._shutdown_message))
505 self.log.debug(str(self._shutdown_message))
506 # A very short sleep to give zmq time to flush its message buffers
506 # A very short sleep to give zmq time to flush its message buffers
507 # before Python truly shuts down.
507 # before Python truly shuts down.
508 time.sleep(0.01)
508 time.sleep(0.01)
509
509
510
510
511 #------------------------------------------------------------------------------
511 #------------------------------------------------------------------------------
512 # Eventloops for integrating the Kernel into different GUIs
512 # Eventloops for integrating the Kernel into different GUIs
513 #------------------------------------------------------------------------------
513 #------------------------------------------------------------------------------
514
514
515
515
516 def loop_qt4(kernel):
516 def loop_qt4(kernel):
517 """Start a kernel with PyQt4 event loop integration."""
517 """Start a kernel with PyQt4 event loop integration."""
518
518
519 from IPython.external.qt_for_kernel import QtCore
519 from IPython.external.qt_for_kernel import QtCore
520 from IPython.lib.guisupport import get_app_qt4, start_event_loop_qt4
520 from IPython.lib.guisupport import get_app_qt4, start_event_loop_qt4
521
521
522 kernel.app = get_app_qt4([" "])
522 kernel.app = get_app_qt4([" "])
523 kernel.app.setQuitOnLastWindowClosed(False)
523 kernel.app.setQuitOnLastWindowClosed(False)
524 kernel.timer = QtCore.QTimer()
524 kernel.timer = QtCore.QTimer()
525 kernel.timer.timeout.connect(kernel.do_one_iteration)
525 kernel.timer.timeout.connect(kernel.do_one_iteration)
526 # Units for the timer are in milliseconds
526 # Units for the timer are in milliseconds
527 kernel.timer.start(1000*kernel._poll_interval)
527 kernel.timer.start(1000*kernel._poll_interval)
528 start_event_loop_qt4(kernel.app)
528 start_event_loop_qt4(kernel.app)
529
529
530
530
531 def loop_wx(kernel):
531 def loop_wx(kernel):
532 """Start a kernel with wx event loop support."""
532 """Start a kernel with wx event loop support."""
533
533
534 import wx
534 import wx
535 from IPython.lib.guisupport import start_event_loop_wx
535 from IPython.lib.guisupport import start_event_loop_wx
536
536
537 doi = kernel.do_one_iteration
537 doi = kernel.do_one_iteration
538 # Wx uses milliseconds
538 # Wx uses milliseconds
539 poll_interval = int(1000*kernel._poll_interval)
539 poll_interval = int(1000*kernel._poll_interval)
540
540
541 # We have to put the wx.Timer in a wx.Frame for it to fire properly.
541 # We have to put the wx.Timer in a wx.Frame for it to fire properly.
542 # We make the Frame hidden when we create it in the main app below.
542 # We make the Frame hidden when we create it in the main app below.
543 class TimerFrame(wx.Frame):
543 class TimerFrame(wx.Frame):
544 def __init__(self, func):
544 def __init__(self, func):
545 wx.Frame.__init__(self, None, -1)
545 wx.Frame.__init__(self, None, -1)
546 self.timer = wx.Timer(self)
546 self.timer = wx.Timer(self)
547 # Units for the timer are in milliseconds
547 # Units for the timer are in milliseconds
548 self.timer.Start(poll_interval)
548 self.timer.Start(poll_interval)
549 self.Bind(wx.EVT_TIMER, self.on_timer)
549 self.Bind(wx.EVT_TIMER, self.on_timer)
550 self.func = func
550 self.func = func
551
551
552 def on_timer(self, event):
552 def on_timer(self, event):
553 self.func()
553 self.func()
554
554
555 # We need a custom wx.App to create our Frame subclass that has the
555 # We need a custom wx.App to create our Frame subclass that has the
556 # wx.Timer to drive the ZMQ event loop.
556 # wx.Timer to drive the ZMQ event loop.
557 class IPWxApp(wx.App):
557 class IPWxApp(wx.App):
558 def OnInit(self):
558 def OnInit(self):
559 self.frame = TimerFrame(doi)
559 self.frame = TimerFrame(doi)
560 self.frame.Show(False)
560 self.frame.Show(False)
561 return True
561 return True
562
562
563 # The redirect=False here makes sure that wx doesn't replace
563 # The redirect=False here makes sure that wx doesn't replace
564 # sys.stdout/stderr with its own classes.
564 # sys.stdout/stderr with its own classes.
565 kernel.app = IPWxApp(redirect=False)
565 kernel.app = IPWxApp(redirect=False)
566 start_event_loop_wx(kernel.app)
566 start_event_loop_wx(kernel.app)
567
567
568
568
569 def loop_tk(kernel):
569 def loop_tk(kernel):
570 """Start a kernel with the Tk event loop."""
570 """Start a kernel with the Tk event loop."""
571
571
572 import Tkinter
572 import Tkinter
573 doi = kernel.do_one_iteration
573 doi = kernel.do_one_iteration
574 # Tk uses milliseconds
574 # Tk uses milliseconds
575 poll_interval = int(1000*kernel._poll_interval)
575 poll_interval = int(1000*kernel._poll_interval)
576 # For Tkinter, we create a Tk object and call its withdraw method.
576 # For Tkinter, we create a Tk object and call its withdraw method.
577 class Timer(object):
577 class Timer(object):
578 def __init__(self, func):
578 def __init__(self, func):
579 self.app = Tkinter.Tk()
579 self.app = Tkinter.Tk()
580 self.app.withdraw()
580 self.app.withdraw()
581 self.func = func
581 self.func = func
582
582
583 def on_timer(self):
583 def on_timer(self):
584 self.func()
584 self.func()
585 self.app.after(poll_interval, self.on_timer)
585 self.app.after(poll_interval, self.on_timer)
586
586
587 def start(self):
587 def start(self):
588 self.on_timer() # Call it once to get things going.
588 self.on_timer() # Call it once to get things going.
589 self.app.mainloop()
589 self.app.mainloop()
590
590
591 kernel.timer = Timer(doi)
591 kernel.timer = Timer(doi)
592 kernel.timer.start()
592 kernel.timer.start()
593
593
594
594
595 def loop_gtk(kernel):
595 def loop_gtk(kernel):
596 """Start the kernel, coordinating with the GTK event loop"""
596 """Start the kernel, coordinating with the GTK event loop"""
597 from .gui.gtkembed import GTKEmbed
597 from .gui.gtkembed import GTKEmbed
598
598
599 gtk_kernel = GTKEmbed(kernel)
599 gtk_kernel = GTKEmbed(kernel)
600 gtk_kernel.start()
600 gtk_kernel.start()
601
601
602
602
603 def loop_cocoa(kernel):
603 def loop_cocoa(kernel):
604 """Start the kernel, coordinating with the Cocoa CFRunLoop event loop
604 """Start the kernel, coordinating with the Cocoa CFRunLoop event loop
605 via the matplotlib MacOSX backend.
605 via the matplotlib MacOSX backend.
606 """
606 """
607 import matplotlib
607 import matplotlib
608 if matplotlib.__version__ < '1.1.0':
608 if matplotlib.__version__ < '1.1.0':
609 kernel.log.warn(
609 kernel.log.warn(
610 "MacOSX backend in matplotlib %s doesn't have a Timer, "
610 "MacOSX backend in matplotlib %s doesn't have a Timer, "
611 "falling back on Tk for CFRunLoop integration. Note that "
611 "falling back on Tk for CFRunLoop integration. Note that "
612 "even this won't work if Tk is linked against X11 instead of "
612 "even this won't work if Tk is linked against X11 instead of "
613 "Cocoa (e.g. EPD). To use the MacOSX backend in the kernel, "
613 "Cocoa (e.g. EPD). To use the MacOSX backend in the kernel, "
614 "you must use matplotlib >= 1.1.0, or a native libtk."
614 "you must use matplotlib >= 1.1.0, or a native libtk."
615 )
615 )
616 return loop_tk(kernel)
616 return loop_tk(kernel)
617
617
618 from matplotlib.backends.backend_macosx import TimerMac, show
618 from matplotlib.backends.backend_macosx import TimerMac, show
619
619
620 # scale interval for sec->ms
620 # scale interval for sec->ms
621 poll_interval = int(1000*kernel._poll_interval)
621 poll_interval = int(1000*kernel._poll_interval)
622
622
623 real_excepthook = sys.excepthook
623 real_excepthook = sys.excepthook
624 def handle_int(etype, value, tb):
624 def handle_int(etype, value, tb):
625 """don't let KeyboardInterrupts look like crashes"""
625 """don't let KeyboardInterrupts look like crashes"""
626 if etype is KeyboardInterrupt:
626 if etype is KeyboardInterrupt:
627 io.raw_print("KeyboardInterrupt caught in CFRunLoop")
627 io.raw_print("KeyboardInterrupt caught in CFRunLoop")
628 else:
628 else:
629 real_excepthook(etype, value, tb)
629 real_excepthook(etype, value, tb)
630
630
631 # add doi() as a Timer to the CFRunLoop
631 # add doi() as a Timer to the CFRunLoop
632 def doi():
632 def doi():
633 # restore excepthook during IPython code
633 # restore excepthook during IPython code
634 sys.excepthook = real_excepthook
634 sys.excepthook = real_excepthook
635 kernel.do_one_iteration()
635 kernel.do_one_iteration()
636 # and back:
636 # and back:
637 sys.excepthook = handle_int
637 sys.excepthook = handle_int
638
638
639 t = TimerMac(poll_interval)
639 t = TimerMac(poll_interval)
640 t.add_callback(doi)
640 t.add_callback(doi)
641 t.start()
641 t.start()
642
642
643 # but still need a Poller for when there are no active windows,
643 # but still need a Poller for when there are no active windows,
644 # during which time mainloop() returns immediately
644 # during which time mainloop() returns immediately
645 poller = zmq.Poller()
645 poller = zmq.Poller()
646 poller.register(kernel.shell_socket, zmq.POLLIN)
646 poller.register(kernel.shell_socket, zmq.POLLIN)
647
647
648 while True:
648 while True:
649 try:
649 try:
650 # double nested try/except, to properly catch KeyboardInterrupt
650 # double nested try/except, to properly catch KeyboardInterrupt
651 # due to pyzmq Issue #130
651 # due to pyzmq Issue #130
652 try:
652 try:
653 # don't let interrupts during mainloop invoke crash_handler:
653 # don't let interrupts during mainloop invoke crash_handler:
654 sys.excepthook = handle_int
654 sys.excepthook = handle_int
655 show.mainloop()
655 show.mainloop()
656 sys.excepthook = real_excepthook
656 sys.excepthook = real_excepthook
657 # use poller if mainloop returned (no windows)
657 # use poller if mainloop returned (no windows)
658 # scale by extra factor of 10, since it's a real poll
658 # scale by extra factor of 10, since it's a real poll
659 poller.poll(10*poll_interval)
659 poller.poll(10*poll_interval)
660 kernel.do_one_iteration()
660 kernel.do_one_iteration()
661 except:
661 except:
662 raise
662 raise
663 except KeyboardInterrupt:
663 except KeyboardInterrupt:
664 # Ctrl-C shouldn't crash the kernel
664 # Ctrl-C shouldn't crash the kernel
665 io.raw_print("KeyboardInterrupt caught in kernel")
665 io.raw_print("KeyboardInterrupt caught in kernel")
666 finally:
666 finally:
667 # ensure excepthook is restored
667 # ensure excepthook is restored
668 sys.excepthook = real_excepthook
668 sys.excepthook = real_excepthook
669
669
670 # mapping of keys to loop functions
670 # mapping of keys to loop functions
671 loop_map = {
671 loop_map = {
672 'qt' : loop_qt4,
672 'qt' : loop_qt4,
673 'qt4': loop_qt4,
673 'qt4': loop_qt4,
674 'inline': None,
674 'inline': None,
675 'osx': loop_cocoa,
675 'osx': loop_cocoa,
676 'wx' : loop_wx,
676 'wx' : loop_wx,
677 'tk' : loop_tk,
677 'tk' : loop_tk,
678 'gtk': loop_gtk,
678 'gtk': loop_gtk,
679 }
679 }
680
680
681 def enable_gui(gui, kernel=None):
681 def enable_gui(gui, kernel=None):
682 """Enable integration with a give GUI"""
682 """Enable integration with a give GUI"""
683 if kernel is None:
683 if kernel is None:
684 kernel = IPKernelApp.instance().kernel
684 kernel = IPKernelApp.instance().kernel
685 if gui not in loop_map:
685 if gui not in loop_map:
686 raise ValueError("GUI %r not supported" % gui)
686 raise ValueError("GUI %r not supported" % gui)
687 loop = loop_map[gui]
687 loop = loop_map[gui]
688 if kernel.eventloop is not None and kernel.eventloop is not loop:
688 if kernel.eventloop is not None and kernel.eventloop is not loop:
689 raise RuntimeError("Cannot activate multiple GUI eventloops")
689 raise RuntimeError("Cannot activate multiple GUI eventloops")
690 kernel.eventloop = loop
690 kernel.eventloop = loop
691
691
692
692
693 #-----------------------------------------------------------------------------
693 #-----------------------------------------------------------------------------
694 # Aliases and Flags for the IPKernelApp
694 # Aliases and Flags for the IPKernelApp
695 #-----------------------------------------------------------------------------
695 #-----------------------------------------------------------------------------
696
696
697 flags = dict(kernel_flags)
697 flags = dict(kernel_flags)
698 flags.update(shell_flags)
698 flags.update(shell_flags)
699
699
700 addflag = lambda *args: flags.update(boolean_flag(*args))
700 addflag = lambda *args: flags.update(boolean_flag(*args))
701
701
702 flags['pylab'] = (
702 flags['pylab'] = (
703 {'IPKernelApp' : {'pylab' : 'auto'}},
703 {'IPKernelApp' : {'pylab' : 'auto'}},
704 """Pre-load matplotlib and numpy for interactive use with
704 """Pre-load matplotlib and numpy for interactive use with
705 the default matplotlib backend."""
705 the default matplotlib backend."""
706 )
706 )
707
707
708 aliases = dict(kernel_aliases)
708 aliases = dict(kernel_aliases)
709 aliases.update(shell_aliases)
709 aliases.update(shell_aliases)
710
710
711 # it's possible we don't want short aliases for *all* of these:
711 # it's possible we don't want short aliases for *all* of these:
712 aliases.update(dict(
712 aliases.update(dict(
713 pylab='IPKernelApp.pylab',
713 pylab='IPKernelApp.pylab',
714 ))
714 ))
715
715
716 #-----------------------------------------------------------------------------
716 #-----------------------------------------------------------------------------
717 # The IPKernelApp class
717 # The IPKernelApp class
718 #-----------------------------------------------------------------------------
718 #-----------------------------------------------------------------------------
719
719
720 class IPKernelApp(KernelApp, InteractiveShellApp):
720 class IPKernelApp(KernelApp, InteractiveShellApp):
721 name = 'ipkernel'
721 name = 'ipkernel'
722
722
723 aliases = Dict(aliases)
723 aliases = Dict(aliases)
724 flags = Dict(flags)
724 flags = Dict(flags)
725 classes = [Kernel, ZMQInteractiveShell, ProfileDir, Session]
725 classes = [Kernel, ZMQInteractiveShell, ProfileDir, Session]
726 # configurables
726 # configurables
727 pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'inline', 'auto'],
727 pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'inline', 'auto'],
728 config=True,
728 config=True,
729 help="""Pre-load matplotlib and numpy for interactive use,
729 help="""Pre-load matplotlib and numpy for interactive use,
730 selecting a particular matplotlib backend and loop integration.
730 selecting a particular matplotlib backend and loop integration.
731 """
731 """
732 )
732 )
733
734 @catch_config
733 def initialize(self, argv=None):
735 def initialize(self, argv=None):
734 super(IPKernelApp, self).initialize(argv)
736 super(IPKernelApp, self).initialize(argv)
735 self.init_shell()
737 self.init_shell()
736 self.init_extensions()
738 self.init_extensions()
737 self.init_code()
739 self.init_code()
738
740
739 def init_kernel(self):
741 def init_kernel(self):
740 kernel_factory = Kernel
742 kernel_factory = Kernel
741
743
742 if self.pylab:
744 if self.pylab:
743 gui, backend = pylabtools.find_gui_and_backend(self.pylab)
745 gui, backend = pylabtools.find_gui_and_backend(self.pylab)
744
746
745 kernel = kernel_factory(config=self.config, session=self.session,
747 kernel = kernel_factory(config=self.config, session=self.session,
746 shell_socket=self.shell_socket,
748 shell_socket=self.shell_socket,
747 iopub_socket=self.iopub_socket,
749 iopub_socket=self.iopub_socket,
748 stdin_socket=self.stdin_socket,
750 stdin_socket=self.stdin_socket,
749 log=self.log,
751 log=self.log,
750 )
752 )
751 self.kernel = kernel
753 self.kernel = kernel
752 kernel.record_ports(self.ports)
754 kernel.record_ports(self.ports)
753
755
754 if self.pylab:
756 if self.pylab:
755 kernel.shell.enable_pylab(gui, import_all=self.pylab_import_all)
757 kernel.shell.enable_pylab(gui, import_all=self.pylab_import_all)
756
758
757 def init_shell(self):
759 def init_shell(self):
758 self.shell = self.kernel.shell
760 self.shell = self.kernel.shell
759
761
760
762
761 #-----------------------------------------------------------------------------
763 #-----------------------------------------------------------------------------
762 # Kernel main and launch functions
764 # Kernel main and launch functions
763 #-----------------------------------------------------------------------------
765 #-----------------------------------------------------------------------------
764
766
765 def launch_kernel(*args, **kwargs):
767 def launch_kernel(*args, **kwargs):
766 """Launches a localhost IPython kernel, binding to the specified ports.
768 """Launches a localhost IPython kernel, binding to the specified ports.
767
769
768 This function simply calls entry_point.base_launch_kernel with the right first
770 This function simply calls entry_point.base_launch_kernel with the right first
769 command to start an ipkernel. See base_launch_kernel for arguments.
771 command to start an ipkernel. See base_launch_kernel for arguments.
770
772
771 Returns
773 Returns
772 -------
774 -------
773 A tuple of form:
775 A tuple of form:
774 (kernel_process, shell_port, iopub_port, stdin_port, hb_port)
776 (kernel_process, shell_port, iopub_port, stdin_port, hb_port)
775 where kernel_process is a Popen object and the ports are integers.
777 where kernel_process is a Popen object and the ports are integers.
776 """
778 """
777 return base_launch_kernel('from IPython.zmq.ipkernel import main; main()',
779 return base_launch_kernel('from IPython.zmq.ipkernel import main; main()',
778 *args, **kwargs)
780 *args, **kwargs)
779
781
780
782
781 def main():
783 def main():
782 """Run an IPKernel as an application"""
784 """Run an IPKernel as an application"""
783 app = IPKernelApp.instance()
785 app = IPKernelApp.instance()
784 app.initialize()
786 app.initialize()
785 app.start()
787 app.start()
786
788
787
789
788 if __name__ == '__main__':
790 if __name__ == '__main__':
789 main()
791 main()
@@ -1,298 +1,299 b''
1 """An Application for launching a kernel
1 """An Application for launching a kernel
2
2
3 Authors
3 Authors
4 -------
4 -------
5 * MinRK
5 * MinRK
6 """
6 """
7 #-----------------------------------------------------------------------------
7 #-----------------------------------------------------------------------------
8 # Copyright (C) 2011 The IPython Development Team
8 # Copyright (C) 2011 The IPython Development Team
9 #
9 #
10 # Distributed under the terms of the BSD License. The full license is in
10 # Distributed under the terms of the BSD License. The full license is in
11 # the file COPYING.txt, distributed as part of this software.
11 # the file COPYING.txt, distributed as part of this software.
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13
13
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15 # Imports
15 # Imports
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17
17
18 # Standard library imports.
18 # Standard library imports.
19 import json
19 import json
20 import os
20 import os
21 import sys
21 import sys
22
22
23 # System library imports.
23 # System library imports.
24 import zmq
24 import zmq
25
25
26 # IPython imports.
26 # IPython imports.
27 from IPython.core.ultratb import FormattedTB
27 from IPython.core.ultratb import FormattedTB
28 from IPython.core.application import (
28 from IPython.core.application import (
29 BaseIPythonApplication, base_flags, base_aliases
29 BaseIPythonApplication, base_flags, base_aliases, catch_config
30 )
30 )
31 from IPython.utils import io
31 from IPython.utils import io
32 from IPython.utils.localinterfaces import LOCALHOST
32 from IPython.utils.localinterfaces import LOCALHOST
33 from IPython.utils.path import filefind
33 from IPython.utils.path import filefind
34 from IPython.utils.py3compat import str_to_bytes
34 from IPython.utils.py3compat import str_to_bytes
35 from IPython.utils.traitlets import (Any, Instance, Dict, Unicode, Int, Bool,
35 from IPython.utils.traitlets import (Any, Instance, Dict, Unicode, Int, Bool,
36 DottedObjectName)
36 DottedObjectName)
37 from IPython.utils.importstring import import_item
37 from IPython.utils.importstring import import_item
38 # local imports
38 # local imports
39 from IPython.zmq.entry_point import write_connection_file
39 from IPython.zmq.entry_point import write_connection_file
40 from IPython.zmq.heartbeat import Heartbeat
40 from IPython.zmq.heartbeat import Heartbeat
41 from IPython.zmq.parentpoller import ParentPollerUnix, ParentPollerWindows
41 from IPython.zmq.parentpoller import ParentPollerUnix, ParentPollerWindows
42 from IPython.zmq.session import (
42 from IPython.zmq.session import (
43 Session, session_flags, session_aliases, default_secure,
43 Session, session_flags, session_aliases, default_secure,
44 )
44 )
45
45
46
46
47 #-----------------------------------------------------------------------------
47 #-----------------------------------------------------------------------------
48 # Flags and Aliases
48 # Flags and Aliases
49 #-----------------------------------------------------------------------------
49 #-----------------------------------------------------------------------------
50
50
51 kernel_aliases = dict(base_aliases)
51 kernel_aliases = dict(base_aliases)
52 kernel_aliases.update({
52 kernel_aliases.update({
53 'ip' : 'KernelApp.ip',
53 'ip' : 'KernelApp.ip',
54 'hb' : 'KernelApp.hb_port',
54 'hb' : 'KernelApp.hb_port',
55 'shell' : 'KernelApp.shell_port',
55 'shell' : 'KernelApp.shell_port',
56 'iopub' : 'KernelApp.iopub_port',
56 'iopub' : 'KernelApp.iopub_port',
57 'stdin' : 'KernelApp.stdin_port',
57 'stdin' : 'KernelApp.stdin_port',
58 'f' : 'KernelApp.connection_file',
58 'f' : 'KernelApp.connection_file',
59 'parent': 'KernelApp.parent',
59 'parent': 'KernelApp.parent',
60 })
60 })
61 if sys.platform.startswith('win'):
61 if sys.platform.startswith('win'):
62 kernel_aliases['interrupt'] = 'KernelApp.interrupt'
62 kernel_aliases['interrupt'] = 'KernelApp.interrupt'
63
63
64 kernel_flags = dict(base_flags)
64 kernel_flags = dict(base_flags)
65 kernel_flags.update({
65 kernel_flags.update({
66 'no-stdout' : (
66 'no-stdout' : (
67 {'KernelApp' : {'no_stdout' : True}},
67 {'KernelApp' : {'no_stdout' : True}},
68 "redirect stdout to the null device"),
68 "redirect stdout to the null device"),
69 'no-stderr' : (
69 'no-stderr' : (
70 {'KernelApp' : {'no_stderr' : True}},
70 {'KernelApp' : {'no_stderr' : True}},
71 "redirect stderr to the null device"),
71 "redirect stderr to the null device"),
72 })
72 })
73
73
74 # inherit flags&aliases for Sessions
74 # inherit flags&aliases for Sessions
75 kernel_aliases.update(session_aliases)
75 kernel_aliases.update(session_aliases)
76 kernel_flags.update(session_flags)
76 kernel_flags.update(session_flags)
77
77
78
78
79
79
80 #-----------------------------------------------------------------------------
80 #-----------------------------------------------------------------------------
81 # Application class for starting a Kernel
81 # Application class for starting a Kernel
82 #-----------------------------------------------------------------------------
82 #-----------------------------------------------------------------------------
83
83
84 class KernelApp(BaseIPythonApplication):
84 class KernelApp(BaseIPythonApplication):
85 name='pykernel'
85 name='pykernel'
86 aliases = Dict(kernel_aliases)
86 aliases = Dict(kernel_aliases)
87 flags = Dict(kernel_flags)
87 flags = Dict(kernel_flags)
88 classes = [Session]
88 classes = [Session]
89 # the kernel class, as an importstring
89 # the kernel class, as an importstring
90 kernel_class = DottedObjectName('IPython.zmq.pykernel.Kernel')
90 kernel_class = DottedObjectName('IPython.zmq.pykernel.Kernel')
91 kernel = Any()
91 kernel = Any()
92 poller = Any() # don't restrict this even though current pollers are all Threads
92 poller = Any() # don't restrict this even though current pollers are all Threads
93 heartbeat = Instance(Heartbeat)
93 heartbeat = Instance(Heartbeat)
94 session = Instance('IPython.zmq.session.Session')
94 session = Instance('IPython.zmq.session.Session')
95 ports = Dict()
95 ports = Dict()
96
96
97 # inherit config file name from parent:
97 # inherit config file name from parent:
98 parent_appname = Unicode(config=True)
98 parent_appname = Unicode(config=True)
99 def _parent_appname_changed(self, name, old, new):
99 def _parent_appname_changed(self, name, old, new):
100 if self.config_file_specified:
100 if self.config_file_specified:
101 # it was manually specified, ignore
101 # it was manually specified, ignore
102 return
102 return
103 self.config_file_name = new.replace('-','_') + u'_config.py'
103 self.config_file_name = new.replace('-','_') + u'_config.py'
104 # don't let this count as specifying the config file
104 # don't let this count as specifying the config file
105 self.config_file_specified = False
105 self.config_file_specified = False
106
106
107 # connection info:
107 # connection info:
108 ip = Unicode(LOCALHOST, config=True,
108 ip = Unicode(LOCALHOST, config=True,
109 help="Set the IP or interface on which the kernel will listen.")
109 help="Set the IP or interface on which the kernel will listen.")
110 hb_port = Int(0, config=True, help="set the heartbeat port [default: random]")
110 hb_port = Int(0, config=True, help="set the heartbeat port [default: random]")
111 shell_port = Int(0, config=True, help="set the shell (XREP) port [default: random]")
111 shell_port = Int(0, config=True, help="set the shell (XREP) port [default: random]")
112 iopub_port = Int(0, config=True, help="set the iopub (PUB) port [default: random]")
112 iopub_port = Int(0, config=True, help="set the iopub (PUB) port [default: random]")
113 stdin_port = Int(0, config=True, help="set the stdin (XREQ) port [default: random]")
113 stdin_port = Int(0, config=True, help="set the stdin (XREQ) port [default: random]")
114 connection_file = Unicode('', config=True,
114 connection_file = Unicode('', config=True,
115 help="""JSON file in which to store connection info [default: kernel-<pid>.json]
115 help="""JSON file in which to store connection info [default: kernel-<pid>.json]
116
116
117 This file will contain the IP, ports, and authentication key needed to connect
117 This file will contain the IP, ports, and authentication key needed to connect
118 clients to this kernel. By default, this file will be created in the security-dir
118 clients to this kernel. By default, this file will be created in the security-dir
119 of the current profile, but can be specified by absolute path.
119 of the current profile, but can be specified by absolute path.
120 """)
120 """)
121
121
122 # streams, etc.
122 # streams, etc.
123 no_stdout = Bool(False, config=True, help="redirect stdout to the null device")
123 no_stdout = Bool(False, config=True, help="redirect stdout to the null device")
124 no_stderr = Bool(False, config=True, help="redirect stderr to the null device")
124 no_stderr = Bool(False, config=True, help="redirect stderr to the null device")
125 outstream_class = DottedObjectName('IPython.zmq.iostream.OutStream',
125 outstream_class = DottedObjectName('IPython.zmq.iostream.OutStream',
126 config=True, help="The importstring for the OutStream factory")
126 config=True, help="The importstring for the OutStream factory")
127 displayhook_class = DottedObjectName('IPython.zmq.displayhook.ZMQDisplayHook',
127 displayhook_class = DottedObjectName('IPython.zmq.displayhook.ZMQDisplayHook',
128 config=True, help="The importstring for the DisplayHook factory")
128 config=True, help="The importstring for the DisplayHook factory")
129
129
130 # polling
130 # polling
131 parent = Int(0, config=True,
131 parent = Int(0, config=True,
132 help="""kill this process if its parent dies. On Windows, the argument
132 help="""kill this process if its parent dies. On Windows, the argument
133 specifies the HANDLE of the parent process, otherwise it is simply boolean.
133 specifies the HANDLE of the parent process, otherwise it is simply boolean.
134 """)
134 """)
135 interrupt = Int(0, config=True,
135 interrupt = Int(0, config=True,
136 help="""ONLY USED ON WINDOWS
136 help="""ONLY USED ON WINDOWS
137 Interrupt this process when the parent is signalled.
137 Interrupt this process when the parent is signalled.
138 """)
138 """)
139
139
140 def init_crash_handler(self):
140 def init_crash_handler(self):
141 # Install minimal exception handling
141 # Install minimal exception handling
142 sys.excepthook = FormattedTB(mode='Verbose', color_scheme='NoColor',
142 sys.excepthook = FormattedTB(mode='Verbose', color_scheme='NoColor',
143 ostream=sys.__stdout__)
143 ostream=sys.__stdout__)
144
144
145 def init_poller(self):
145 def init_poller(self):
146 if sys.platform == 'win32':
146 if sys.platform == 'win32':
147 if self.interrupt or self.parent:
147 if self.interrupt or self.parent:
148 self.poller = ParentPollerWindows(self.interrupt, self.parent)
148 self.poller = ParentPollerWindows(self.interrupt, self.parent)
149 elif self.parent:
149 elif self.parent:
150 self.poller = ParentPollerUnix()
150 self.poller = ParentPollerUnix()
151
151
152 def _bind_socket(self, s, port):
152 def _bind_socket(self, s, port):
153 iface = 'tcp://%s' % self.ip
153 iface = 'tcp://%s' % self.ip
154 if port <= 0:
154 if port <= 0:
155 port = s.bind_to_random_port(iface)
155 port = s.bind_to_random_port(iface)
156 else:
156 else:
157 s.bind(iface + ':%i'%port)
157 s.bind(iface + ':%i'%port)
158 return port
158 return port
159
159
160 def load_connection_file(self):
160 def load_connection_file(self):
161 """load ip/port/hmac config from JSON connection file"""
161 """load ip/port/hmac config from JSON connection file"""
162 try:
162 try:
163 fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
163 fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
164 except IOError:
164 except IOError:
165 self.log.debug("Connection file not found: %s", self.connection_file)
165 self.log.debug("Connection file not found: %s", self.connection_file)
166 return
166 return
167 self.log.debug(u"Loading connection file %s", fname)
167 self.log.debug(u"Loading connection file %s", fname)
168 with open(fname) as f:
168 with open(fname) as f:
169 s = f.read()
169 s = f.read()
170 cfg = json.loads(s)
170 cfg = json.loads(s)
171 if self.ip == LOCALHOST and 'ip' in cfg:
171 if self.ip == LOCALHOST and 'ip' in cfg:
172 # not overridden by config or cl_args
172 # not overridden by config or cl_args
173 self.ip = cfg['ip']
173 self.ip = cfg['ip']
174 for channel in ('hb', 'shell', 'iopub', 'stdin'):
174 for channel in ('hb', 'shell', 'iopub', 'stdin'):
175 name = channel + '_port'
175 name = channel + '_port'
176 if getattr(self, name) == 0 and name in cfg:
176 if getattr(self, name) == 0 and name in cfg:
177 # not overridden by config or cl_args
177 # not overridden by config or cl_args
178 setattr(self, name, cfg[name])
178 setattr(self, name, cfg[name])
179 if 'key' in cfg:
179 if 'key' in cfg:
180 self.config.Session.key = str_to_bytes(cfg['key'])
180 self.config.Session.key = str_to_bytes(cfg['key'])
181
181
182 def write_connection_file(self):
182 def write_connection_file(self):
183 """write connection info to JSON file"""
183 """write connection info to JSON file"""
184 if os.path.basename(self.connection_file) == self.connection_file:
184 if os.path.basename(self.connection_file) == self.connection_file:
185 cf = os.path.join(self.profile_dir.security_dir, self.connection_file)
185 cf = os.path.join(self.profile_dir.security_dir, self.connection_file)
186 else:
186 else:
187 cf = self.connection_file
187 cf = self.connection_file
188 write_connection_file(cf, ip=self.ip, key=self.session.key,
188 write_connection_file(cf, ip=self.ip, key=self.session.key,
189 shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port,
189 shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port,
190 iopub_port=self.iopub_port)
190 iopub_port=self.iopub_port)
191
191
192 def init_connection_file(self):
192 def init_connection_file(self):
193 if not self.connection_file:
193 if not self.connection_file:
194 self.connection_file = "kernel-%s.json"%os.getpid()
194 self.connection_file = "kernel-%s.json"%os.getpid()
195 try:
195 try:
196 self.load_connection_file()
196 self.load_connection_file()
197 except Exception:
197 except Exception:
198 self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True)
198 self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True)
199 self.exit(1)
199 self.exit(1)
200
200
201 def init_sockets(self):
201 def init_sockets(self):
202 # Create a context, a session, and the kernel sockets.
202 # Create a context, a session, and the kernel sockets.
203 self.log.info("Starting the kernel at pid: %i", os.getpid())
203 self.log.info("Starting the kernel at pid: %i", os.getpid())
204 context = zmq.Context.instance()
204 context = zmq.Context.instance()
205 # Uncomment this to try closing the context.
205 # Uncomment this to try closing the context.
206 # atexit.register(context.term)
206 # atexit.register(context.term)
207
207
208 self.shell_socket = context.socket(zmq.ROUTER)
208 self.shell_socket = context.socket(zmq.ROUTER)
209 self.shell_port = self._bind_socket(self.shell_socket, self.shell_port)
209 self.shell_port = self._bind_socket(self.shell_socket, self.shell_port)
210 self.log.debug("shell ROUTER Channel on port: %i"%self.shell_port)
210 self.log.debug("shell ROUTER Channel on port: %i"%self.shell_port)
211
211
212 self.iopub_socket = context.socket(zmq.PUB)
212 self.iopub_socket = context.socket(zmq.PUB)
213 self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port)
213 self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port)
214 self.log.debug("iopub PUB Channel on port: %i"%self.iopub_port)
214 self.log.debug("iopub PUB Channel on port: %i"%self.iopub_port)
215
215
216 self.stdin_socket = context.socket(zmq.ROUTER)
216 self.stdin_socket = context.socket(zmq.ROUTER)
217 self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port)
217 self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port)
218 self.log.debug("stdin ROUTER Channel on port: %i"%self.stdin_port)
218 self.log.debug("stdin ROUTER Channel on port: %i"%self.stdin_port)
219
219
220 self.heartbeat = Heartbeat(context, (self.ip, self.hb_port))
220 self.heartbeat = Heartbeat(context, (self.ip, self.hb_port))
221 self.hb_port = self.heartbeat.port
221 self.hb_port = self.heartbeat.port
222 self.log.debug("Heartbeat REP Channel on port: %i"%self.hb_port)
222 self.log.debug("Heartbeat REP Channel on port: %i"%self.hb_port)
223
223
224 # Helper to make it easier to connect to an existing kernel.
224 # Helper to make it easier to connect to an existing kernel.
225 # set log-level to critical, to make sure it is output
225 # set log-level to critical, to make sure it is output
226 self.log.critical("To connect another client to this kernel, use:")
226 self.log.critical("To connect another client to this kernel, use:")
227
227
228 basename = os.path.basename(self.connection_file)
228 basename = os.path.basename(self.connection_file)
229 if basename == self.connection_file or \
229 if basename == self.connection_file or \
230 os.path.dirname(self.connection_file) == self.profile_dir.security_dir:
230 os.path.dirname(self.connection_file) == self.profile_dir.security_dir:
231 # use shortname
231 # use shortname
232 tail = basename
232 tail = basename
233 if self.profile != 'default':
233 if self.profile != 'default':
234 tail += " --profile %s" % self.profile
234 tail += " --profile %s" % self.profile
235 else:
235 else:
236 tail = self.connection_file
236 tail = self.connection_file
237 self.log.critical("--existing %s", tail)
237 self.log.critical("--existing %s", tail)
238
238
239
239
240 self.ports = dict(shell=self.shell_port, iopub=self.iopub_port,
240 self.ports = dict(shell=self.shell_port, iopub=self.iopub_port,
241 stdin=self.stdin_port, hb=self.hb_port)
241 stdin=self.stdin_port, hb=self.hb_port)
242
242
243 def init_session(self):
243 def init_session(self):
244 """create our session object"""
244 """create our session object"""
245 default_secure(self.config)
245 default_secure(self.config)
246 self.session = Session(config=self.config, username=u'kernel')
246 self.session = Session(config=self.config, username=u'kernel')
247
247
248 def init_blackhole(self):
248 def init_blackhole(self):
249 """redirects stdout/stderr to devnull if necessary"""
249 """redirects stdout/stderr to devnull if necessary"""
250 if self.no_stdout or self.no_stderr:
250 if self.no_stdout or self.no_stderr:
251 blackhole = file(os.devnull, 'w')
251 blackhole = file(os.devnull, 'w')
252 if self.no_stdout:
252 if self.no_stdout:
253 sys.stdout = sys.__stdout__ = blackhole
253 sys.stdout = sys.__stdout__ = blackhole
254 if self.no_stderr:
254 if self.no_stderr:
255 sys.stderr = sys.__stderr__ = blackhole
255 sys.stderr = sys.__stderr__ = blackhole
256
256
257 def init_io(self):
257 def init_io(self):
258 """Redirect input streams and set a display hook."""
258 """Redirect input streams and set a display hook."""
259 if self.outstream_class:
259 if self.outstream_class:
260 outstream_factory = import_item(str(self.outstream_class))
260 outstream_factory = import_item(str(self.outstream_class))
261 sys.stdout = outstream_factory(self.session, self.iopub_socket, u'stdout')
261 sys.stdout = outstream_factory(self.session, self.iopub_socket, u'stdout')
262 sys.stderr = outstream_factory(self.session, self.iopub_socket, u'stderr')
262 sys.stderr = outstream_factory(self.session, self.iopub_socket, u'stderr')
263 if self.displayhook_class:
263 if self.displayhook_class:
264 displayhook_factory = import_item(str(self.displayhook_class))
264 displayhook_factory = import_item(str(self.displayhook_class))
265 sys.displayhook = displayhook_factory(self.session, self.iopub_socket)
265 sys.displayhook = displayhook_factory(self.session, self.iopub_socket)
266
266
267 def init_kernel(self):
267 def init_kernel(self):
268 """Create the Kernel object itself"""
268 """Create the Kernel object itself"""
269 kernel_factory = import_item(str(self.kernel_class))
269 kernel_factory = import_item(str(self.kernel_class))
270 self.kernel = kernel_factory(config=self.config, session=self.session,
270 self.kernel = kernel_factory(config=self.config, session=self.session,
271 shell_socket=self.shell_socket,
271 shell_socket=self.shell_socket,
272 iopub_socket=self.iopub_socket,
272 iopub_socket=self.iopub_socket,
273 stdin_socket=self.stdin_socket,
273 stdin_socket=self.stdin_socket,
274 log=self.log
274 log=self.log
275 )
275 )
276 self.kernel.record_ports(self.ports)
276 self.kernel.record_ports(self.ports)
277
277
278 @catch_config
278 def initialize(self, argv=None):
279 def initialize(self, argv=None):
279 super(KernelApp, self).initialize(argv)
280 super(KernelApp, self).initialize(argv)
280 self.init_blackhole()
281 self.init_blackhole()
281 self.init_connection_file()
282 self.init_connection_file()
282 self.init_session()
283 self.init_session()
283 self.init_poller()
284 self.init_poller()
284 self.init_sockets()
285 self.init_sockets()
285 # writing connection file must be *after* init_sockets
286 # writing connection file must be *after* init_sockets
286 self.write_connection_file()
287 self.write_connection_file()
287 self.init_io()
288 self.init_io()
288 self.init_kernel()
289 self.init_kernel()
289
290
290 def start(self):
291 def start(self):
291 self.heartbeat.start()
292 self.heartbeat.start()
292 if self.poller is not None:
293 if self.poller is not None:
293 self.poller.start()
294 self.poller.start()
294 try:
295 try:
295 self.kernel.start()
296 self.kernel.start()
296 except KeyboardInterrupt:
297 except KeyboardInterrupt:
297 pass
298 pass
298
299
General Comments 0
You need to be logged in to leave comments. Login now